From 405624294e0613242e41bdd41b731bc36ce61c56 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Fri, 10 Jan 2025 18:41:21 +0100 Subject: [PATCH 01/11] feat(compute): Add pg_duckdb extension v0.2.0 --- compute/compute-node.Dockerfile | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 303daec24020..92af9e296576 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -34,11 +34,11 @@ RUN case $DEBIAN_VERSION in \ echo "Unknown Debian version ${DEBIAN_VERSION}" && exit 1 \ ;; \ esac && \ - apt update && \ + apt update && \ apt install --no-install-recommends --no-install-suggests -y \ ninja-build git autoconf automake libtool build-essential bison flex libreadline-dev \ zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget ca-certificates pkg-config libssl-dev \ - libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd \ + libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd g++ \ $VERSION_INSTALLS \ && apt clean && rm -rf /var/lib/apt/lists/* @@ -1176,6 +1176,25 @@ RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.0/p make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_mooncake.control +######################################################################################### +# +# Layer "pg-duckdb-pg-build" +# compile pg_duckdb extension +# +######################################################################################### + +FROM build-deps AS pg-duckdb-pg-build +ARG PG_VERSION +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +ENV PATH="/usr/local/pgsql/bin/:$PATH" + +# pg_duckdb build requires source dir to be a git repo to get submodules +RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git pg_duckdb-src && \ + cd pg_duckdb-src && \ + make install -j $(getconf _NPROCESSORS_ONLN) && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control + ######################################################################################### # # Layer "pg_repack" @@ -1241,6 +1260,7 @@ COPY --from=pg-ivm-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-partman-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-mooncake-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-repack-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=pg-duckdb-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ From 1ae0ab255469d8dd2a6f65b74173b02fcded49dc Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Thu, 16 Jan 2025 10:17:37 +0100 Subject: [PATCH 02/11] in Neon we don't allow superuser so we need to grant some superuser only functions to neon_superuser --- compute/compute-node.Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 92af9e296576..800c07d75eb9 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1190,10 +1190,22 @@ COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ ENV PATH="/usr/local/pgsql/bin/:$PATH" # pg_duckdb build requires source dir to be a git repo to get submodules +# allow neon_superuser to execute some functions that in pg_duckdb are available to superuser only +# cache management functions duckdb.cache(), duckdb.cache_info(), duckdb.cache_delete() +# extension management function duckdb.install_extension() +# for debugging purposes raw query and reset ddb duckdb.raw_query(), duckdb.recycle_ddb() RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git pg_duckdb-src && \ cd pg_duckdb-src && \ make install -j $(getconf _NPROCESSORS_ONLN) && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ + file=/usr/local/pgsql/share/extension/pg_duckdb--0.2.0--0.3.0.sql && \ + echo 'GRANT ALL ON FUNCTION duckdb.cache(TEXT, TEXT) TO neon_superuser;' >> $file && \ + echo 'GRANT ALL ON FUNCTION duckdb.cache_info() TO neon_superuser;' >> $file && \ + echo 'GRANT ALL ON FUNCTION duckdb.cache_delete(cache_key TEXT) TO neon_superuser;' >> $file && \ + echo 'GRANT ALL ON FUNCTION duckdb.install_extension(TEXT) TO neon_superuser;' >> $file && \ + echo 'GRANT ALL ON FUNCTION duckdb.raw_query(TEXT) TO neon_superuser' >> $file && \ + echo 'GRANT ALL ON PROCEDURE duckdb.recycle_ddb() TO neon_superuser;' >> $file + ######################################################################################### # From d6a4896070fb1bfb08b4a207e365e30b1cf3dda4 Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Thu, 16 Jan 2025 10:33:12 +0100 Subject: [PATCH 03/11] use correct contrib sql file --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index a81bc87da481..a0f97644ecb0 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1129,7 +1129,7 @@ RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git cd pg_duckdb-src && \ make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ - file=/usr/local/pgsql/share/extension/pg_duckdb--0.2.0--0.3.0.sql && \ + file=/usr/local/pgsql/share/extension/pg_duckdb--0.1.0--0.2.0.sql && \ echo 'GRANT ALL ON FUNCTION duckdb.cache(TEXT, TEXT) TO neon_superuser;' >> $file && \ echo 'GRANT ALL ON FUNCTION duckdb.cache_info() TO neon_superuser;' >> $file && \ echo 'GRANT ALL ON FUNCTION duckdb.cache_delete(cache_key TEXT) TO neon_superuser;' >> $file && \ From 1ffd697090e0c284c282c3c84667806b97c2db88 Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Thu, 16 Jan 2025 15:20:53 +0100 Subject: [PATCH 04/11] forgot semicolon --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index a0f97644ecb0..ad3092f1f2a0 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1134,7 +1134,7 @@ RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git echo 'GRANT ALL ON FUNCTION duckdb.cache_info() TO neon_superuser;' >> $file && \ echo 'GRANT ALL ON FUNCTION duckdb.cache_delete(cache_key TEXT) TO neon_superuser;' >> $file && \ echo 'GRANT ALL ON FUNCTION duckdb.install_extension(TEXT) TO neon_superuser;' >> $file && \ - echo 'GRANT ALL ON FUNCTION duckdb.raw_query(TEXT) TO neon_superuser' >> $file && \ + echo 'GRANT ALL ON FUNCTION duckdb.raw_query(TEXT) TO neon_superuser;' >> $file && \ echo 'GRANT ALL ON PROCEDURE duckdb.recycle_ddb() TO neon_superuser;' >> $file From b6c010035f51548796503800f7844d138881963f Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Fri, 17 Jan 2025 11:20:55 +0100 Subject: [PATCH 05/11] patch for duckdb that allows to install extension httpfs even if disabled_filesystems='LocalFileSystem' --- compute/compute-node.Dockerfile | 6 ++ compute/patches/duckdb-v1-1-3.patch | 97 +++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 compute/patches/duckdb-v1-1-3.patch diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index ad3092f1f2a0..037ecc6f2ff7 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1117,9 +1117,11 @@ RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.0/p FROM build-deps AS pg-duckdb-pg-build ARG PG_VERSION COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY compute/patches/duckdb-v1-1-3.patch /duckdb-v1-1-3.patch ENV PATH="/usr/local/pgsql/bin/:$PATH" + # pg_duckdb build requires source dir to be a git repo to get submodules # allow neon_superuser to execute some functions that in pg_duckdb are available to superuser only # cache management functions duckdb.cache(), duckdb.cache_info(), duckdb.cache_delete() @@ -1127,6 +1129,10 @@ ENV PATH="/usr/local/pgsql/bin/:$PATH" # for debugging purposes raw query and reset ddb duckdb.raw_query(), duckdb.recycle_ddb() RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git pg_duckdb-src && \ cd pg_duckdb-src && \ + git submodule update --init --recursive && \ + cd thirdparty/duckdb && \ + patch -p1 < /duckdb-v1-1-3.patch && \ + cd ../.. && \ make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ file=/usr/local/pgsql/share/extension/pg_duckdb--0.1.0--0.2.0.sql && \ diff --git a/compute/patches/duckdb-v1-1-3.patch b/compute/patches/duckdb-v1-1-3.patch new file mode 100644 index 000000000000..8c97a9ccdd56 --- /dev/null +++ b/compute/patches/duckdb-v1-1-3.patch @@ -0,0 +1,97 @@ +diff --git a/src/common/virtual_file_system.cpp b/src/common/virtual_file_system.cpp +index 74892a4e05..60e9e7af88 100644 +--- a/src/common/virtual_file_system.cpp ++++ b/src/common/virtual_file_system.cpp +@@ -173,7 +173,9 @@ void VirtualFileSystem::SetDisabledFileSystems(const vector &names) { + + FileSystem &VirtualFileSystem::FindFileSystem(const string &path) { + auto &fs = FindFileSystemInternal(path); +- if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) { ++ // we allow LocalFileSystem access to duckdb itself (by specifiying trustedContext=true) ++ // but not to duckdb users if disabled_file_systems='LocalFileSystem' ++ if (!isTrustedContext && !disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) { + throw PermissionException("File system %s has been disabled by configuration", fs.GetName()); + } + return fs; +diff --git a/src/include/duckdb/common/file_system.hpp b/src/include/duckdb/common/file_system.hpp +index 0b83c4f393..58d70889cf 100644 +--- a/src/include/duckdb/common/file_system.hpp ++++ b/src/include/duckdb/common/file_system.hpp +@@ -256,6 +256,30 @@ public: + DynamicCastCheck(this); + return reinterpret_cast(*this); + } ++public: ++ DUCKDB_API virtual void setTrusted(){ ++ this->isTrustedContext = true; ++ } ++ DUCKDB_API virtual void setUntrusted(){ ++ this->isTrustedContext = false; ++ } ++ ++public: ++ class TrustedContext { ++ public: ++ TrustedContext(FileSystem &fs) : fileSystem(fs) { ++ fileSystem.setTrusted(); ++ } ++ ~TrustedContext() { ++ fileSystem.setUntrusted(); ++ } ++ ++ private: ++ FileSystem &fileSystem; ++ }; ++ ++protected: ++ bool isTrustedContext = false; + }; + + } // namespace duckdb +diff --git a/src/include/duckdb/common/opener_file_system.hpp b/src/include/duckdb/common/opener_file_system.hpp +index 2d35512b21..d1597173ee 100644 +--- a/src/include/duckdb/common/opener_file_system.hpp ++++ b/src/include/duckdb/common/opener_file_system.hpp +@@ -143,6 +143,18 @@ public: + vector ListSubSystems() override { + return GetFileSystem().ListSubSystems(); + } ++ ++private: ++ ++ virtual void setTrusted() override { ++ this->isTrustedContext = true; ++ GetFileSystem().setTrusted(); ++ } ++ ++ virtual void setUntrusted() override { ++ this->isTrustedContext = false; ++ GetFileSystem().setUntrusted(); ++ } + }; + + } // namespace duckdb +diff --git a/src/main/extension/extension_install.cpp b/src/main/extension/extension_install.cpp +index 1258d95ead..fc9ce1c77d 100644 +--- a/src/main/extension/extension_install.cpp ++++ b/src/main/extension/extension_install.cpp +@@ -57,6 +57,7 @@ const vector ExtensionHelper::PathComponents() { + } + + duckdb::string ExtensionHelper::DefaultExtensionFolder(FileSystem &fs) { ++ FileSystem::TrustedContext trusted(fs); + string home_directory = fs.GetHomeDirectory(); + // exception if the home directory does not exist, don't create whatever we think is home + if (!fs.DirectoryExists(home_directory)) { +diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp +index b0282a7103..c2765f97c6 100644 +--- a/src/main/extension/extension_load.cpp ++++ b/src/main/extension/extension_load.cpp +@@ -293,6 +293,7 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const + if (!db.config.options.enable_external_access) { + throw PermissionException("Loading external extensions is disabled through configuration"); + } ++ FileSystem::TrustedContext trusted(fs); + auto filename = fs.ConvertSeparators(extension); + + bool direct_load; From a6a42c58c3e66287779f4f9d5d12fc3f61f36021 Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Fri, 17 Jan 2025 12:00:43 +0100 Subject: [PATCH 06/11] fix typo --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 037ecc6f2ff7..86fc77b37cf2 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1130,7 +1130,7 @@ ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git pg_duckdb-src && \ cd pg_duckdb-src && \ git submodule update --init --recursive && \ - cd thirdparty/duckdb && \ + cd third_party/duckdb && \ patch -p1 < /duckdb-v1-1-3.patch && \ cd ../.. && \ make install -j $(getconf _NPROCESSORS_ONLN) && \ From 0654c03d182d69296a4092a6097d831a0b0dff5f Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Sun, 19 Jan 2025 18:11:20 +0100 Subject: [PATCH 07/11] try to statically link some important duckdb extensions --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 86fc77b37cf2..5dd8fc6957d6 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1133,7 +1133,7 @@ RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git cd third_party/duckdb && \ patch -p1 < /duckdb-v1-1-3.patch && \ cd ../.. && \ - make install -j $(getconf _NPROCESSORS_ONLN) && \ + CORE_EXTENSIONS='autocomplete;httpfs;icu;json;tpch' make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ file=/usr/local/pgsql/share/extension/pg_duckdb--0.1.0--0.2.0.sql && \ echo 'GRANT ALL ON FUNCTION duckdb.cache(TEXT, TEXT) TO neon_superuser;' >> $file && \ From 7ab475520df0620bf26b458e1d1b48592d877b06 Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Sun, 19 Jan 2025 18:13:15 +0100 Subject: [PATCH 08/11] remove unneeded tpch extension --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 5dd8fc6957d6..3641b5c34b49 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1133,7 +1133,7 @@ RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git cd third_party/duckdb && \ patch -p1 < /duckdb-v1-1-3.patch && \ cd ../.. && \ - CORE_EXTENSIONS='autocomplete;httpfs;icu;json;tpch' make install -j $(getconf _NPROCESSORS_ONLN) && \ + CORE_EXTENSIONS='autocomplete;httpfs;icu;json' make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ file=/usr/local/pgsql/share/extension/pg_duckdb--0.1.0--0.2.0.sql && \ echo 'GRANT ALL ON FUNCTION duckdb.cache(TEXT, TEXT) TO neon_superuser;' >> $file && \ From aadbdf19d6423fed57f685e19c6f68db9b682d9a Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Sun, 19 Jan 2025 18:39:15 +0100 Subject: [PATCH 09/11] previous commit caused compile errors --- compute/compute-node.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 3641b5c34b49..86fc77b37cf2 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1133,7 +1133,7 @@ RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git cd third_party/duckdb && \ patch -p1 < /duckdb-v1-1-3.patch && \ cd ../.. && \ - CORE_EXTENSIONS='autocomplete;httpfs;icu;json' make install -j $(getconf _NPROCESSORS_ONLN) && \ + make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ file=/usr/local/pgsql/share/extension/pg_duckdb--0.1.0--0.2.0.sql && \ echo 'GRANT ALL ON FUNCTION duckdb.cache(TEXT, TEXT) TO neon_superuser;' >> $file && \ From 76a4654424fa3239f0970cdf0914f9b4dfff6a1c Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Fri, 24 Jan 2025 11:53:58 +0100 Subject: [PATCH 10/11] need another patch foro pg_duckdb which has its own cached copy of httpfs_filesystem --- compute/compute-node.Dockerfile | 2 + compute/patches/pg_duckdb-0-2-0.patch | 61 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 compute/patches/pg_duckdb-0-2-0.patch diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 86fc77b37cf2..7cd165c92a98 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1118,6 +1118,7 @@ FROM build-deps AS pg-duckdb-pg-build ARG PG_VERSION COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY compute/patches/duckdb-v1-1-3.patch /duckdb-v1-1-3.patch +COPY compute/patches/pg_duckdb-0-2-0.patch /pg_duckdb-0-2-0.patch ENV PATH="/usr/local/pgsql/bin/:$PATH" @@ -1133,6 +1134,7 @@ RUN git clone --depth 1 --branch v0.2.0 https://github.com/duckdb/pg_duckdb.git cd third_party/duckdb && \ patch -p1 < /duckdb-v1-1-3.patch && \ cd ../.. && \ + patch -p1 < /pg_duckdb-0-2-0.patch && \ make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control && \ file=/usr/local/pgsql/share/extension/pg_duckdb--0.1.0--0.2.0.sql && \ diff --git a/compute/patches/pg_duckdb-0-2-0.patch b/compute/patches/pg_duckdb-0-2-0.patch new file mode 100644 index 000000000000..ab7e6ef8d3da --- /dev/null +++ b/compute/patches/pg_duckdb-0-2-0.patch @@ -0,0 +1,61 @@ +diff --git a/third_party/cached_httpfs/http_file_cache.cpp b/third_party/cached_httpfs/http_file_cache.cpp +index 813db57..c6e4c3a 100644 +--- a/third_party/cached_httpfs/http_file_cache.cpp ++++ b/third_party/cached_httpfs/http_file_cache.cpp +@@ -4,20 +4,22 @@ namespace duckdb { + + CachedFile::CachedFile(const string &cache_dir, FileSystem &fs, const string &key, bool cache_file) : cache_directory(cache_dir), fs(fs) { + file_name = cache_dir + "/" + key; +- ++ FileSystem::TrustedContext trusted1(fs); + GetDirectoryCacheLock(cache_dir); + + FileOpenFlags flags = + FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS | FileLockType::READ_LOCK; ++ FileSystem::TrustedContext trusted2(fs); + handle = fs.OpenFile(file_name, flags); + if (handle) { + initialized = true; + size = handle->GetFileSize(); + } else if (cache_file) { + flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE | FileLockType::WRITE_LOCK; ++ FileSystem::TrustedContext trusted3(fs); + handle = fs.OpenFile(file_name, flags); + } +- ++ FileSystem::TrustedContext trusted4(fs); + ReleaseDirectoryCacheLock(); + } + +@@ -30,6 +32,7 @@ void CachedFile::GetDirectoryCacheLock(const string &cache_dir) { + FileOpenFlags flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE | + FileFlags::FILE_FLAGS_EXCLUSIVE_CREATE | FileFlags::FILE_FLAGS_NULL_IF_EXISTS | + FileLockType::WRITE_LOCK; ++ FileSystem::TrustedContext trusted(fs); + directory_lock_handle = fs.OpenFile(lock_file, flags); + if (directory_lock_handle == nullptr) { + flags = FileFlags::FILE_FLAGS_WRITE | FileLockType::WRITE_LOCK; +@@ -52,6 +55,7 @@ void CachedFileHandle::WriteMetadata(const string &cache_key, const string &remo + D_ASSERT(!file->initialized); + string metadata_file_name = file->cache_directory + "/" + cache_key + ".meta"; + FileOpenFlags flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE | FileLockType::WRITE_LOCK; ++ FileSystem::TrustedContext trusted(file->fs); + auto handle = file->fs.OpenFile(metadata_file_name, flags); + auto cached_file_timestamp = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + string metadata_info = cache_key + "," + remote_path + "," + std::to_string(total_size) + "," + std::to_string(cached_file_timestamp); +@@ -66,6 +70,7 @@ void CachedFileHandle::SetInitialized(idx_t total_size) { + file->initialized = true; + file->handle->Close(); + FileOpenFlags flags = FileFlags::FILE_FLAGS_READ | FileLockType::READ_LOCK; ++ FileSystem::TrustedContext trusted(file->fs); + file->handle = file->fs.OpenFile(file->file_name, flags); + } + +@@ -93,6 +98,7 @@ void CachedFileHandle::Read(void *buffer, idx_t length, idx_t offset) { + //! Get cache entry, create if not exists only if caching is enabled + shared_ptr HTTPFileCache::GetCachedFile(const string &cache_dir, const string &key, bool cache_file) { + lock_guard lock(cached_files_mutex); ++ FileSystem::TrustedContext trusted(db->GetFileSystem()); + auto it = cached_files.find(key); + if (it != cached_files.end()) { + return it->second; From fa2657da9e2f4d33e3ebe777284ee681aa0b1194 Mon Sep 17 00:00:00 2001 From: BodoBolero Date: Sat, 25 Jan 2025 18:00:29 +0100 Subject: [PATCH 11/11] to install extensions we need additional trusted context (use case: SELECT duckdb.install_extension('iceberg') ;) --- compute/patches/duckdb-v1-1-3.patch | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/compute/patches/duckdb-v1-1-3.patch b/compute/patches/duckdb-v1-1-3.patch index 8c97a9ccdd56..13cb96cfc08d 100644 --- a/compute/patches/duckdb-v1-1-3.patch +++ b/compute/patches/duckdb-v1-1-3.patch @@ -72,7 +72,7 @@ index 2d35512b21..d1597173ee 100644 } // namespace duckdb diff --git a/src/main/extension/extension_install.cpp b/src/main/extension/extension_install.cpp -index 1258d95ead..fc9ce1c77d 100644 +index 1258d95ead..def1ba71cb 100644 --- a/src/main/extension/extension_install.cpp +++ b/src/main/extension/extension_install.cpp @@ -57,6 +57,7 @@ const vector ExtensionHelper::PathComponents() { @@ -83,6 +83,31 @@ index 1258d95ead..fc9ce1c77d 100644 string home_directory = fs.GetHomeDirectory(); // exception if the home directory does not exist, don't create whatever we think is home if (!fs.DirectoryExists(home_directory)) { +@@ -84,6 +85,7 @@ string ExtensionHelper::ExtensionDirectory(DatabaseInstance &db, FileSystem &fs) + extension_directory = DefaultExtensionFolder(fs); + } + { ++ FileSystem::TrustedContext trusted(fs); + extension_directory = fs.ConvertSeparators(extension_directory); + // expand ~ in extension directory + extension_directory = fs.ExpandPath(extension_directory); +@@ -107,6 +109,7 @@ string ExtensionHelper::ExtensionDirectory(DatabaseInstance &db, FileSystem &fs) + + auto path_components = PathComponents(); + for (auto &path_ele : path_components) { ++ FileSystem::TrustedContext trusted(fs); + extension_directory = fs.JoinPath(extension_directory, path_ele); + if (!fs.DirectoryExists(extension_directory)) { + fs.CreateDirectory(extension_directory); +@@ -538,7 +541,7 @@ ExtensionHelper::InstallExtensionInternal(DatabaseInstance &db, FileSystem &fs, + if (!db.config.options.enable_external_access) { + throw PermissionException("Installing extensions is disabled through configuration"); + } +- ++ FileSystem::TrustedContext trusted(fs); + auto extension_name = ApplyExtensionAlias(fs.ExtractBaseName(extension)); + string local_extension_path = fs.JoinPath(local_path, extension_name + ".duckdb_extension"); + string temp_path = local_extension_path + ".tmp-" + UUID::ToString(UUID::GenerateRandomUUID()); diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp index b0282a7103..c2765f97c6 100644 --- a/src/main/extension/extension_load.cpp