Repository: duckdb/ducklake (2026-06-05T20:17:04Z)

Issue:
CHECKPOINT handles missing S3 files inconsistently depending on credential method
### What happens?
 
When a DuckLake catalog references Parquet files that no longer exist in S3, the behavior of `CHECKPOINT` differs depending on how S3 credentials are configured:
 
- `CREATE SECRET`: CHECKPOINT succeeds. Missing files are handled gracefully.
- `SET s3_*` statements: CHECKPOINT fails with HTTP 403.
 
The individual `ducklake_cleanup_old_files()` CALL also succeeds with either credential method — the inconsistency is specific to CHECKPOINT.

- Data storage: AWS S3

### To Reproduce

```sql
ATTACH 'ducklake:postgres:host=... dbname=...' AS lake
  (DATA_PATH 's3://bucket/prefix/', METADATA_SCHEMA 'lake_catalog');
 
CREATE TABLE lake.main.test (id INT);
INSERT INTO lake.main.test VALUES (1);
DELETE FROM lake.main.test WHERE id = 1;
INSERT INTO lake.main.test VALUES (2);
 
CALL ducklake_expire_snapshots('lake', older_than => now());
CALL ducklake_rewrite_data_files('lake', delete_threshold => 0.0);
 
-- Manually remove a file scheduled for cleanup:
-- aws s3 rm s3://bucket/prefix/main/test/ducklake-<uuid>-delete.parquet
```
 
### With SET statements — fails:
```sql
SET s3_region = 'us-east-2';
SET s3_access_key_id = '...';
SET s3_secret_access_key = '...';
 
CHECKPOINT lake;
-- Error: Failed to perform CHECKPOINT; in DuckLake: Failed to get files
-- scheduled for deletion from DuckLake: HTTP GET error reading
-- 's3://bucket/prefix/**' (HTTP 403 Forbidden)
```
 
### With CREATE SECRET — succeeds:
```sql
CREATE SECRET (TYPE S3, KEY_ID '...', SECRET '...', REGION 'us-east-2');
 
CHECKPOINT lake;
-- No error. Missing file handled gracefully.
```
 
### Direct CALL — also succeeds with either method:
```sql
CALL ducklake_cleanup_old_files('lake', cleanup_all => true);
-- No error. Reports file cleaned, removes catalog reference.
```

### OS:

Ubuntu

### DuckDB Version:

DuckDB v1.5.2 and v1.5.3 (both reproduce)

### DuckLake Version:

415a9ebd

### DuckDB Client:

DuckDB CLI v1.5.2

### Hardware:

N/A

### Full Name:

Patrick Kilgore

### Affiliation:

Gravity

### What is the latest build you tested with? If possible, we recommend testing with the latest nightly build.

I have tested with a stable release

### Did you include all relevant data sets for reproducing the issue?

Not applicable - the reproduction does not require a data set

### Did you include all code required to reproduce the issue?

- [x] Yes, I have

### Did you include all relevant configuration (e.g., CPU architecture, Python version, Linux distribution) to reproduce the issue?

- [x] Yes, I have


Discussion:
I gave the nightly a shot, but I couldn't even get ducklake to install.



Patch:
diff --git a/src/common/ducklake_util.cpp b/src/common/ducklake_util.cpp
index c8b43161d37..302bf6da951 100644
--- a/src/common/ducklake_util.cpp
+++ b/src/common/ducklake_util.cpp
@@ -10,6 +10,8 @@
 #include "duckdb/planner/filter/expression_filter.hpp"
 #include "duckdb/planner/filter/table_filter_functions.hpp"
 #include "duckdb/function/scalar/variant_utils.hpp"
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/main/database.hpp"
 
 #include <cmath>
 
@@ -448,4 +450,23 @@ string DuckLakeUtil::ChunkRowToSQL(DuckLakeMetadataManager &metadata_manager, Cl
 	return result;
 }
 
+void DuckLakeUtil::CopyExtensionSettings(ClientContext &from, ClientContext &to) {
+	auto &db_config = DBConfig::GetConfig(from);
+	for (auto &entry : db_config.GetExtensionSettings()) {
+		auto &option = entry.second;
+		if (!option.setting_index.IsValid()) {
+			continue;
+		}
+		auto setting_index = option.setting_index.GetIndex();
+		if (!from.config.user_settings.IsSet(setting_index)) {
+			continue;
+		}
+		Value value;
+		if (!from.TryGetCurrentSetting(entry.first, value)) {
+			continue;
+		}
+		to.config.user_settings.SetUserSetting(setting_index, value);
+	}
+}
+
 } // namespace duckdb
diff --git a/src/include/common/ducklake_util.hpp b/src/include/common/ducklake_util.hpp
index 842be2e86a8..b115205c2b4 100644
--- a/src/include/common/ducklake_util.hpp
+++ b/src/include/common/ducklake_util.hpp
@@ -66,6 +66,10 @@ class DuckLakeUtil {
 	                            idx_t row);
 	//! Throws if any column in the list conflicts with inlined data system columns
 	static void ValidateNoInlinedSystemColumns(const ColumnList &columns, const string &table_name = "");
+
+	//! Copy extension-registered settings from one context onto another. Core engine settings
+	//! are not copied.
+	static void CopyExtensionSettings(ClientContext &from, ClientContext &to);
 };
 
 } // namespace duckdb
diff --git a/src/storage/ducklake_checkpoint.cpp b/src/storage/ducklake_checkpoint.cpp
index 88cec016ea0..395ee0bdb7b 100644
--- a/src/storage/ducklake_checkpoint.cpp
+++ b/src/storage/ducklake_checkpoint.cpp
@@ -1,16 +1,19 @@
 #include "ducklake_extension.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/connection.hpp"
 #include "storage/ducklake_transaction_manager.hpp"
 #include "storage/ducklake_catalog.hpp"
 #include "storage/ducklake_schema_entry.hpp"
 #include "storage/ducklake_table_entry.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/common/sql_identifier.hpp"
+#include "common/ducklake_util.hpp"
 
 namespace duckdb {
 
 void DuckLakeTransactionManager::Checkpoint(ClientContext &context, bool force) {
 	auto conn = make_uniq<Connection>(ducklake_catalog.GetDatabase());
+	DuckLakeUtil::CopyExtensionSettings(context, *conn->context);
 	// 1. We first flush inlined data, since these can generate many files, which would be important for compaction
 	// 2. We expire snapshots since these can create more compaction opportunities.
 	// 3. We call the compaction functions, merge_adjacent and rewrite, unclear what is the best order here.
diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp
index d06bb53b689..f7b4f892355 100644
--- a/src/storage/ducklake_transaction.cpp
+++ b/src/storage/ducklake_transaction.cpp
@@ -760,6 +760,10 @@ Connection &DuckLakeTransaction::GetConnection() {
 	lock_guard<mutex> lock(connection_lock);
 	if (!connection) {
 		connection = make_uniq<Connection>(db);
+		auto caller_context = context.lock();
+		if (caller_context) {
+			DuckLakeUtil::CopyExtensionSettings(*caller_context, *connection->context);
+		}
 		// set the search path to the metadata catalog
 		auto &client_data = ClientData::Get(*connection->context);
 		// ensure we are only looking in the ducklake catalog schema during querying


Test patch:
diff --git a/test/sql/checkpoint/checkpoint_set_credentials.test b/test/sql/checkpoint/checkpoint_set_credentials.test
new file mode 100644
index 00000000000..c7e7885a78d
--- /dev/null
+++ b/test/sql/checkpoint/checkpoint_set_credentials.test
@@ -0,0 +1,100 @@
+# name: test/sql/checkpoint/checkpoint_set_credentials.test
+# description: Test that CHECKPOINT works when S3 credentials are configured via SET statements (not CREATE SECRET)
+# group: [checkpoint]
+
+require ducklake
+
+require parquet
+
+require httpfs
+
+require icu
+
+require-env S3_TEST_SERVER_AVAILABLE 1
+
+test-env DUCKLAKE_CONNECTION {TEST_DIR}/{UUID}.db
+
+# Drop any pre-existing S3 secret (e.g. from the minio test config's on_init)
+# so that SET statements are the only credential source.
+statement ok
+DROP SECRET IF EXISTS __default_s3;
+
+# Configure S3 credentials via SET statements (not CREATE SECRET)
+statement ok
+SET s3_access_key_id = 'admin';
+
+statement ok
+SET s3_secret_access_key = 'password';
+
+statement ok
+SET s3_endpoint = '127.0.0.1:9000';
+
+statement ok
+SET s3_url_style = 'path';
+
+statement ok
+SET s3_use_ssl = false;
+
+statement ok
+ATTACH 'ducklake:{DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH 's3://mybucket/checkpoint_set_creds')
+
+statement ok
+USE ducklake;
+
+# Create table with expire_snapshot/delete old_file
+statement ok
+CREATE TABLE test(i INTEGER)
+
+statement ok
+INSERT INTO test FROM range(1000)
+
+statement ok
+DELETE FROM test
+
+statement ok
+DROP TABLE test
+
+statement ok
+CALL ducklake_expire_snapshots('ducklake', dry_run => false, versions => [2])
+
+# Create table with inlined and merge adjacent possibility
+statement ok
+CREATE TABLE t (a INTEGER)
+
+statement ok
+INSERT INTO t VALUES (1), (2)
+
+statement ok
+INSERT INTO t VALUES (2), (3)
+
+statement ok
+INSERT INTO t VALUES (4)
+
+# Create table with data rewrite
+statement ok
+CREATE TABLE t_2 (a INTEGER)
+
+statement ok
+INSERT INTO t_2 SELECT i FROM range(100) t(i)
+
+statement ok
+DELETE FROM t_2 WHERE a < 98;
+
+# Create orphan file
+statement ok
+COPY t_2 TO 's3://mybucket/checkpoint_set_creds/main/t/orphan.parquet';
+
+statement ok
+CALL ducklake.set_option('delete_older_than', '1 millisecond');
+
+# CHECKPOINT must succeed with credentials configured via SET.
+statement ok
+CHECKPOINT;
+
+# Verify orphan was cleaned up
+query I
+SELECT COUNT(*)
+FROM GLOB('s3://mybucket/checkpoint_set_creds/main/**')
+WHERE file LIKE '%orphan%';
+----
+0
\ No newline at end of file