Repository: datafusion-contrib/datafusion-table-providers
Pull request #247: Upgrade datafusion to version 45
Author: login: nuno-faria
Created: 2025-03-03T13:07:53Z | Merged: 2025-03-11T14:35:17Z



Diff:
diff --git a/Cargo.toml b/Cargo.toml
index 509f5384..ef0ef7d9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,32 +8,32 @@ license = "Apache-2.0"
 description = "Extend the capabilities of DataFusion to support additional data sources via implementations of the `TableProvider` trait."
 
 [dependencies]
-arrow-flight = { version = "53", optional = true, features = [
+arrow-flight = { version = "54.2.1", optional = true, features = [
   "flight-sql-experimental",
   "tls",
 ] }
-arrow-odbc = { version = "=14.0.1", optional = true }
+arrow-odbc = { version = "=14.2.0", optional = true }
 async-stream = { version = "0.3", optional = true }
 async-trait = "0.1"
 bb8 = { version = "0.9", optional = true }
 bb8-postgres = { version = "0.9", optional = true }
 bigdecimal = "0.4"
 byteorder = "1.5.0"
-chrono = ">= 0.4.34, < 0.4.40"
+chrono = "0.4"
 dashmap = "6.1.0"
-datafusion = { version = "44", default-features = false }
-datafusion-federation = { version = "=0.3.4", features = [
+datafusion = { version = "45", default-features = false }
+datafusion-federation = { version = "=0.3.6", features = [
   "sql",
 ], optional = true }
-datafusion-proto = { version = "44", optional = true }
-duckdb = { version = "=1.1.1", features = [
+datafusion-proto = { version = "45", optional = true }
+duckdb = { version = "=1.2.0", features = [
   "bundled",
   "r2d2",
   "vtab",
   "vtab-arrow",
   "appender-arrow",
 ], optional = true }
-libduckdb-sys = { version = "=1.1.1", optional = true }
+libduckdb-sys = { version = "=1.2.0", optional = true }
 dyn-clone = { version = "1.0", optional = true }
 fallible-iterator = "0.3.0"
 fundu = "2.0.1"
@@ -48,7 +48,7 @@ mysql_async = { version = "0.35", features = [
 ], optional = true }
 native-tls = { version = "0.2", optional = true }
 num-bigint = "0.4"
-odbc-api = { version = "10.1", optional = true }
+odbc-api = { version = "11.1", optional = true }
 pem = { version = "3.0.4", optional = true }
 postgres-native-tls = { version = "0.5.0", optional = true }
 prost = { version = "0.13", optional = true }
diff --git a/src/duckdb/creator.rs b/src/duckdb/creator.rs
index 387f2869..0a08993c 100644
--- a/src/duckdb/creator.rs
+++ b/src/duckdb/creator.rs
@@ -404,7 +404,8 @@ pub(crate) mod tests {
 
             let arc_created_table = Arc::new(created_table);
 
-            let duckdb_sink = DuckDBDataSink::new(arc_created_table, *overwrite, None);
+            let duckdb_sink =
+                DuckDBDataSink::new(arc_created_table, *overwrite, None, Arc::clone(&schema));
             let data_sink: Arc<dyn DataSink> = Arc::new(duckdb_sink);
             let rows_written = data_sink
                 .write_all(
@@ -460,7 +461,8 @@ pub(crate) mod tests {
 
             let arc_created_table = Arc::new(created_table);
 
-            let duckdb_sink = DuckDBDataSink::new(arc_created_table, *overwrite, None);
+            let duckdb_sink =
+                DuckDBDataSink::new(arc_created_table, *overwrite, None, Arc::clone(&schema));
             let data_sink: Arc<dyn DataSink> = Arc::new(duckdb_sink);
             let rows_written = data_sink
                 .write_all(
diff --git a/src/duckdb/write.rs b/src/duckdb/write.rs
index ae90f598..1ae68e60 100644
--- a/src/duckdb/write.rs
+++ b/src/duckdb/write.rs
@@ -103,8 +103,8 @@ impl TableProvider for DuckDBTableWriter {
                 Arc::clone(&self.duckdb),
                 op == InsertOp::Overwrite,
                 self.on_conflict.clone(),
+                self.schema(),
             )),
-            self.schema(),
             None,
         )) as _)
     }
@@ -115,6 +115,7 @@ pub(crate) struct DuckDBDataSink {
     duckdb: Arc<DuckDB>,
     overwrite: bool,
     on_conflict: Option<OnConflict>,
+    schema: SchemaRef,
 }
 
 #[async_trait]
@@ -127,6 +128,10 @@ impl DataSink for DuckDBDataSink {
         None
     }
 
+    fn schema(&self) -> &SchemaRef {
+        &self.schema
+    }
+
     async fn write_all(
         &self,
         mut data: SendableRecordBatchStream,
@@ -235,11 +240,13 @@ impl DuckDBDataSink {
         duckdb: Arc<DuckDB>,
         overwrite: bool,
         on_conflict: Option<OnConflict>,
+        schema: SchemaRef,
     ) -> Self {
         Self {
             duckdb,
             overwrite,
             on_conflict,
+            schema,
         }
     }
 }
diff --git a/src/mysql/write.rs b/src/mysql/write.rs
index 6be1a51a..484b4abc 100644
--- a/src/mysql/write.rs
+++ b/src/mysql/write.rs
@@ -85,8 +85,8 @@ impl TableProvider for MySQLTableWriter {
                 Arc::clone(&self.mysql),
                 op == InsertOp::Overwrite,
                 self.on_conflict.clone(),
+                self.schema(),
             )),
-            self.schema(),
             None,
         )))
     }
@@ -96,6 +96,7 @@ pub struct MySQLDataSink {
     pub mysql: Arc<MySQL>,
     pub overwrite: bool,
     pub on_conflict: Option<OnConflict>,
+    schema: SchemaRef,
 }
 
 #[async_trait]
@@ -108,6 +109,10 @@ impl DataSink for MySQLDataSink {
         None
     }
 
+    fn schema(&self) -> &SchemaRef {
+        &self.schema
+    }
+
     async fn write_all(
         &self,
         mut data: SendableRecordBatchStream,
@@ -168,11 +173,17 @@ impl DataSink for MySQLDataSink {
 }
 
 impl MySQLDataSink {
-    pub fn new(mysql: Arc<MySQL>, overwrite: bool, on_conflict: Option<OnConflict>) -> Self {
+    pub fn new(
+        mysql: Arc<MySQL>,
+        overwrite: bool,
+        on_conflict: Option<OnConflict>,
+        schema: SchemaRef,
+    ) -> Self {
         Self {
             mysql,
             overwrite,
             on_conflict,
+            schema,
         }
     }
 }
diff --git a/src/postgres/write.rs b/src/postgres/write.rs
index a032024c..23ba25be 100644
--- a/src/postgres/write.rs
+++ b/src/postgres/write.rs
@@ -92,8 +92,8 @@ impl TableProvider for PostgresTableWriter {
                 Arc::clone(&self.postgres),
                 op == InsertOp::Overwrite,
                 self.on_conflict.clone(),
+                self.schema(),
             )),
-            self.schema(),
             None,
         )) as _)
     }
@@ -104,6 +104,7 @@ struct PostgresDataSink {
     postgres: Arc<Postgres>,
     overwrite: bool,
     on_conflict: Option<OnConflict>,
+    schema: SchemaRef,
 }
 
 #[async_trait]
@@ -116,6 +117,10 @@ impl DataSink for PostgresDataSink {
         None
     }
 
+    fn schema(&self) -> &SchemaRef {
+        &self.schema
+    }
+
     async fn write_all(
         &self,
         mut data: SendableRecordBatchStream,
@@ -174,11 +179,17 @@ impl DataSink for PostgresDataSink {
 }
 
 impl PostgresDataSink {
-    fn new(postgres: Arc<Postgres>, overwrite: bool, on_conflict: Option<OnConflict>) -> Self {
+    fn new(
+        postgres: Arc<Postgres>,
+        overwrite: bool,
+        on_conflict: Option<OnConflict>,
+        schema: SchemaRef,
+    ) -> Self {
         Self {
             postgres,
             overwrite,
             on_conflict,
+            schema,
         }
     }
 }
diff --git a/src/sqlite/write.rs b/src/sqlite/write.rs
index 4d02e949..6bb2c5a6 100644
--- a/src/sqlite/write.rs
+++ b/src/sqlite/write.rs
@@ -93,8 +93,8 @@ impl TableProvider for SqliteTableWriter {
                 Arc::clone(&self.sqlite),
                 op == InsertOp::Overwrite,
                 self.on_conflict.clone(),
+                self.schema(),
             )),
-            self.schema(),
             None,
         )) as _)
     }
@@ -105,6 +105,7 @@ struct SqliteDataSink {
     sqlite: Arc<Sqlite>,
     overwrite: bool,
     on_conflict: Option<OnConflict>,
+    schema: SchemaRef,
 }
 
 #[async_trait]
@@ -117,6 +118,10 @@ impl DataSink for SqliteDataSink {
         None
     }
 
+    fn schema(&self) -> &SchemaRef {
+        &self.schema
+    }
+
     async fn write_all(
         &self,
         data: SendableRecordBatchStream,
@@ -207,11 +212,17 @@ impl DataSink for SqliteDataSink {
 }
 
 impl SqliteDataSink {
-    fn new(sqlite: Arc<Sqlite>, overwrite: bool, on_conflict: Option<OnConflict>) -> Self {
+    fn new(
+        sqlite: Arc<Sqlite>,
+        overwrite: bool,
+        on_conflict: Option<OnConflict>,
+        schema: SchemaRef,
+    ) -> Self {
         Self {
             sqlite,
             overwrite,
             on_conflict,
+            schema,
         }
     }
 }


Reviews:
user: login: jychen7
body: the change LGTM. CI failure is another issue, could be fixed by https://github.com/datafusion-contrib/datafusion-table-providers/pull/249
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-09T04:27:12Z
user: login: jychen7
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-09T04:40:54Z
user: login: jychen7
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-09T04:41:18Z
user: login: phillipleblanc
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-11T14:32:48Z

Review comments:
diff_hunk: @@ -404,7 +404,8 @@ pub(crate) mod tests {
 
             let arc_created_table = Arc::new(created_table);
 
-            let duckdb_sink = DuckDBDataSink::new(arc_created_table, *overwrite, None);
+            let duckdb_sink =
+                DuckDBDataSink::new(arc_created_table, *overwrite, None, schema.clone());
path: src/duckdb/creator.rs
user: login: jychen7
body: ```suggestion
                DuckDBDataSink::new(arc_created_table, *overwrite, None, Arc::clone(&schema));
```
[minor]
created_at: 2025-03-09T04:40:54Z
author_association: MEMBER
diff_hunk: @@ -460,7 +461,8 @@ pub(crate) mod tests {
 
             let arc_created_table = Arc::new(created_table);
 
-            let duckdb_sink = DuckDBDataSink::new(arc_created_table, *overwrite, None);
+            let duckdb_sink =
+                DuckDBDataSink::new(arc_created_table, *overwrite, None, schema.clone());
path: src/duckdb/creator.rs
user: login: jychen7
body: ```suggestion
                DuckDBDataSink::new(arc_created_table, *overwrite, None, Arc::clone(&schema));
```
[minor]
created_at: 2025-03-09T04:41:18Z
author_association: MEMBER

PR comments:
user: login: phillipleblanc
created_at: 2025-03-11T12:27:01Z
body: I'm taking a look at the remaining failures
author_association: COLLABORATOR
user: login: nuno-faria
created_at: 2025-03-11T14:25:00Z
body: The `undefined reference to typeinfo for duckdb::ListLambdaBindData` error seems related to a build error of DuckDB on Linux: https://github.com/duckdb/duckdb-rs/issues/436.

Once v1.2.1 is released (https://github.com/duckdb/duckdb-rs/pull/462), this should be fixed after an upgrade to that version.
author_association: CONTRIBUTOR
user: login: phillipleblanc
created_at: 2025-03-11T14:30:48Z
body: > The `undefined reference to typeinfo for duckdb::ListLambdaBindData` error seems related to a build error of DuckDB on Linux: [duckdb/duckdb-rs#436](https://github.com/duckdb/duckdb-rs/issues/436).
> 
> Once v1.2.1 is released ([duckdb/duckdb-rs#462](https://github.com/duckdb/duckdb-rs/pull/462)), this should be fixed after an upgrade to that version.

Yeah, given that all tests pass locally for me - I'm going to merge this PR and then upgrade to 1.2.1 when its released (hopefully soon).
author_association: COLLABORATOR