Repository: apache/arrow-adbc
Pull request #2669: feat(csharp/src/Drivers/Apache/Spark): Add Lz4 compression support to arrow batch reader
Author: login: eric-wang-1990
Created: 2025-04-03T08:32:35Z | Merged: 2025-04-04T20:20:58Z

1. Create a new file [Lz4Utilities.cs](https://github.com/apache/arrow-adbc/compare/main...eric-wang-1990:arrow-adbc:add_lz4_to_arrowbatch#diff-7275463b6b9fcc3cf3b954580e3ded8b0b7237a90e0f4aea33eb11e613f3db39) to abstract common Lz4 decompress util functions for both cloud fetch and arrow batch.
2. Add support for decompress arrow batch with Lz4
3. Rename adbc.spark.cloudfetch.lz4.enabled to adbc.spark.lz4Compression.enabled since it is not specific to cloudfetch
4. Add test to test both cloudFetch and arrowBatch in StatementTests.



Diff:
diff --git a/csharp/src/Drivers/Apache/Spark/CloudFetch/SparkCloudFetchReader.cs b/csharp/src/Drivers/Apache/Spark/CloudFetch/SparkCloudFetchReader.cs
index 343bb5a0da..a96d6647b6 100644
--- a/csharp/src/Drivers/Apache/Spark/CloudFetch/SparkCloudFetchReader.cs
+++ b/csharp/src/Drivers/Apache/Spark/CloudFetch/SparkCloudFetchReader.cs
@@ -22,7 +22,9 @@
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
+using Apache.Arrow.Adbc.Drivers.Apache;
 using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
+using Apache.Arrow.Adbc.Drivers.Apache.Spark;
 using Apache.Arrow.Ipc;
 using Apache.Hive.Service.Rpc.Thrift;
 using K4os.Compression.LZ4.Streams;
@@ -161,59 +163,52 @@ private HttpClient HttpClient
                     var link = this.resultLinks[this.linkIndex++];
                     byte[]? fileData = null;
 
-                    // Retry logic for downloading files
-                    for (int retry = 0; retry < this.maxRetries; retry++)
+                    try
                     {
-                        try
+                        // Try to download with retry logic
+                        for (int retry = 0; retry < this.maxRetries; retry++)
                         {
-                            fileData = await DownloadFileAsync(link.FileLink, cancellationToken);
-                            break; // Success, exit retry loop
+                            try
+                            {
+                                fileData = await DownloadFileAsync(link.FileLink, cancellationToken);
+                                break; // Success, exit retry loop
+                            }
+                            catch (Exception) when (retry < this.maxRetries - 1)
+                            {
+                                // Only delay and retry if we haven't reached max retries
+                                await Task.Delay(this.retryDelayMs * (retry + 1), cancellationToken);
+                            }
                         }
-                        catch (Exception ex) when (retry < this.maxRetries - 1)
+
+                        // If download still failed after all retries
+                        if (fileData == null)
                         {
-                            // Log the error and retry
-                            Debug.WriteLine($"Error downloading file (attempt {retry + 1}/{this.maxRetries}): {ex.Message}");
-                            await Task.Delay(this.retryDelayMs * (retry + 1), cancellationToken);
+                            throw new AdbcException($"Failed to download CloudFetch data from {link.FileLink} after {this.maxRetries} attempts");
                         }
-                    }
 
-                    // Process the downloaded file data
-                    MemoryStream dataStream;
+                        ReadOnlyMemory<byte> dataToUse = new ReadOnlyMemory<byte>(fileData);
 
-                    // If the data is LZ4 compressed, decompress it
-                    if (this.isLz4Compressed)
-                    {
-                        try
-                        {
-                            dataStream = new MemoryStream();
-                            using (var inputStream = new MemoryStream(fileData!))
-                            using (var decompressor = LZ4Stream.Decode(inputStream))
-                            {
-                                await decompressor.CopyToAsync(dataStream);
-                            }
-                            dataStream.Position = 0;
-                        }
-                        catch (Exception ex)
+                        // If the data is LZ4 compressed, decompress it
+                        if (this.isLz4Compressed)
                         {
-                            Debug.WriteLine($"Error decompressing data: {ex.Message}");
-                            continue; // Skip this link and try the next one
+                            dataToUse = Lz4Utilities.DecompressLz4(fileData);
                         }
-                    }
-                    else
-                    {
-                        dataStream = new MemoryStream(fileData!);
-                    }
 
-                    try
-                    {
-                        this.currentReader = new ArrowStreamReader(dataStream);
+                        // Use ChunkStream which supports ReadOnlyMemory<byte> directly
+                        this.currentReader = new ArrowStreamReader(new ChunkStream(this.schema, dataToUse));
                         continue;
                     }
                     catch (Exception ex)
                     {
-                        Debug.WriteLine($"Error creating Arrow reader: {ex.Message}");
-                        dataStream.Dispose();
-                        continue; // Skip this link and try the next one
+                        // Create concise error message based on exception type
+                        string errorPrefix = $"CloudFetch link {this.linkIndex-1}:";
+                        string errorMessage = ex switch
+                        {
+                            _ when ex.GetType().Name.Contains("LZ4") => $"{errorPrefix} LZ4 decompression failed - Data may be corrupted",
+                            HttpRequestException or TaskCanceledException => $"{errorPrefix} Download failed - {ex.Message}",
+                            _ => $"{errorPrefix} Processing failed - {ex.Message}" // Default case for any other exception
+                        };
+                        throw new AdbcException(errorMessage, ex);
                     }
                 }
 
@@ -242,9 +237,7 @@ private HttpClient HttpClient
                 }
                 catch (Exception ex)
                 {
-                    Debug.WriteLine($"Error fetching results from server: {ex.Message}");
-                    this.statement = null; // Mark as done due to error
-                    return null;
+                    throw new AdbcException($"Server request failed - {ex.Message}", ex);
                 }
 
                 // Check if we have URL-based results
diff --git a/csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs b/csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs
new file mode 100644
index 0000000000..42cf218781
--- /dev/null
+++ b/csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs
@@ -0,0 +1,57 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Buffers;
+using System.IO;
+using K4os.Compression.LZ4.Streams;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
+{
+    /// <summary>
+    /// Utility class for LZ4 compression/decompression operations.
+    /// </summary>
+    internal static class Lz4Utilities
+    {
+        /// <summary>
+        /// Decompresses LZ4 compressed data into memory.
+        /// </summary>
+        /// <param name="compressedData">The compressed data bytes.</param>
+        /// <returns>A ReadOnlyMemory containing the decompressed data.</returns>
+        /// <exception cref="AdbcException">Thrown when decompression fails.</exception>
+        public static ReadOnlyMemory<byte> DecompressLz4(byte[] compressedData)
+        {
+            try
+            {
+                var outputStream = new MemoryStream();
+                using (var inputStream = new MemoryStream(compressedData))
+                using (var decompressor = LZ4Stream.Decode(inputStream))
+                {
+                    decompressor.CopyTo(outputStream);
+                }
+                // Get the underlying buffer and its valid length without copying
+                return new ReadOnlyMemory<byte>(outputStream.GetBuffer(), 0, (int)outputStream.Length);
+                // Note: We're not disposing the outputStream here because we're returning its buffer.
+                // The memory will be reclaimed when the ReadOnlyMemory is no longer referenced.
+            }
+            catch (Exception ex)
+            {
+                throw new AdbcException($"Failed to decompress LZ4 data: {ex.Message}", ex);
+            }
+        }
+    }
+}
diff --git a/csharp/src/Drivers/Apache/Spark/SparkDatabricksConnection.cs b/csharp/src/Drivers/Apache/Spark/SparkDatabricksConnection.cs
index 75705189b8..2e8e52e72b 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkDatabricksConnection.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkDatabricksConnection.cs
@@ -56,7 +56,7 @@ internal override IArrowArrayStream NewReader<T>(T statement, Schema schema, TGe
             }
             else
             {
-                return new SparkDatabricksReader(statement, schema);
+                return new SparkDatabricksReader(statement, schema, isLz4Compressed);
             }
         }
 
diff --git a/csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs b/csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
index 0e0166926f..fd8731f73e 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
@@ -15,12 +15,15 @@
 * limitations under the License.
 */
 
+using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
 using Apache.Arrow.Ipc;
 using Apache.Hive.Service.Rpc.Thrift;
+using K4os.Compression.LZ4.Streams;
 
 namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
 {
@@ -31,11 +34,18 @@ internal sealed class SparkDatabricksReader : IArrowArrayStream
         List<TSparkArrowBatch>? batches;
         int index;
         IArrowReader? reader;
+        bool isLz4Compressed;
 
         public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
+            : this(statement, schema, false)
+        {
+        }
+
+        public SparkDatabricksReader(HiveServer2Statement statement, Schema schema, bool isLz4Compressed)
         {
             this.statement = statement;
             this.schema = schema;
+            this.isLz4Compressed = isLz4Compressed;
         }
 
         public Schema Schema { get { return schema; } }
@@ -56,7 +66,7 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
 
                 if (this.batches != null && this.index < this.batches.Count)
                 {
-                    this.reader = new ArrowStreamReader(new ChunkStream(this.schema, this.batches[this.index++].Batch));
+                    ProcessFetchedBatches();
                     continue;
                 }
 
@@ -70,6 +80,8 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
 
                 TFetchResultsReq request = new TFetchResultsReq(this.statement.OperationHandle!, TFetchOrientation.FETCH_NEXT, this.statement.BatchSize);
                 TFetchResultsResp response = await this.statement.Connection.Client!.FetchResults(request, cancellationToken);
+
+                // Make sure we get the arrowBatches
                 this.batches = response.Results.ArrowBatches;
 
                 if (!response.HasMoreRows)
@@ -79,6 +91,43 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
             }
         }
 
+        private void ProcessFetchedBatches()
+        {
+            var batch = this.batches![this.index];
+
+            // Ensure batch data exists
+            if (batch.Batch == null || batch.Batch.Length == 0)
+            {
+                this.index++;
+                return;
+            }
+
+            try
+            {
+                ReadOnlyMemory<byte> dataToUse = new ReadOnlyMemory<byte>(batch.Batch);
+
+                // If LZ4 compression is enabled, decompress the data
+                if (isLz4Compressed)
+                {
+                    dataToUse = Lz4Utilities.DecompressLz4(batch.Batch);
+                }
+
+                // Always use ChunkStream which ensures proper schema handling
+                this.reader = new ArrowStreamReader(new ChunkStream(this.schema, dataToUse));
+            }
+            catch (Exception ex)
+            {
+                // Create concise error message based on exception type
+                string errorMessage = ex switch
+                {
+                    _ when ex.GetType().Name.Contains("LZ4") => $"Batch {this.index}: LZ4 decompression failed - Data may be corrupted",
+                    _ => $"Batch {this.index}: Processing failed - {ex.Message}" // Default case for any other exception
+                };
+                throw new AdbcException(errorMessage, ex);
+            }
+            this.index++;
+        }
+
         public void Dispose()
         {
         }
diff --git a/csharp/src/Drivers/Apache/Spark/SparkStatement.cs b/csharp/src/Drivers/Apache/Spark/SparkStatement.cs
index 4c4e61562a..eeb8ba60b5 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkStatement.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkStatement.cs
@@ -155,9 +155,10 @@ internal void SetMaxBytesPerFile(long maxBytesPerFile)
         /// </summary>
         public sealed class Options : ApacheParameters
         {
+            // Lz4 compression option
+            public const string CanDecompressLz4 = "adbc.spark.lz4_compression.enabled";
             // CloudFetch options
             public const string UseCloudFetch = "adbc.spark.cloudfetch.enabled";
-            public const string CanDecompressLz4 = "adbc.spark.cloudfetch.lz4.enabled";
             public const string MaxBytesPerFile = "adbc.spark.cloudfetch.max_bytes_per_file";
         }
     }
diff --git a/csharp/src/Drivers/Apache/Thrift/ChunkStream.cs b/csharp/src/Drivers/Apache/Thrift/ChunkStream.cs
index 4db0e80764..499e4930c4 100644
--- a/csharp/src/Drivers/Apache/Thrift/ChunkStream.cs
+++ b/csharp/src/Drivers/Apache/Thrift/ChunkStream.cs
@@ -26,11 +26,17 @@ namespace Apache.Arrow.Adbc.Drivers.Apache
     internal class ChunkStream : Stream
     {
         ReadOnlyMemory<byte> currentBuffer;
-        byte[] data;
+        ReadOnlyMemory<byte> data;
         bool first;
         int position;
 
         public ChunkStream(Schema schema, byte[] data)
+            : this(schema, new ReadOnlyMemory<byte>(data))
+        {
+            // Call the other constructor to avoid duplication
+        }
+
+        public ChunkStream(Schema schema, ReadOnlyMemory<byte> data)
         {
             MemoryStream buffer = new MemoryStream();
             ArrowStreamWriter writer = new ArrowStreamWriter(buffer, schema, leaveOpen: true);
@@ -70,7 +76,7 @@ public override int Read(byte[] buffer, int offset, int count)
                 {
                     return 0;
                 }
-                this.currentBuffer = new ReadOnlyMemory<byte>(this.data);
+                this.currentBuffer = this.data;
                 this.position = 0;
                 remaining = this.currentBuffer.Length - this.position;
             }
diff --git a/csharp/test/Drivers/Apache/Spark/StatementTests.cs b/csharp/test/Drivers/Apache/Spark/StatementTests.cs
index da170a66d7..b313d1d62a 100644
--- a/csharp/test/Drivers/Apache/Spark/StatementTests.cs
+++ b/csharp/test/Drivers/Apache/Spark/StatementTests.cs
@@ -17,6 +17,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Apache.Arrow.Adbc.Drivers.Apache.Spark;
 using Apache.Arrow.Adbc.Tests.Drivers.Apache.Common;
@@ -39,6 +40,50 @@ internal override void StatementTimeoutTest(StatementWithExceptions statementWit
             base.StatementTimeoutTest(statementWithExceptions);
         }
 
+        [SkippableTheory]
+        [InlineData(true, "CloudFetch enabled")]
+        [InlineData(false, "CloudFetch disabled")]
+        public async Task LZ4DecompressionCapabilityTest(bool useCloudFetch, string configName)
+        {
+            OutputHelper?.WriteLine($"Testing with LZ4 decompression capability enabled ({configName})");
+
+            // Create a connection using the test configuration
+            using AdbcConnection connection = NewConnection();
+            using var statement = connection.CreateStatement();
+
+            // Set options for LZ4 decompression (enabled by default) and CloudFetch as specified
+            statement.SetOption(SparkStatement.Options.UseCloudFetch, useCloudFetch.ToString().ToLower());
+            OutputHelper?.WriteLine($"CloudFetch is {(useCloudFetch ? "enabled" : "disabled")}");
+            OutputHelper?.WriteLine("LZ4 decompression capability is enabled by default");
+
+            // Execute a query that should return data
+            statement.SqlQuery = "SELECT id, CAST(id AS STRING) as id_string, id * 2 as id_doubled FROM RANGE(100)";
+            QueryResult result = statement.ExecuteQuery();
+
+            // Verify we have a valid stream
+            Assert.NotNull(result.Stream);
+
+            // Read all batches
+            int totalRows = 0;
+            int batchCount = 0;
+
+            while (result.Stream != null)
+            {
+                using var batch = await result.Stream.ReadNextRecordBatchAsync();
+                if (batch == null)
+                    break;
+
+                batchCount++;
+                totalRows += batch.Length;
+                OutputHelper?.WriteLine($"Batch {batchCount}: Read {batch.Length} rows");
+            }
+
+            // Verify we got all rows
+            Assert.Equal(100, totalRows);
+            OutputHelper?.WriteLine($"Successfully read {totalRows} rows in {batchCount} batches with {configName}");
+            OutputHelper?.WriteLine("NOTE: Whether actual LZ4 compression was used is determined by the server");
+        }
+
         internal class LongRunningStatementTimeoutTestData : ShortRunningStatementTimeoutTestData
         {
             public LongRunningStatementTimeoutTestData()


Reviews:
user: login: CurtHagenlocher
body: Thanks for the change! I've asked for some changes and made some (optional) suggestions for improvements.
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T14:58:15Z
user: login: eric-wang-1990
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T20:48:50Z
user: login: eric-wang-1990
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T20:57:00Z
user: login: eric-wang-1990
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T21:00:38Z
user: login: eric-wang-1990
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T21:01:18Z
user: login: eric-wang-1990
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T21:22:02Z
user: login: CurtHagenlocher
body: Thanks!
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T20:20:48Z

Review comments:
diff_hunk: @@ -0,0 +1,57 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using K4os.Compression.LZ4.Streams;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
+{
+    /// <summary>
+    /// Utility class for LZ4 compression/decompression operations.
+    /// </summary>
+    internal static class Lz4Utilities
+    {
+        /// <summary>
+        /// Decompresses LZ4 compressed data into a memory stream.
+        /// </summary>
+        /// <param name="compressedData">The compressed data bytes.</param>
+        /// <param name="cancellationToken">A cancellation token.</param>
+        /// <returns>A memory stream containing the decompressed data, positioned at the beginning of the stream.</returns>
+        /// <exception cref="AdbcException">Thrown when decompression fails.</exception>
+        public static async Task<MemoryStream> DecompressLz4Async(byte[] compressedData, CancellationToken cancellationToken = default)
+        {
+            try
+            {
+                var outputStream = new MemoryStream();
+                using (var inputStream = new MemoryStream(compressedData))
+                using (var decompressor = LZ4Stream.Decode(inputStream))
+                {
+                    await decompressor.CopyToAsync(outputStream);
+                }
+                outputStream.Position = 0;
+                return outputStream;
+            }
+            catch (Exception ex)
+            {
+                throw new AdbcException($"Failed to decompress LZ4 data: {ex.Message}", ex);
+            }
+        }
+    }
+} 
path: csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs
user: login: CurtHagenlocher
body: There's a single trailing space on this last line which is triggering the formatting error.
created_at: 2025-04-03T14:19:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -79,6 +91,49 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
             }
         }
 
+        private async Task ProcessFetchedBatchesAsync(CancellationToken cancellationToken)
+        {
+            var batch = this.batches![this.index];
+
+            // Ensure batch data exists
+            if (batch.Batch == null || batch.Batch.Length == 0)
+            {
+                this.index++;
+                return;
+            }
+
+            try
+            {
+                byte[] dataToUse = batch.Batch;
+
+                // If LZ4 compression is enabled, try to decompress the data
+                if (isLz4Compressed)
+                {
+                    try
+                    {
+                        var dataStream = await Lz4Utilities.DecompressLz4Async(batch.Batch, cancellationToken);
+                        dataToUse = dataStream.ToArray();
path: csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
user: login: CurtHagenlocher
body: This could be optimized to remove one allocation and one buffer copy by doing the following:
1) Make `dataToUse` be a `ReadOnlyMemory<byte>` instead of an array.
2) After computing `dataStream`, do `dataToUse = new ReadOnlyMemory<byte>(dataStream.GetBuffer(), 0, dataStream.Length);`
3) Change ChunkStream to accept a `ReadOnlyMemory<byte>` instead of an array as an argument.

If we know the uncompressed length in advance we could do even better. (One of the benefits of the Arrow compressed formats is that it reports that in the header -- but we obviously can't use that format if the server isn't producing it... .)
created_at: 2025-04-03T14:37:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,57 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using K4os.Compression.LZ4.Streams;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
+{
+    /// <summary>
+    /// Utility class for LZ4 compression/decompression operations.
+    /// </summary>
+    internal static class Lz4Utilities
+    {
+        /// <summary>
+        /// Decompresses LZ4 compressed data into a memory stream.
+        /// </summary>
+        /// <param name="compressedData">The compressed data bytes.</param>
+        /// <param name="cancellationToken">A cancellation token.</param>
+        /// <returns>A memory stream containing the decompressed data, positioned at the beginning of the stream.</returns>
+        /// <exception cref="AdbcException">Thrown when decompression fails.</exception>
+        public static async Task<MemoryStream> DecompressLz4Async(byte[] compressedData, CancellationToken cancellationToken = default)
path: csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs
user: login: CurtHagenlocher
body: FWIW, as this is entirely CPU-bound there's no advantage (and maybe even a small perf hit) in making it `async`.
created_at: 2025-04-03T14:43:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -79,6 +91,49 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
             }
         }
 
+        private async Task ProcessFetchedBatchesAsync(CancellationToken cancellationToken)
+        {
+            var batch = this.batches![this.index];
+
+            // Ensure batch data exists
+            if (batch.Batch == null || batch.Batch.Length == 0)
+            {
+                this.index++;
+                return;
+            }
+
+            try
+            {
+                byte[] dataToUse = batch.Batch;
+
+                // If LZ4 compression is enabled, try to decompress the data
+                if (isLz4Compressed)
+                {
+                    try
+                    {
+                        var dataStream = await Lz4Utilities.DecompressLz4Async(batch.Batch, cancellationToken);
path: csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
user: login: CurtHagenlocher
body: ```suggestion
                        using dataStream = await Lz4Utilities.DecompressLz4Async(batch.Batch, cancellationToken);
```
Then the explicit `Dispose()` can be removed.
created_at: 2025-04-03T14:45:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -79,6 +91,49 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
             }
         }
 
+        private async Task ProcessFetchedBatchesAsync(CancellationToken cancellationToken)
+        {
+            var batch = this.batches![this.index];
+
+            // Ensure batch data exists
+            if (batch.Batch == null || batch.Batch.Length == 0)
+            {
+                this.index++;
+                return;
+            }
+
+            try
+            {
+                byte[] dataToUse = batch.Batch;
+
+                // If LZ4 compression is enabled, try to decompress the data
+                if (isLz4Compressed)
+                {
+                    try
+                    {
+                        var dataStream = await Lz4Utilities.DecompressLz4Async(batch.Batch, cancellationToken);
+                        dataToUse = dataStream.ToArray();
+                        dataStream.Dispose();
+                    }
+                    catch (Exception ex)
+                    {
+                        // If decompression fails, use the original data
+                        System.Diagnostics.Debug.WriteLine($"Failed to decompress LZ4 data: {ex.Message}");
+                    }
+                }
+
+                // Always use ChunkStream which ensures proper schema handling
+                this.reader = new ArrowStreamReader(new ChunkStream(this.schema, dataToUse));
+            }
+            catch (Exception ex)
+            {
+                // Log any errors and skip this batch
+                System.Diagnostics.Debug.WriteLine($"Error processing batch: {ex.Message}");
path: csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
user: login: CurtHagenlocher
body: It seems like a bad idea to just suppress the error. Are there circumstances under which dropping these rows do not result in silent data loss?
created_at: 2025-04-03T14:46:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -155,9 +155,10 @@ internal void SetMaxBytesPerFile(long maxBytesPerFile)
         /// </summary>
         public sealed class Options : ApacheParameters
         {
+            // Lz4 compression option
+            public const string CanDecompressLz4 = "adbc.spark.lz4Compression.enabled";
path: csharp/src/Drivers/Apache/Spark/SparkStatement.cs
user: login: CurtHagenlocher
body: option names in ADBC tend heavily to "snake case" instead of "camel case" e.g. `"lz4_compression"` vs `"lz4Compression"`.
created_at: 2025-04-03T14:53:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -155,9 +155,10 @@ internal void SetMaxBytesPerFile(long maxBytesPerFile)
         /// </summary>
         public sealed class Options : ApacheParameters
path: csharp/src/Drivers/Apache/Spark/SparkStatement.cs
user: login: CurtHagenlocher
body: Unrelated to this change, it would be nice if these properties were listed with the others in csharp/src/Drivers/Apache/Spark/README.md. (That could be done as a followup.)
created_at: 2025-04-03T14:57:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -79,6 +91,49 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
             }
         }
 
+        private async Task ProcessFetchedBatchesAsync(CancellationToken cancellationToken)
+        {
+            var batch = this.batches![this.index];
+
+            // Ensure batch data exists
+            if (batch.Batch == null || batch.Batch.Length == 0)
+            {
+                this.index++;
+                return;
+            }
+
+            try
+            {
+                byte[] dataToUse = batch.Batch;
+
+                // If LZ4 compression is enabled, try to decompress the data
+                if (isLz4Compressed)
+                {
+                    try
+                    {
+                        var dataStream = await Lz4Utilities.DecompressLz4Async(batch.Batch, cancellationToken);
+                        dataToUse = dataStream.ToArray();
path: csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
user: login: eric-wang-1990
body: Sounds good, will change.
created_at: 2025-04-03T20:48:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,57 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using K4os.Compression.LZ4.Streams;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
+{
+    /// <summary>
+    /// Utility class for LZ4 compression/decompression operations.
+    /// </summary>
+    internal static class Lz4Utilities
+    {
+        /// <summary>
+        /// Decompresses LZ4 compressed data into a memory stream.
+        /// </summary>
+        /// <param name="compressedData">The compressed data bytes.</param>
+        /// <param name="cancellationToken">A cancellation token.</param>
+        /// <returns>A memory stream containing the decompressed data, positioned at the beginning of the stream.</returns>
+        /// <exception cref="AdbcException">Thrown when decompression fails.</exception>
+        public static async Task<MemoryStream> DecompressLz4Async(byte[] compressedData, CancellationToken cancellationToken = default)
path: csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs
user: login: eric-wang-1990
body: Makes sense, will change to sync.
created_at: 2025-04-03T20:57:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -79,6 +91,49 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
             }
         }
 
+        private async Task ProcessFetchedBatchesAsync(CancellationToken cancellationToken)
+        {
+            var batch = this.batches![this.index];
+
+            // Ensure batch data exists
+            if (batch.Batch == null || batch.Batch.Length == 0)
+            {
+                this.index++;
+                return;
+            }
+
+            try
+            {
+                byte[] dataToUse = batch.Batch;
+
+                // If LZ4 compression is enabled, try to decompress the data
+                if (isLz4Compressed)
+                {
+                    try
+                    {
+                        var dataStream = await Lz4Utilities.DecompressLz4Async(batch.Batch, cancellationToken);
+                        dataToUse = dataStream.ToArray();
+                        dataStream.Dispose();
+                    }
+                    catch (Exception ex)
+                    {
+                        // If decompression fails, use the original data
+                        System.Diagnostics.Debug.WriteLine($"Failed to decompress LZ4 data: {ex.Message}");
+                    }
+                }
+
+                // Always use ChunkStream which ensures proper schema handling
+                this.reader = new ArrowStreamReader(new ChunkStream(this.schema, dataToUse));
+            }
+            catch (Exception ex)
+            {
+                // Log any errors and skip this batch
+                System.Diagnostics.Debug.WriteLine($"Error processing batch: {ex.Message}");
path: csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
user: login: eric-wang-1990
body: Yeah I was planning to throw, somehow this part got ommited. We do not want partial data and should definitely throw here.
created_at: 2025-04-03T21:00:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -79,6 +91,49 @@ public SparkDatabricksReader(HiveServer2Statement statement, Schema schema)
             }
         }
 
+        private async Task ProcessFetchedBatchesAsync(CancellationToken cancellationToken)
+        {
+            var batch = this.batches![this.index];
+
+            // Ensure batch data exists
+            if (batch.Batch == null || batch.Batch.Length == 0)
+            {
+                this.index++;
+                return;
+            }
+
+            try
+            {
+                byte[] dataToUse = batch.Batch;
+
+                // If LZ4 compression is enabled, try to decompress the data
+                if (isLz4Compressed)
+                {
+                    try
+                    {
+                        var dataStream = await Lz4Utilities.DecompressLz4Async(batch.Batch, cancellationToken);
path: csharp/src/Drivers/Apache/Spark/SparkDatabricksReader.cs
user: login: eric-wang-1990
body: This is not needed since now it will return a ReadOnlyMemory
created_at: 2025-04-03T21:01:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -155,9 +155,10 @@ internal void SetMaxBytesPerFile(long maxBytesPerFile)
         /// </summary>
         public sealed class Options : ApacheParameters
path: csharp/src/Drivers/Apache/Spark/SparkStatement.cs
user: login: eric-wang-1990
body: These params are statement level for now, I will create a follow up to change it to connection level.
created_at: 2025-04-03T21:22:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,57 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Buffers;
+using System.IO;
+using K4os.Compression.LZ4.Streams;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
+{
+    /// <summary>
+    /// Utility class for LZ4 compression/decompression operations.
+    /// </summary>
+    internal static class Lz4Utilities
+    {
+        /// <summary>
+        /// Decompresses LZ4 compressed data into memory.
+        /// </summary>
+        /// <param name="compressedData">The compressed data bytes.</param>
+        /// <returns>A ReadOnlyMemory containing the decompressed data.</returns>
+        /// <exception cref="AdbcException">Thrown when decompression fails.</exception>
+        public static ReadOnlyMemory<byte> DecompressLz4(byte[] compressedData)
+        {
+            try
+            {
+                var outputStream = new MemoryStream();
+                using (var inputStream = new MemoryStream(compressedData))
+                using (var decompressor = LZ4Stream.Decode(inputStream))
+                {
+                    decompressor.CopyTo(outputStream);
+                }
+                // Get the underlying buffer and its valid length without copying
+                return new ReadOnlyMemory<byte>(outputStream.GetBuffer(), 0, (int)outputStream.Length);
+                // Note: We're not disposing the outputStream here because we're returning its buffer.
+                // The memory will be reclaimed when the ReadOnlyMemory is no longer referenced.
path: csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs
user: login: CurtHagenlocher
body: I don't know that we necessarily need to remove or change the comment, but this isn't strictly correct. There are a lot of misunderstandings about `IDisposable`, but disposing a `MemoryStream` does nothing to its internal buffer. Because the buffer is a managed array, it will not get garbage-collected until there are no more references to it -- even if the `MemoryStream` that created it is disposed.


created_at: 2025-04-04T17:53:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,57 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Buffers;
+using System.IO;
+using K4os.Compression.LZ4.Streams;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
+{
+    /// <summary>
+    /// Utility class for LZ4 compression/decompression operations.
+    /// </summary>
+    internal static class Lz4Utilities
+    {
+        /// <summary>
+        /// Decompresses LZ4 compressed data into memory.
+        /// </summary>
+        /// <param name="compressedData">The compressed data bytes.</param>
+        /// <returns>A ReadOnlyMemory containing the decompressed data.</returns>
+        /// <exception cref="AdbcException">Thrown when decompression fails.</exception>
+        public static ReadOnlyMemory<byte> DecompressLz4(byte[] compressedData)
+        {
+            try
+            {
+                var outputStream = new MemoryStream();
+                using (var inputStream = new MemoryStream(compressedData))
+                using (var decompressor = LZ4Stream.Decode(inputStream))
+                {
+                    decompressor.CopyTo(outputStream);
+                }
+                // Get the underlying buffer and its valid length without copying
+                return new ReadOnlyMemory<byte>(outputStream.GetBuffer(), 0, (int)outputStream.Length);
+                // Note: We're not disposing the outputStream here because we're returning its buffer.
+                // The memory will be reclaimed when the ReadOnlyMemory is no longer referenced.
path: csharp/src/Drivers/Apache/Spark/Lz4Utilities.cs
user: login: CurtHagenlocher
body: It's fair to not want to assume anything about what `MemoryStream.Dispose` might do to its internal buffer, but we're now assuming instead that a `MemoryStream` doesn't need to be disposed. (This happens to be true.)
created_at: 2025-04-04T20:20:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: lidavidm
created_at: 2025-04-03T10:32:56Z
body: Just curious, does Spark/Databricks not use Arrow IPC compression? It compresses the file as a whole separately?
author_association: MEMBER
user: login: CurtHagenlocher
created_at: 2025-04-03T14:16:57Z
body: > Just curious, does Spark/Databricks not use Arrow IPC compression? It compresses the file as a whole separately?

Yeah I asked a similar question in a previous PR, but apparently it doesn't (or it wouldn't need a separate out-of-band flag for the compression setting).
author_association: CONTRIBUTOR