Repository: apache/arrow-rs
Pull request #6047 (2024-07-12 15:17:44)

Minor API adjustments for StringViewBuilder
# Which issue does this PR close?

<!--
We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
-->

Closes #.

# Rationale for this change
Current string view builder won't allow users to know what has already being inserted to the builder.
However, in many real use cases, like DataFusion's ArrowByteViewMap, or user want to deduplicate the string view, they want to know what's already in the builder.


<!--
Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes.
-->

# What changes are included in this PR?
1. We have implemented `get_value()`, but not visible to user. This pr make the function visible.

2. previous gc will call deduplicate string, I think we should get rid of that. 

3. Added allocated_size, useful for memory accounting, especially for DataFusion usage.
<!--
There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
-->

# Are there any user-facing changes?


<!--
If there are user-facing changes then we may require documentation to be updated before approving the PR.
-->

<!---
If there are any breaking changes to public APIs, please add the `breaking change` label.
-->



Patch:
diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml
index 422f676cb..bdbfc0bec 100644
--- a/.github/workflows/object_store.yml
+++ b/.github/workflows/object_store.yml
@@ -198,3 +198,16 @@ jobs:
         run: cargo build --target wasm32-unknown-unknown
       - name: Build wasm32-wasi
         run: cargo build --target wasm32-wasi
+
+  windows:
+    name: cargo test LocalFileSystem (win64)
+    runs-on: windows-latest
+    defaults:
+      run:
+        working-directory: object_store
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Run LocalFileSystem tests
+        run: cargo test local::tests
diff --git a/arrow-array/src/array/byte_view_array.rs b/arrow-array/src/array/byte_view_array.rs
index a00bf7271..5ce150a7a 100644
--- a/arrow-array/src/array/byte_view_array.rs
+++ b/arrow-array/src/array/byte_view_array.rs
@@ -324,9 +324,11 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     /// Note that it will copy the array regardless of whether the original array is compact.
     /// Use with caution as this can be an expensive operation, only use it when you are sure that the view
     /// array is significantly smaller than when it is originally created, e.g., after filtering or slicing.
+    ///
+    /// Note: this function does not attempt to canonicalize / deduplicate values. For this
+    /// feature see  [`GenericByteViewBuilder::with_deduplicate_strings`].
     pub fn gc(&self) -> Self {
-        let mut builder =
-            GenericByteViewBuilder::<T>::with_capacity(self.len()).with_deduplicate_strings();
+        let mut builder = GenericByteViewBuilder::<T>::with_capacity(self.len());
 
         for v in self.iter() {
             builder.append_option(v);
diff --git a/arrow-array/src/array/dictionary_array.rs b/arrow-array/src/array/dictionary_array.rs
index 045917a1b..d6c5dd4c3 100644
--- a/arrow-array/src/array/dictionary_array.rs
+++ b/arrow-array/src/array/dictionary_array.rs
@@ -1025,13 +1025,13 @@ mod tests {
         let value_data = ArrayData::builder(DataType::Int8)
             .len(8)
             .add_buffer(Buffer::from(
-                &[10_i8, 11, 12, 13, 14, 15, 16, 17].to_byte_slice(),
+                [10_i8, 11, 12, 13, 14, 15, 16, 17].to_byte_slice(),
             ))
             .build()
             .unwrap();
 
         // Construct a buffer for value offsets, for the nested array:
-        let keys = Buffer::from(&[2_i16, 3, 4].to_byte_slice());
+        let keys = Buffer::from([2_i16, 3, 4].to_byte_slice());
 
         // Construct a dictionary array from the above two
         let key_type = DataType::Int16;
diff --git a/arrow-array/src/array/fixed_size_list_array.rs b/arrow-array/src/array/fixed_size_list_array.rs
index 6f3a76908..0d57d9a69 100644
--- a/arrow-array/src/array/fixed_size_list_array.rs
+++ b/arrow-array/src/array/fixed_size_list_array.rs
@@ -674,7 +674,7 @@ mod tests {
         assert_eq!(err.to_string(), "Invalid argument error: Found unmasked nulls for non-nullable FixedSizeListArray field \"item\"");
 
         // Valid as nulls in child masked by parent
-        let nulls = NullBuffer::new(BooleanBuffer::new(vec![0b0000101].into(), 0, 3));
+        let nulls = NullBuffer::new(BooleanBuffer::new(Buffer::from([0b0000101]), 0, 3));
         FixedSizeListArray::new(field, 2, values.clone(), Some(nulls));
 
         let field = Arc::new(Field::new("item", DataType::Int64, true));
diff --git a/arrow-array/src/array/map_array.rs b/arrow-array/src/array/map_array.rs
index ed22ba1d8..bddf202bd 100644
--- a/arrow-array/src/array/map_array.rs
+++ b/arrow-array/src/array/map_array.rs
@@ -448,20 +448,20 @@ mod tests {
         // Construct key and values
         let keys_data = ArrayData::builder(DataType::Int32)
             .len(8)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .add_buffer(Buffer::from([0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
             .build()
             .unwrap();
         let values_data = ArrayData::builder(DataType::UInt32)
             .len(8)
             .add_buffer(Buffer::from(
-                &[0u32, 10, 20, 30, 40, 50, 60, 70].to_byte_slice(),
+                [0u32, 10, 20, 30, 40, 50, 60, 70].to_byte_slice(),
             ))
             .build()
             .unwrap();
 
         // Construct a buffer for value offsets, for the nested array:
         //  [[0, 1, 2], [3, 4, 5], [6, 7]]
-        let entry_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
+        let entry_offsets = Buffer::from([0, 3, 6, 8].to_byte_slice());
 
         let keys = Arc::new(Field::new("keys", DataType::Int32, false));
         let values = Arc::new(Field::new("values", DataType::UInt32, false));
@@ -493,13 +493,13 @@ mod tests {
         // Construct key and values
         let key_data = ArrayData::builder(DataType::Int32)
             .len(8)
-            .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+            .add_buffer(Buffer::from([0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
             .build()
             .unwrap();
         let value_data = ArrayData::builder(DataType::UInt32)
             .len(8)
             .add_buffer(Buffer::from(
-                &[0u32, 10, 20, 0, 40, 0, 60, 70].to_byte_slice(),
+                [0u32, 10, 20, 0, 40, 0, 60, 70].to_byte_slice(),
             ))
             .null_bit_buffer(Some(Buffer::from(&[0b11010110])))
             .build()
@@ -507,7 +507,7 @@ mod tests {
 
         // Construct a buffer for value offsets, for the nested array:
         //  [[0, 1, 2], [3, 4, 5], [6, 7]]
-        let entry_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
+        let entry_offsets = Buffer::from([0, 3, 6, 8].to_byte_slice());
 
         let keys_field = Arc::new(Field::new("keys", DataType::Int32, false));
         let values_field = Arc::new(Field::new("values", DataType::UInt32, true));
@@ -617,18 +617,18 @@ mod tests {
         // Construct key and values
         let keys_data = ArrayData::builder(DataType::Int32)
             .len(5)
-            .add_buffer(Buffer::from(&[3, 4, 5, 6, 7].to_byte_slice()))
+            .add_buffer(Buffer::from([3, 4, 5, 6, 7].to_byte_slice()))
             .build()
             .unwrap();
         let values_data = ArrayData::builder(DataType::UInt32)
             .len(5)
-            .add_buffer(Buffer::from(&[30u32, 40, 50, 60, 70].to_byte_slice()))
+            .add_buffer(Buffer::from([30u32, 40, 50, 60, 70].to_byte_slice()))
             .build()
             .unwrap();
 
         // Construct a buffer for value offsets, for the nested array:
         //  [[3, 4, 5], [6, 7]]
-        let entry_offsets = Buffer::from(&[0, 3, 5].to_byte_slice());
+        let entry_offsets = Buffer::from([0, 3, 5].to_byte_slice());
 
         let keys = Arc::new(Field::new("keys", DataType::Int32, false));
         let values = Arc::new(Field::new("values", DataType::UInt32, false));
diff --git a/arrow-array/src/array/struct_array.rs b/arrow-array/src/array/struct_array.rs
index ae292944e..44a7f38c3 100644
--- a/arrow-array/src/array/struct_array.rs
+++ b/arrow-array/src/array/struct_array.rs
@@ -549,7 +549,7 @@ mod tests {
         let expected_string_data = ArrayData::builder(DataType::Utf8)
             .len(4)
             .null_bit_buffer(Some(Buffer::from(&[9_u8])))
-            .add_buffer(Buffer::from(&[0, 3, 3, 3, 7].to_byte_slice()))
+            .add_buffer(Buffer::from([0, 3, 3, 3, 7].to_byte_slice()))
             .add_buffer(Buffer::from(b"joemark"))
             .build()
             .unwrap();
@@ -557,7 +557,7 @@ mod tests {
         let expected_int_data = ArrayData::builder(DataType::Int32)
             .len(4)
             .null_bit_buffer(Some(Buffer::from(&[11_u8])))
-            .add_buffer(Buffer::from(&[1, 2, 0, 4].to_byte_slice()))
+            .add_buffer(Buffer::from([1, 2, 0, 4].to_byte_slice()))
             .build()
             .unwrap();
 
diff --git a/arrow-array/src/builder/generic_bytes_view_builder.rs b/arrow-array/src/builder/generic_bytes_view_builder.rs
index dda553545..587255cc6 100644
--- a/arrow-array/src/builder/generic_bytes_view_builder.rs
+++ b/arrow-array/src/builder/generic_bytes_view_builder.rs
@@ -201,7 +201,8 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
 
     /// Returns the value at the given index
     /// Useful if we want to know what value has been inserted to the builder
-    fn get_value(&self, index: usize) -> &[u8] {
+    /// The index has to be smaller than `self.len()`, otherwise it will panic
+    pub fn get_value(&self, index: usize) -> &[u8] {
         let view = self.views_builder.as_slice().get(index).unwrap();
         let len = *view as u32;
         if len <= 12 {
@@ -337,6 +338,19 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
     pub fn validity_slice(&self) -> Option<&[u8]> {
         self.null_buffer_builder.as_slice()
     }
+
+    /// Return the allocated size of this builder in bytes, useful for memory accounting.
+    pub fn allocated_size(&self) -> usize {
+        let views = self.views_builder.capacity() * std::mem::size_of::<u128>();
+        let null = self.null_buffer_builder.allocated_size();
+        let buffer_size = self.completed.iter().map(|b| b.capacity()).sum::<usize>();
+        let in_progress = self.in_progress.capacity();
+        let tracker = match &self.string_tracker {
+            Some((ht, _)) => ht.capacity() * std::mem::size_of::<usize>(),
+            None => 0,
+        };
+        buffer_size + in_progress + tracker + views + null
+    }
 }
 
 impl<T: ByteViewType + ?Sized> Default for GenericByteViewBuilder<T> {
diff --git a/arrow-buffer/src/buffer/immutable.rs b/arrow-buffer/src/buffer/immutable.rs
index 2c743842f..52e201ca1 100644
--- a/arrow-buffer/src/buffer/immutable.rs
+++ b/arrow-buffer/src/buffer/immutable.rs
@@ -543,7 +543,7 @@ mod tests {
 
     #[test]
     fn test_access_concurrently() {
-        let buffer = Buffer::from(vec![1, 2, 3, 4, 5]);
+        let buffer = Buffer::from([1, 2, 3, 4, 5]);
         let buffer2 = buffer.clone();
         assert_eq!([1, 2, 3, 4, 5], buffer.as_slice());
 
diff --git a/arrow-buffer/src/builder/null.rs b/arrow-buffer/src/builder/null.rs
index 55b3303c9..a1cea6ef2 100644
--- a/arrow-buffer/src/builder/null.rs
+++ b/arrow-buffer/src/builder/null.rs
@@ -161,6 +161,14 @@ impl NullBufferBuilder {
     pub fn as_slice_mut(&mut self) -> Option<&mut [u8]> {
         self.bitmap_builder.as_mut().map(|b| b.as_slice_mut())
     }
+
+    /// Return the allocated size of this builder, in bytes, useful for memory accounting.
+    pub fn allocated_size(&self) -> usize {
+        self.bitmap_builder
+            .as_ref()
+            .map(|b| b.capacity())
+            .unwrap_or(0)
+    }
 }
 
 impl NullBufferBuilder {
diff --git a/arrow-buffer/src/util/bit_chunk_iterator.rs b/arrow-buffer/src/util/bit_chunk_iterator.rs
index 9e4fb8268..440450908 100644
--- a/arrow-buffer/src/util/bit_chunk_iterator.rs
+++ b/arrow-buffer/src/util/bit_chunk_iterator.rs
@@ -456,7 +456,7 @@ mod tests {
         const ALLOC_SIZE: usize = 4 * 1024;
         let input = vec![0xFF_u8; ALLOC_SIZE];
 
-        let buffer: Buffer = Buffer::from(input);
+        let buffer: Buffer = Buffer::from_vec(input);
 
         let bitchunks = buffer.bit_chunks(57, ALLOC_SIZE * 8 - 57);
 
diff --git a/arrow-cast/src/base64.rs b/arrow-cast/src/base64.rs
index 50c742362..534b21878 100644
--- a/arrow-cast/src/base64.rs
+++ b/arrow-cast/src/base64.rs
@@ -20,7 +20,7 @@
 //! [`StringArray`]: arrow_array::StringArray
 
 use arrow_array::{Array, GenericBinaryArray, GenericStringArray, OffsetSizeTrait};
-use arrow_buffer::OffsetBuffer;
+use arrow_buffer::{Buffer, OffsetBuffer};
 use arrow_schema::ArrowError;
 use base64::encoded_len;
 use base64::engine::Config;
@@ -50,7 +50,9 @@ pub fn b64_encode<E: Engine, O: OffsetSizeTrait>(
     assert_eq!(offset, buffer_len);
 
     // Safety: Base64 is valid UTF-8
-    unsafe { GenericStringArray::new_unchecked(offsets, buffer.into(), array.nulls().cloned()) }
+    unsafe {
+        GenericStringArray::new_unchecked(offsets, Buffer::from_vec(buffer), array.nulls().cloned())
+    }
 }
 
 /// Base64 decode each element of `array` with the provided [`Engine`]
@@ -79,7 +81,7 @@ pub fn b64_decode<E: Engine, O: OffsetSizeTrait>(
 
     Ok(GenericBinaryArray::new(
         offsets,
-        buffer.into(),
+        Buffer::from_vec(buffer),
         array.nulls().cloned(),
     ))
 }
diff --git a/arrow-data/src/data.rs b/arrow-data/src/data.rs
index b6ff4518e..d6d6b90b6 100644
--- a/arrow-data/src/data.rs
+++ b/arrow-data/src/data.rs
@@ -1959,7 +1959,7 @@ mod tests {
             .len(20)
             .offset(5)
             .add_buffer(b1)
-            .null_bit_buffer(Some(Buffer::from(vec![
+            .null_bit_buffer(Some(Buffer::from([
                 0b01011111, 0b10110101, 0b01100011, 0b00011110,
             ])))
             .build()
@@ -2164,7 +2164,7 @@ mod tests {
 
     #[test]
     fn test_count_nulls() {
-        let buffer = Buffer::from(vec![0b00010110, 0b10011111]);
+        let buffer = Buffer::from([0b00010110, 0b10011111]);
         let buffer = NullBuffer::new(BooleanBuffer::new(buffer, 0, 16));
         let count = count_nulls(Some(&buffer), 0, 16);
         assert_eq!(count, 7);
diff --git a/arrow-flight/src/utils.rs b/arrow-flight/src/utils.rs
index c1e2d61fc..37d7ff9e7 100644
--- a/arrow-flight/src/utils.rs
+++ b/arrow-flight/src/utils.rs
@@ -94,7 +94,7 @@ pub fn flight_data_to_arrow_batch(
         })
         .map(|batch| {
             reader::read_record_batch(
-                &Buffer::from(&data.data_body),
+                &Buffer::from(data.data_body.as_ref()),
                 batch,
                 schema,
                 dictionaries_by_id,
diff --git a/arrow-ipc/src/compression.rs b/arrow-ipc/src/compression.rs
index 0d8b7b4c1..47ea7785c 100644
--- a/arrow-ipc/src/compression.rs
+++ b/arrow-ipc/src/compression.rs
@@ -103,8 +103,8 @@ impl CompressionCodec {
         } else if let Ok(decompressed_length) = usize::try_from(decompressed_length) {
             // decompress data using the codec
             let input_data = &input[(LENGTH_OF_PREFIX_DATA as usize)..];
-            self.decompress(input_data, decompressed_length as _)?
-                .into()
+            let v = self.decompress(input_data, decompressed_length as _)?;
+            Buffer::from_vec(v)
         } else {
             return Err(ArrowError::IpcError(format!(
                 "Invalid uncompressed length: {decompressed_length}"
diff --git a/arrow-json/src/reader/mod.rs b/arrow-json/src/reader/mod.rs
index 3e1c5d2fc..97d9c8962 100644
--- a/arrow-json/src/reader/mod.rs
+++ b/arrow-json/src/reader/mod.rs
@@ -1850,7 +1850,7 @@ mod tests {
         let c = ArrayDataBuilder::new(c_field.data_type().clone())
             .len(7)
             .add_child_data(d.to_data())
-            .null_bit_buffer(Some(Buffer::from(vec![0b00111011])))
+            .null_bit_buffer(Some(Buffer::from([0b00111011])))
             .build()
             .unwrap();
         let b = BooleanArray::from(vec![
@@ -1866,14 +1866,14 @@ mod tests {
             .len(7)
             .add_child_data(b.to_data())
             .add_child_data(c.clone())
-            .null_bit_buffer(Some(Buffer::from(vec![0b00111111])))
+            .null_bit_buffer(Some(Buffer::from([0b00111111])))
             .build()
             .unwrap();
         let a_list = ArrayDataBuilder::new(a_field.data_type().clone())
             .len(6)
             .add_buffer(Buffer::from_slice_ref([0i32, 2, 3, 6, 6, 6, 7]))
             .add_child_data(a)
-            .null_bit_buffer(Some(Buffer::from(vec![0b00110111])))
+            .null_bit_buffer(Some(Buffer::from([0b00110111])))
             .build()
             .unwrap();
         let expected = make_array(a_list);
diff --git a/arrow-json/src/writer.rs b/arrow-json/src/writer.rs
index ef4141d7a..86d2e88d9 100644
--- a/arrow-json/src/writer.rs
+++ b/arrow-json/src/writer.rs
@@ -927,12 +927,12 @@ mod tests {
 
         let a_values = StringArray::from(vec!["a", "a1", "b", "c", "d", "e"]);
         // list column rows: ["a", "a1"], ["b"], ["c"], ["d"], ["e"]
-        let a_value_offsets = Buffer::from(&[0, 2, 3, 4, 5, 6].to_byte_slice());
+        let a_value_offsets = Buffer::from([0, 2, 3, 4, 5, 6].to_byte_slice());
         let a_list_data = ArrayData::builder(field_c1.data_type().clone())
             .len(5)
             .add_buffer(a_value_offsets)
             .add_child_data(a_values.into_data())
-            .null_bit_buffer(Some(Buffer::from(vec![0b00011111])))
+            .null_bit_buffer(Some(Buffer::from([0b00011111])))
             .build()
             .unwrap();
         let a = ListArray::from(a_list_data);
@@ -976,17 +976,17 @@ mod tests {
         // list column rows: [[1, 2], [3]], [], [[4, 5, 6]]
         let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
 
-        let a_value_offsets = Buffer::from(&[0, 2, 3, 6].to_byte_slice());
+        let a_value_offsets = Buffer::from([0, 2, 3, 6].to_byte_slice());
         // Construct a list array from the above two
         let a_list_data = ArrayData::builder(list_inner_type.data_type().clone())
             .len(3)
             .add_buffer(a_value_offsets)
-            .null_bit_buffer(Some(Buffer::from(vec![0b00000111])))
+            .null_bit_buffer(Some(Buffer::from([0b00000111])))
             .add_child_data(a_values.into_data())
             .build()
             .unwrap();
 
-        let c1_value_offsets = Buffer::from(&[0, 2, 2, 3].to_byte_slice());
+        let c1_value_offsets = Buffer::from([0, 2, 2, 3].to_byte_slice());
         let c1_list_data = ArrayData::builder(field_c1.data_type().clone())
             .len(3)
             .add_buffer(c1_value_offsets)
@@ -1058,12 +1058,12 @@ mod tests {
         // [{"c11": 1, "c12": {"c121": "e"}}, {"c12": {"c121": "f"}}],
         // null,
         // [{"c11": 5, "c12": {"c121": "g"}}]
-        let c1_value_offsets = Buffer::from(&[0, 2, 2, 3].to_byte_slice());
+        let c1_value_offsets = Buffer::from([0, 2, 2, 3].to_byte_slice());
         let c1_list_data = ArrayData::builder(field_c1.data_type().clone())
             .len(3)
             .add_buffer(c1_value_offsets)
             .add_child_data(struct_values.into_data())
-            .null_bit_buffer(Some(Buffer::from(vec![0b00000101])))
+            .null_bit_buffer(Some(Buffer::from([0b00000101])))
             .build()
             .unwrap();
         let c1 = ListArray::from(c1_list_data);
@@ -1225,7 +1225,7 @@ mod tests {
         );
 
         // [{"foo": 10}, null, {}, {"bar": 20, "baz": 30, "qux": 40}, {"quux": 50}, {}]
-        let entry_offsets = Buffer::from(&[0, 1, 1, 1, 4, 5, 5].to_byte_slice());
+        let entry_offsets = Buffer::from([0, 1, 1, 1, 4, 5, 5].to_byte_slice());
         let valid_buffer = Buffer::from([0b00111101]);
 
         let map_data = ArrayData::builder(map_data_type.clone())
@@ -1408,7 +1408,7 @@ mod tests {
             );
 
             // [{"list":[{"int32":1,"utf8":"a"},{"int32":null,"utf8":"b"}]},{"list":null},{"list":[{int32":5,"utf8":null}]},{"list":null}]
-            let entry_offsets = Buffer::from(&[0, 2, 2, 3, 3].to_byte_slice());
+            let entry_offsets = Buffer::from([0, 2, 2, 3, 3].to_byte_slice());
             let data = ArrayData::builder(field.data_type().clone())
                 .len(4)
                 .add_buffer(entry_offsets)
diff --git a/arrow-select/src/dictionary.rs b/arrow-select/src/dictionary.rs
index d0b6fcfc3..2a532600b 100644
--- a/arrow-select/src/dictionary.rs
+++ b/arrow-select/src/dictionary.rs
@@ -297,7 +297,7 @@ mod tests {
 
     #[test]
     fn test_merge_nulls() {
-        let buffer = Buffer::from("helloworldbingohelloworld");
+        let buffer = Buffer::from(b"helloworldbingohelloworld");
         let offsets = OffsetBuffer::from_lengths([5, 5, 5, 5, 5]);
         let nulls = NullBuffer::from(vec![true, false, true, true, true]);
         let values = StringArray::new(offsets, buffer, Some(nulls));
diff --git a/arrow-string/src/substring.rs b/arrow-string/src/substring.rs
index f5fe81103..fc2f6c8ad 100644
--- a/arrow-string/src/substring.rs
+++ b/arrow-string/src/substring.rs
@@ -732,7 +732,7 @@ mod tests {
     }
 
     fn generic_string_with_non_zero_offset<O: OffsetSizeTrait>() {
-        let values = "hellotherearrow";
+        let values = b"hellotherearrow";
         let offsets = &[
             O::zero(),
             O::from_usize(5).unwrap(),
@@ -867,7 +867,7 @@ mod tests {
         let data = ArrayData::builder(GenericStringArray::<O>::DATA_TYPE)
             .len(2)
             .add_buffer(Buffer::from_slice_ref(offsets))
-            .add_buffer(Buffer::from(values))
+            .add_buffer(Buffer::from(values.as_bytes()))
             .null_bit_buffer(Some(Buffer::from(bitmap)))
             .offset(1)
             .build()
diff --git a/arrow/examples/builders.rs b/arrow/examples/builders.rs
index ad6b87964..5c8cd51c5 100644
--- a/arrow/examples/builders.rs
+++ b/arrow/examples/builders.rs
@@ -88,13 +88,13 @@ fn main() {
     // buffer.
     let value_data = ArrayData::builder(DataType::Int32)
         .len(8)
-        .add_buffer(Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
+        .add_buffer(Buffer::from([0, 1, 2, 3, 4, 5, 6, 7].to_byte_slice()))
         .build()
         .unwrap();
 
     // Construct a buffer for value offsets, for the nested array:
     //  [[0, 1, 2], [3, 4, 5], [6, 7]]
-    let value_offsets = Buffer::from(&[0, 3, 6, 8].to_byte_slice());
+    let value_offsets = Buffer::from([0, 3, 6, 8].to_byte_slice());
 
     // Construct a list array from the above two
     let list_data_type = DataType::List(Arc::new(Field::new("item", DataType::Int32, false)));
diff --git a/arrow/examples/tensor_builder.rs b/arrow/examples/tensor_builder.rs
index 90ad1b486..45440533d 100644
--- a/arrow/examples/tensor_builder.rs
+++ b/arrow/examples/tensor_builder.rs
@@ -57,7 +57,7 @@ fn main() -> Result<()> {
 
     // In order to build a tensor from an array the function to_byte_slice add the
     // required padding to the elements in the array.
-    let buf = Buffer::from(&[0, 1, 2, 3, 4, 5, 6, 7, 9, 10].to_byte_slice());
+    let buf = Buffer::from([0, 1, 2, 3, 4, 5, 6, 7, 9, 10].to_byte_slice());
     let tensor = Int32Tensor::try_new(buf, Some(vec![2, 5]), None, None)?;
     println!("\nInt32 Tensor");
     println!("{tensor:?}");
diff --git a/object_store/src/local.rs b/object_store/src/local.rs
index 484738929..db4b4b050 100644
--- a/object_store/src/local.rs
+++ b/object_store/src/local.rs
@@ -297,7 +297,22 @@ impl LocalFileSystem {
                 path: location.as_ref()
             }
         );
-        self.config.prefix_to_filesystem(location)
+        let path = self.config.prefix_to_filesystem(location)?;
+
+        #[cfg(target_os = "windows")]
+        let path = {
+            let path = path.to_string_lossy();
+
+            // Assume the first char is the drive letter and the next is a colon.
+            let mut out = String::new();
+            let drive = &path[..2]; // The drive letter and colon (e.g., "C:")
+            let filepath = &path[2..].replace(':', "%3A"); // Replace subsequent colons
+            out.push_str(drive);
+            out.push_str(filepath);
+            PathBuf::from(out)
+        };
+
+        Ok(path)
     }
 
     /// Enable automatic cleanup of empty directories when deleting files
@@ -1053,6 +1068,7 @@ mod tests {
     use super::*;
 
     #[tokio::test]
+    #[cfg(target_family = "unix")]
     async fn file_test() {
         let root = TempDir::new().unwrap();
         let integration = LocalFileSystem::new_with_prefix(root.path()).unwrap();
@@ -1069,6 +1085,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(target_family = "unix")]
     fn test_non_tokio() {
         let root = TempDir::new().unwrap();
         let integration = LocalFileSystem::new_with_prefix(root.path()).unwrap();
@@ -1481,6 +1498,28 @@ mod tests {
         assert_eq!(list, vec![c, a]);
     }
 
+    #[tokio::test]
+    #[cfg(target_os = "windows")]
+    async fn filesystem_filename_with_colon() {
+        let root = TempDir::new().unwrap();
+        let integration = LocalFileSystem::new_with_prefix(root.path()).unwrap();
+        let path = Path::parse("file%3Aname.parquet").unwrap();
+        let location = Path::parse("file:name.parquet").unwrap();
+
+        integration.put(&location, "test".into()).await.unwrap();
+        let list = flatten_list_stream(&integration, None).await.unwrap();
+        assert_eq!(list, vec![path.clone()]);
+
+        let result = integration
+            .get(&location)
+            .await
+            .unwrap()
+            .bytes()
+            .await
+            .unwrap();
+        assert_eq!(result, Bytes::from("test"));
+    }
+
     #[tokio::test]
     async fn delete_dirs_automatically() {
         let root = TempDir::new().unwrap();
diff --git a/parquet/src/arrow/array_reader/byte_view_array.rs b/parquet/src/arrow/array_reader/byte_view_array.rs
index d1a0313dc..5845e2c08 100644
--- a/parquet/src/arrow/array_reader/byte_view_array.rs
+++ b/parquet/src/arrow/array_reader/byte_view_array.rs
@@ -28,6 +28,7 @@ use crate::encodings::decoding::{Decoder, DeltaBitPackDecoder};
 use crate::errors::{ParquetError, Result};
 use crate::schema::types::ColumnDescPtr;
 use arrow_array::{builder::make_view, ArrayRef};
+use arrow_buffer::Buffer;
 use arrow_data::ByteView;
 use arrow_schema::DataType as ArrowType;
 use bytes::Bytes;
@@ -666,7 +667,7 @@ impl ByteViewArrayDecoderDelta {
             v
         };
 
-        let actual_block_id = output.append_block(array_buffer.into());
+        let actual_block_id = output.append_block(Buffer::from_vec(array_buffer));
         assert_eq!(actual_block_id, buffer_id);
         Ok(read)
     }
diff --git a/parquet/src/arrow/array_reader/list_array.rs b/parquet/src/arrow/array_reader/list_array.rs
index 7c66c5c23..e1752f30c 100644
--- a/parquet/src/arrow/array_reader/list_array.rs
+++ b/parquet/src/arrow/array_reader/list_array.rs
@@ -213,7 +213,7 @@ impl<OffsetSize: OffsetSizeTrait> ArrayReader for ListArrayReader<OffsetSize> {
             return Err(general_err!("Failed to reconstruct list from level data"));
         }
 
-        let value_offsets = Buffer::from(&list_offsets.to_byte_slice());
+        let value_offsets = Buffer::from(list_offsets.to_byte_slice());
 
         let mut data_builder = ArrayData::builder(self.get_data_type().clone())
             .len(list_offsets.len() - 1)
diff --git a/parquet/src/arrow/arrow_writer/levels.rs b/parquet/src/arrow/arrow_writer/levels.rs
index c50e612b2..3e828bbdd 100644
--- a/parquet/src/arrow/arrow_writer/levels.rs
+++ b/parquet/src/arrow/arrow_writer/levels.rs
@@ -1050,7 +1050,7 @@ mod tests {
         let a_list_data = ArrayData::builder(a_list_type.clone())
             .len(5)
             .add_buffer(a_value_offsets)
-            .null_bit_buffer(Some(Buffer::from(vec![0b00011011])))
+            .null_bit_buffer(Some(Buffer::from([0b00011011])))
             .add_child_data(a_values.to_data())
             .build()
             .unwrap();
@@ -1116,7 +1116,7 @@ mod tests {
 
         // Construct a buffer for value offsets, for the nested array:
         //  [[1], [2, 3], null, [4, 5, 6], [7, 8, 9, 10]]
-        let g_value_offsets = arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+        let g_value_offsets = arrow::buffer::Buffer::from([0, 1, 3, 3, 6, 10].to_byte_slice());
 
         // Construct a list array from the above two
         let g_list_data = ArrayData::builder(struct_field_g.data_type().clone())
diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs
index f83e56c7a..cf46f3b64 100644
--- a/parquet/src/arrow/arrow_writer/mod.rs
+++ b/parquet/src/arrow/arrow_writer/mod.rs
@@ -1206,7 +1206,7 @@ mod tests {
 
         // Construct a buffer for value offsets, for the nested array:
         //  [[1], [2, 3], null, [4, 5, 6], [7, 8, 9, 10]]
-        let a_value_offsets = arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+        let a_value_offsets = arrow::buffer::Buffer::from([0, 1, 3, 3, 6, 10].to_byte_slice());
 
         // Construct a list array from the above two
         let a_list_data = ArrayData::builder(DataType::List(Arc::new(Field::new(
@@ -1217,7 +1217,7 @@ mod tests {
         .len(5)
         .add_buffer(a_value_offsets)
         .add_child_data(a_values.into_data())
-        .null_bit_buffer(Some(Buffer::from(vec![0b00011011])))
+        .null_bit_buffer(Some(Buffer::from([0b00011011])))
         .build()
         .unwrap();
         let a = ListArray::from(a_list_data);
@@ -1246,7 +1246,7 @@ mod tests {
 
         // Construct a buffer for value offsets, for the nested array:
         //  [[1], [2, 3], [], [4, 5, 6], [7, 8, 9, 10]]
-        let a_value_offsets = arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+        let a_value_offsets = arrow::buffer::Buffer::from([0, 1, 3, 3, 6, 10].to_byte_slice());
 
         // Construct a list array from the above two
         let a_list_data = ArrayData::builder(DataType::List(Arc::new(Field::new(
@@ -1405,7 +1405,7 @@ mod tests {
 
         // Construct a buffer for value offsets, for the nested array:
         //  [[1], [2, 3], [], [4, 5, 6], [7, 8, 9, 10]]
-        let g_value_offsets = arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+        let g_value_offsets = arrow::buffer::Buffer::from([0, 1, 3, 3, 6, 10].to_byte_slice());
 
         // Construct a list array from the above two
         let g_list_data = ArrayData::builder(struct_field_g.data_type().clone())
@@ -1420,7 +1420,7 @@ mod tests {
             .len(5)
             .add_buffer(g_value_offsets)
             .add_child_data(g_value.to_data())
-            .null_bit_buffer(Some(Buffer::from(vec![0b00011011])))
+            .null_bit_buffer(Some(Buffer::from([0b00011011])))
             .build()
             .unwrap();
         let h = ListArray::from(h_list_data);
@@ -1525,14 +1525,14 @@ mod tests {
         let c = Int32Array::from(vec![Some(1), None, Some(3), None, None, Some(6)]);
         let b_data = ArrayDataBuilder::new(field_b.data_type().clone())
             .len(6)
-            .null_bit_buffer(Some(Buffer::from(vec![0b00100111])))
+            .null_bit_buffer(Some(Buffer::from([0b00100111])))
             .add_child_data(c.into_data())
             .build()
             .unwrap();
         let b = StructArray::from(b_data);
         let a_data = ArrayDataBuilder::new(field_a.data_type().clone())
             .len(6)
-            .null_bit_buffer(Some(Buffer::from(vec![0b00101111])))
+            .null_bit_buffer(Some(Buffer::from([0b00101111])))
             .add_child_data(b.into_data())
             .build()
             .unwrap();
@@ -1595,7 +1595,7 @@ mod tests {
         let c = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
         let b_data = ArrayDataBuilder::new(type_b)
             .len(6)
-            .null_bit_buffer(Some(Buffer::from(vec![0b00100111])))
+            .null_bit_buffer(Some(Buffer::from([0b00100111])))
             .add_child_data(c.into_data())
             .build()
             .unwrap();
@@ -2280,7 +2280,7 @@ mod tests {
 
         // Build [[], null, [null, null]]
         let a_values = NullArray::new(2);
-        let a_value_offsets = arrow::buffer::Buffer::from(&[0, 0, 0, 2].to_byte_slice());
+        let a_value_offsets = arrow::buffer::Buffer::from([0, 0, 0, 2].to_byte_slice());
         let a_list_data = ArrayData::builder(DataType::List(Arc::new(Field::new(
             "item",
             DataType::Null,
@@ -2288,7 +2288,7 @@ mod tests {
         ))))
         .len(3)
         .add_buffer(a_value_offsets)
-        .null_bit_buffer(Some(Buffer::from(vec![0b00000101])))
+        .null_bit_buffer(Some(Buffer::from([0b00000101])))
         .add_child_data(a_values.into_data())
         .build()
         .unwrap();
@@ -2310,7 +2310,7 @@ mod tests {
     #[test]
     fn list_single_column() {
         let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
-        let a_value_offsets = arrow::buffer::Buffer::from(&[0, 1, 3, 3, 6, 10].to_byte_slice());
+        let a_value_offsets = arrow::buffer::Buffer::from([0, 1, 3, 3, 6, 10].to_byte_slice());
         let a_list_data = ArrayData::builder(DataType::List(Arc::new(Field::new(
             "item",
             DataType::Int32,
@@ -2318,7 +2318,7 @@ mod tests {
         ))))
         .len(5)
         .add_buffer(a_value_offsets)
-        .null_bit_buffer(Some(Buffer::from(vec![0b00011011])))
+        .null_bit_buffer(Some(Buffer::from([0b00011011])))
         .add_child_data(a_values.into_data())
         .build()
         .unwrap();
@@ -2334,7 +2334,7 @@ mod tests {
     #[test]
     fn large_list_single_column() {
         let a_values = Int32Array::from(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
-        let a_value_offsets = arrow::buffer::Buffer::from(&[0i64, 1, 3, 3, 6, 10].to_byte_slice());
+        let a_value_offsets = arrow::buffer::Buffer::from([0i64, 1, 3, 3, 6, 10].to_byte_slice());
         let a_list_data = ArrayData::builder(DataType::LargeList(Arc::new(Field::new(
             "large_item",
             DataType::Int32,
@@ -2343,7 +2343,7 @@ mod tests {
         .len(5)
         .add_buffer(a_value_offsets)
         .add_child_data(a_values.into_data())
-        .null_bit_buffer(Some(Buffer::from(vec![0b00011011])))
+        .null_bit_buffer(Some(Buffer::from([0b00011011])))
         .build()
         .unwrap();
 


Test patch:
diff --git a/arrow-integration-test/src/lib.rs b/arrow-integration-test/src/lib.rs
index 66fa9f332..d1486fd5a 100644
--- a/arrow-integration-test/src/lib.rs
+++ b/arrow-integration-test/src/lib.rs
@@ -696,7 +696,7 @@ pub fn array_from_json(
             let list_data = ArrayData::builder(field.data_type().clone())
                 .len(json_col.count)
                 .offset(0)
-                .add_buffer(Buffer::from(&offsets.to_byte_slice()))
+                .add_buffer(Buffer::from(offsets.to_byte_slice()))
                 .add_child_data(child_array.into_data())
                 .null_bit_buffer(Some(null_buf))
                 .build()
@@ -720,7 +720,7 @@ pub fn array_from_json(
             let list_data = ArrayData::builder(field.data_type().clone())
                 .len(json_col.count)
                 .offset(0)
-                .add_buffer(Buffer::from(&offsets.to_byte_slice()))
+                .add_buffer(Buffer::from(offsets.to_byte_slice()))
                 .add_child_data(child_array.into_data())
                 .null_bit_buffer(Some(null_buf))
                 .build()
@@ -839,7 +839,7 @@ pub fn array_from_json(
                 .collect();
             let array_data = ArrayData::builder(field.data_type().clone())
                 .len(json_col.count)
-                .add_buffer(Buffer::from(&offsets.to_byte_slice()))
+                .add_buffer(Buffer::from(offsets.to_byte_slice()))
                 .add_child_data(child_array.into_data())
                 .null_bit_buffer(Some(null_buf))
                 .build()
diff --git a/arrow-integration-testing/src/flight_client_scenarios/integration_test.rs b/arrow-integration-testing/src/flight_client_scenarios/integration_test.rs
index ec88ce36a..1a6c4e28a 100644
--- a/arrow-integration-testing/src/flight_client_scenarios/integration_test.rs
+++ b/arrow-integration-testing/src/flight_client_scenarios/integration_test.rs
@@ -262,7 +262,7 @@ async fn receive_batch_flight_data(
 
     while message.header_type() == ipc::MessageHeader::DictionaryBatch {
         reader::read_dictionary(
-            &Buffer::from(&data.data_body),
+            &Buffer::from(data.data_body.as_ref()),
             message
                 .header_as_dictionary_batch()
                 .expect("Error parsing dictionary"),
diff --git a/arrow-integration-testing/src/flight_server_scenarios/integration_test.rs b/arrow-integration-testing/src/flight_server_scenarios/integration_test.rs
index a03c1cd1a..76eb9d880 100644
--- a/arrow-integration-testing/src/flight_server_scenarios/integration_test.rs
+++ b/arrow-integration-testing/src/flight_server_scenarios/integration_test.rs
@@ -364,7 +364,7 @@ async fn save_uploaded_chunks(
 
                 let batch = record_batch_from_message(
                     message,
-                    &Buffer::from(data.data_body),
+                    &Buffer::from(data.data_body.as_ref()),
                     schema_ref.clone(),
                     &dictionaries_by_id,
                 )
@@ -375,7 +375,7 @@ async fn save_uploaded_chunks(
             ipc::MessageHeader::DictionaryBatch => {
                 dictionary_from_message(
                     message,
-                    &Buffer::from(data.data_body),
+                    &Buffer::from(data.data_body.as_ref()),
                     schema_ref.clone(),
                     &mut dictionaries_by_id,
                 )
diff --git a/arrow/tests/array_equal.rs b/arrow/tests/array_equal.rs
index 15011c547..7ed4dae1e 100644
--- a/arrow/tests/array_equal.rs
+++ b/arrow/tests/array_equal.rs
@@ -445,7 +445,7 @@ fn test_empty_offsets_list_equal() {
     .len(0)
     .add_buffer(Buffer::from([0i32, 2, 3, 4, 6, 7, 8].to_byte_slice()))
     .add_child_data(Int32Array::from(vec![1, 2, -1, -2, 3, 4, -3, -4]).into_data())
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001001])))
+    .null_bit_buffer(Some(Buffer::from([0b00001001])))
     .build()
     .unwrap()
     .into();
@@ -483,7 +483,7 @@ fn test_list_null() {
     .len(6)
     .add_buffer(Buffer::from([0i32, 2, 3, 4, 6, 7, 8].to_byte_slice()))
     .add_child_data(c_values.into_data())
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001001])))
+    .null_bit_buffer(Some(Buffer::from([0b00001001])))
     .build()
     .unwrap()
     .into();
@@ -506,7 +506,7 @@ fn test_list_null() {
     .len(6)
     .add_buffer(Buffer::from([0i32, 2, 3, 4, 6, 7, 8].to_byte_slice()))
     .add_child_data(d_values.into_data())
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001001])))
+    .null_bit_buffer(Some(Buffer::from([0b00001001])))
     .build()
     .unwrap()
     .into();
@@ -807,7 +807,7 @@ fn test_struct_equal_null() {
         Field::new("f1", DataType::Utf8, true),
         Field::new("f2", DataType::Int32, true),
     ])))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001011])))
+    .null_bit_buffer(Some(Buffer::from([0b00001011])))
     .len(5)
     .add_child_data(strings.to_data())
     .add_child_data(ints.to_data())
@@ -819,7 +819,7 @@ fn test_struct_equal_null() {
         Field::new("f1", DataType::Utf8, true),
         Field::new("f2", DataType::Int32, true),
     ])))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001011])))
+    .null_bit_buffer(Some(Buffer::from([0b00001011])))
     .len(5)
     .add_child_data(strings.to_data())
     .add_child_data(ints_non_null.to_data())
@@ -835,7 +835,7 @@ fn test_struct_equal_null() {
         Field::new("f1", DataType::Utf8, true),
         Field::new("f2", DataType::Int32, true),
     ])))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001011])))
+    .null_bit_buffer(Some(Buffer::from([0b00001011])))
     .len(5)
     .add_child_data(strings.to_data())
     .add_child_data(c_ints_non_null.to_data())
@@ -849,7 +849,7 @@ fn test_struct_equal_null() {
     let a = ArrayData::builder(DataType::Struct(
         vec![Field::new("f3", a.data_type().clone(), true)].into(),
     ))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00011110])))
+    .null_bit_buffer(Some(Buffer::from([0b00011110])))
     .len(5)
     .add_child_data(a.to_data())
     .build()
@@ -868,7 +868,7 @@ fn test_struct_equal_null() {
         Field::new("f1", DataType::Utf8, true),
         Field::new("f2", DataType::Int32, true),
     ])))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001011])))
+    .null_bit_buffer(Some(Buffer::from([0b00001011])))
     .len(5)
     .add_child_data(strings.to_data())
     .add_child_data(ints_non_null.to_data())
@@ -878,7 +878,7 @@ fn test_struct_equal_null() {
     let b = ArrayData::builder(DataType::Struct(
         vec![Field::new("f3", b.data_type().clone(), true)].into(),
     ))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00011110])))
+    .null_bit_buffer(Some(Buffer::from([0b00011110])))
     .len(5)
     .add_child_data(b)
     .build()
@@ -909,7 +909,7 @@ fn test_struct_equal_null_variable_size() {
     let a = ArrayData::builder(DataType::Struct(
         vec![Field::new("f1", DataType::Utf8, true)].into(),
     ))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001010])))
+    .null_bit_buffer(Some(Buffer::from([0b00001010])))
     .len(5)
     .add_child_data(strings1.to_data())
     .build()
@@ -919,7 +919,7 @@ fn test_struct_equal_null_variable_size() {
     let b = ArrayData::builder(DataType::Struct(
         vec![Field::new("f1", DataType::Utf8, true)].into(),
     ))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001010])))
+    .null_bit_buffer(Some(Buffer::from([0b00001010])))
     .len(5)
     .add_child_data(strings2.to_data())
     .build()
@@ -939,7 +939,7 @@ fn test_struct_equal_null_variable_size() {
     let c = ArrayData::builder(DataType::Struct(
         vec![Field::new("f1", DataType::Utf8, true)].into(),
     ))
-    .null_bit_buffer(Some(Buffer::from(vec![0b00001011])))
+    .null_bit_buffer(Some(Buffer::from([0b00001011])))
     .len(5)
     .add_child_data(strings3.to_data())
     .build()
diff --git a/arrow/tests/array_validation.rs b/arrow/tests/array_validation.rs
index 41def9051..1321f10f9 100644
--- a/arrow/tests/array_validation.rs
+++ b/arrow/tests/array_validation.rs
@@ -63,7 +63,7 @@ fn test_fixed_width_overflow() {
 #[should_panic(expected = "null_bit_buffer size too small. got 1 needed 2")]
 fn test_bitmap_too_small() {
     let buffer = make_i32_buffer(9);
-    let null_bit_buffer = Buffer::from(vec![0b11111111]);
+    let null_bit_buffer = Buffer::from([0b11111111]);
 
     ArrayData::try_new(
         DataType::Int32,