Repository: apache/arrow-rs
Pull request #5373 (2024-02-07 18:33:40)

feat: utility functions for creating `FixedSizeList` and `LargeList` dtypes
# Which issue does this PR close?

<!--
We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
-->

Closes https://github.com/apache/arrow-rs/issues/5372

 
<!--
Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes.
-->

# What changes are included in this PR?

<!--
There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
-->

adds a few new helper functions to `DataType` and `Field` for initializing `FixedSizeList` and `LargeList`

# Are there any user-facing changes?

just the aforementioned helper functions.

<!--
If there are user-facing changes then we may require documentation to be updated before approving the PR.
-->

<!---
If there are any breaking changes to public APIs, please add the `breaking change` label.
-->



Patch:
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 9c4b28b69..d8d892ea9 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -115,6 +115,8 @@ jobs:
         uses: ./.github/actions/setup-builder
       - name: Install cargo-msrv
         run: cargo install cargo-msrv
+      - name: Downgrade arrow dependencies
+        run: cargo update -p ahash --precise 0.8.7
       - name: Check arrow
         working-directory: arrow
         run: cargo msrv verify
diff --git a/Cargo.toml b/Cargo.toml
index 57e986cfc..bbbf907ba 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -93,4 +93,4 @@ arrow-select = { version = "50.0.0", path = "./arrow-select" }
 arrow-string = { version = "50.0.0", path = "./arrow-string" }
 parquet = { version = "50.0.0", path = "./parquet", default-features = false }
 
-chrono = { version = "0.4.31", default-features = false, features = ["clock"] }
+chrono = { version = "0.4.34", default-features = false, features = ["clock"] }
diff --git a/arrow-arith/src/arity.rs b/arrow-arith/src/arity.rs
index ff8b82a5d..3d8214d89 100644
--- a/arrow-arith/src/arity.rs
+++ b/arrow-arith/src/arity.rs
@@ -358,17 +358,16 @@ where
 
         let slice = builder.values_slice_mut();
 
-        match nulls.try_for_each_valid_idx(|idx| {
+        let r = nulls.try_for_each_valid_idx(|idx| {
             unsafe {
                 *slice.get_unchecked_mut(idx) =
                     op(*slice.get_unchecked(idx), b.value_unchecked(idx))?
             };
             Ok::<_, ArrowError>(())
-        }) {
-            Ok(_) => {}
-            Err(err) => return Ok(Err(err)),
-        };
-
+        });
+        if let Err(err) = r {
+            return Ok(Err(err));
+        }
         let array_builder = builder.finish().into_data().into_builder();
         let array_data = unsafe { array_builder.nulls(Some(nulls)).build_unchecked() };
         Ok(Ok(PrimitiveArray::<T>::from(array_data)))
diff --git a/arrow-array/src/array/list_array.rs b/arrow-array/src/array/list_array.rs
index 9758c112a..ba3719392 100644
--- a/arrow-array/src/array/list_array.rs
+++ b/arrow-array/src/array/list_array.rs
@@ -705,8 +705,7 @@ mod tests {
         let value_offsets = Buffer::from_slice_ref([0i64, 3, 6, 8]);
 
         // Construct a list array from the above two
-        let list_data_type =
-            DataType::LargeList(Arc::new(Field::new("item", DataType::Int32, false)));
+        let list_data_type = DataType::new_large_list(DataType::Int32, false);
         let list_data = ArrayData::builder(list_data_type.clone())
             .len(3)
             .add_buffer(value_offsets.clone())
@@ -863,8 +862,7 @@ mod tests {
         bit_util::set_bit(&mut null_bits, 8);
 
         // Construct a list array from the above two
-        let list_data_type =
-            DataType::LargeList(Arc::new(Field::new("item", DataType::Int32, false)));
+        let list_data_type = DataType::new_large_list(DataType::Int32, false);
         let list_data = ArrayData::builder(list_data_type)
             .len(9)
             .add_buffer(value_offsets)
@@ -929,8 +927,7 @@ mod tests {
         bit_util::set_bit(&mut null_bits, 8);
 
         // Construct a list array from the above two
-        let list_data_type =
-            DataType::LargeList(Arc::new(Field::new("item", DataType::Int32, false)));
+        let list_data_type = DataType::new_large_list(DataType::Int32, false);
         let list_data = ArrayData::builder(list_data_type)
             .len(9)
             .add_buffer(value_offsets)
diff --git a/arrow-array/src/array/primitive_array.rs b/arrow-array/src/array/primitive_array.rs
index a6d5c2277..ca437a569 100644
--- a/arrow-array/src/array/primitive_array.rs
+++ b/arrow-array/src/array/primitive_array.rs
@@ -856,13 +856,14 @@ impl<T: ArrowPrimitiveType> PrimitiveArray<T> {
 
         let (slice, null_buffer) = builder.slices_mut();
 
-        match try_for_each_valid_idx(len, 0, null_count, null_buffer.as_deref(), |idx| {
+        let r = try_for_each_valid_idx(len, 0, null_count, null_buffer.as_deref(), |idx| {
             unsafe { *slice.get_unchecked_mut(idx) = op(*slice.get_unchecked(idx))? };
             Ok::<_, E>(())
-        }) {
-            Ok(_) => {}
-            Err(err) => return Ok(Err(err)),
-        };
+        });
+
+        if let Err(err) = r {
+            return Ok(Err(err));
+        }
 
         Ok(Ok(builder.finish()))
     }
diff --git a/arrow-array/src/record_batch.rs b/arrow-array/src/record_batch.rs
index dab6ae343..d89020a65 100644
--- a/arrow-array/src/record_batch.rs
+++ b/arrow-array/src/record_batch.rs
@@ -355,7 +355,7 @@ impl RecordBatch {
     /// assert_eq!(batch.num_columns(), 1);
     /// ```
     pub fn remove_column(&mut self, index: usize) -> ArrayRef {
-        let mut builder = SchemaBuilder::from(self.schema.fields());
+        let mut builder = SchemaBuilder::from(self.schema.as_ref());
         builder.remove(index);
         self.schema = Arc::new(builder.finish());
         self.columns.remove(index)
@@ -618,6 +618,8 @@ where
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use super::*;
     use crate::{BooleanArray, Int32Array, Int64Array, Int8Array, ListArray, StringArray};
     use arrow_buffer::{Buffer, ToByteSlice};
@@ -1155,4 +1157,31 @@ mod tests {
         let size = get_size(reader);
         assert_eq!(size, 0);
     }
+
+    #[test]
+    fn test_remove_column_maintains_schema_metadata() {
+        let id_array = Int32Array::from(vec![1, 2, 3, 4, 5]);
+        let bool_array = BooleanArray::from(vec![true, false, false, true, true]);
+
+        let mut metadata = HashMap::new();
+        metadata.insert("foo".to_string(), "bar".to_string());
+        let schema = Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("bool", DataType::Boolean, false),
+        ])
+        .with_metadata(metadata);
+
+        let mut batch = RecordBatch::try_new(
+            Arc::new(schema),
+            vec![Arc::new(id_array), Arc::new(bool_array)],
+        )
+        .unwrap();
+
+        let _removed_column = batch.remove_column(0);
+        assert_eq!(batch.schema().metadata().len(), 1);
+        assert_eq!(
+            batch.schema().metadata().get("foo").unwrap().as_str(),
+            "bar"
+        );
+    }
 }
diff --git a/arrow-cast/src/display.rs b/arrow-cast/src/display.rs
index edf7c9394..d74128a20 100644
--- a/arrow-cast/src/display.rs
+++ b/arrow-cast/src/display.rs
@@ -1013,9 +1013,9 @@ mod tests {
         assert_eq!(pretty[2], "0 days 0 hours 0 mins 0.000001000 secs");
         assert_eq!(iso[3], "-PT0.000001S");
         assert_eq!(pretty[3], "0 days 0 hours 0 mins -0.000001000 secs");
-        assert_eq!(iso[4], "P45DT50554.123456789S");
+        assert_eq!(iso[4], "PT3938554.123456789S");
         assert_eq!(pretty[4], "45 days 14 hours 2 mins 34.123456789 secs");
-        assert_eq!(iso[5], "-P45DT50554.123456789S");
+        assert_eq!(iso[5], "-PT3938554.123456789S");
         assert_eq!(pretty[5], "-45 days -14 hours -2 mins -34.123456789 secs");
 
         let array = DurationMicrosecondArray::from(vec![
@@ -1037,9 +1037,9 @@ mod tests {
         assert_eq!(pretty[2], "0 days 0 hours 0 mins 0.001000 secs");
         assert_eq!(iso[3], "-PT0.001S");
         assert_eq!(pretty[3], "0 days 0 hours 0 mins -0.001000 secs");
-        assert_eq!(iso[4], "P45DT50554.123456S");
+        assert_eq!(iso[4], "PT3938554.123456S");
         assert_eq!(pretty[4], "45 days 14 hours 2 mins 34.123456 secs");
-        assert_eq!(iso[5], "-P45DT50554.123456S");
+        assert_eq!(iso[5], "-PT3938554.123456S");
         assert_eq!(pretty[5], "-45 days -14 hours -2 mins -34.123456 secs");
 
         let array = DurationMillisecondArray::from(vec![
@@ -1061,9 +1061,9 @@ mod tests {
         assert_eq!(pretty[2], "0 days 0 hours 0 mins 1.000 secs");
         assert_eq!(iso[3], "-PT1S");
         assert_eq!(pretty[3], "0 days 0 hours 0 mins -1.000 secs");
-        assert_eq!(iso[4], "P45DT50554.123S");
+        assert_eq!(iso[4], "PT3938554.123S");
         assert_eq!(pretty[4], "45 days 14 hours 2 mins 34.123 secs");
-        assert_eq!(iso[5], "-P45DT50554.123S");
+        assert_eq!(iso[5], "-PT3938554.123S");
         assert_eq!(pretty[5], "-45 days -14 hours -2 mins -34.123 secs");
 
         let array = DurationSecondArray::from(vec![
@@ -1085,9 +1085,9 @@ mod tests {
         assert_eq!(pretty[2], "0 days 0 hours 16 mins 40 secs");
         assert_eq!(iso[3], "-PT1000S");
         assert_eq!(pretty[3], "0 days 0 hours -16 mins -40 secs");
-        assert_eq!(iso[4], "P45DT50554S");
+        assert_eq!(iso[4], "PT3938554S");
         assert_eq!(pretty[4], "45 days 14 hours 2 mins 34 secs");
-        assert_eq!(iso[5], "-P45DT50554S");
+        assert_eq!(iso[5], "-PT3938554S");
         assert_eq!(pretty[5], "-45 days -14 hours -2 mins -34 secs");
     }
 
diff --git a/arrow-flight/Cargo.toml b/arrow-flight/Cargo.toml
index 1bea347c3..81940b760 100644
--- a/arrow-flight/Cargo.toml
+++ b/arrow-flight/Cargo.toml
@@ -46,7 +46,7 @@ once_cell = { version = "1", optional = true }
 paste = { version = "1.0" }
 prost = { version = "0.12.1", default-features = false, features = ["prost-derive"] }
 tokio = { version = "1.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] }
-tonic = { version = "0.10.0", default-features = false, features = ["transport", "codegen", "prost"] }
+tonic = { version = "0.11.0", default-features = false, features = ["transport", "codegen", "prost"] }
 
 # CLI-related dependencies
 anyhow = { version = "1.0", optional = true }
diff --git a/arrow-flight/gen/Cargo.toml b/arrow-flight/gen/Cargo.toml
index c0d27b948..2902b215c 100644
--- a/arrow-flight/gen/Cargo.toml
+++ b/arrow-flight/gen/Cargo.toml
@@ -34,4 +34,4 @@ publish = false
 # (and checked in) arrow.flight.protocol.rs from changing
 proc-macro2 = { version = "=1.0.78", default-features = false }
 prost-build = { version = "=0.12.3", default-features = false }
-tonic-build = { version = "=0.10.2", default-features = false, features = ["transport", "prost"] }
+tonic-build = { version = "=0.11.0", default-features = false, features = ["transport", "prost"] }
diff --git a/arrow-ipc/src/convert.rs b/arrow-ipc/src/convert.rs
index b290a09ac..505541f7d 100644
--- a/arrow-ipc/src/convert.rs
+++ b/arrow-ipc/src/convert.rs
@@ -877,6 +877,12 @@ mod tests {
                 Field::new("utf8", DataType::Utf8, false),
                 Field::new("binary", DataType::Binary, false),
                 Field::new_list("list[u8]", Field::new("item", DataType::UInt8, false), true),
+                Field::new_fixed_size_list(
+                    "fixed_size_list[u8]",
+                    Field::new("item", DataType::UInt8, false),
+                    2,
+                    true,
+                ),
                 Field::new_list(
                     "list[struct<float32, int32, bool>]",
                     Field::new_struct(
diff --git a/arrow-json/src/writer.rs b/arrow-json/src/writer.rs
index acca70290..bb494b595 100644
--- a/arrow-json/src/writer.rs
+++ b/arrow-json/src/writer.rs
@@ -833,7 +833,7 @@ mod tests {
     use serde_json::json;
 
     use arrow_array::builder::{Int32Builder, Int64Builder, MapBuilder, StringBuilder};
-    use arrow_buffer::{Buffer, ToByteSlice};
+    use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer, ToByteSlice};
     use arrow_data::ArrayData;
 
     use crate::reader::*;
@@ -953,6 +953,93 @@ mod tests {
         );
     }
 
+    #[test]
+    fn write_list_of_dictionary() {
+        let dict_field = Arc::new(Field::new_dictionary(
+            "item",
+            DataType::Int32,
+            DataType::Utf8,
+            true,
+        ));
+        let schema = Schema::new(vec![Field::new_large_list("l", dict_field.clone(), true)]);
+
+        let dict_array: DictionaryArray<Int32Type> =
+            vec![Some("a"), Some("b"), Some("c"), Some("a"), None, Some("c")]
+                .into_iter()
+                .collect();
+        let list_array = LargeListArray::try_new(
+            dict_field,
+            OffsetBuffer::from_lengths([3_usize, 2, 0, 1]),
+            Arc::new(dict_array),
+            Some(NullBuffer::from_iter([true, true, false, true])),
+        )
+        .unwrap();
+
+        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_array)]).unwrap();
+
+        let mut buf = Vec::new();
+        {
+            let mut writer = LineDelimitedWriter::new(&mut buf);
+            writer.write_batches(&[&batch]).unwrap();
+        }
+
+        assert_json_eq(
+            &buf,
+            r#"{"l":["a","b","c"]}
+{"l":["a",null]}
+{}
+{"l":["c"]}
+"#,
+        );
+    }
+
+    #[test]
+    fn write_list_of_dictionary_large_values() {
+        let dict_field = Arc::new(Field::new_dictionary(
+            "item",
+            DataType::Int32,
+            DataType::LargeUtf8,
+            true,
+        ));
+        let schema = Schema::new(vec![Field::new_large_list("l", dict_field.clone(), true)]);
+
+        let keys = PrimitiveArray::<Int32Type>::from(vec![
+            Some(0),
+            Some(1),
+            Some(2),
+            Some(0),
+            None,
+            Some(2),
+        ]);
+        let values = LargeStringArray::from(vec!["a", "b", "c"]);
+        let dict_array = DictionaryArray::try_new(keys, Arc::new(values)).unwrap();
+
+        let list_array = LargeListArray::try_new(
+            dict_field,
+            OffsetBuffer::from_lengths([3_usize, 2, 0, 1]),
+            Arc::new(dict_array),
+            Some(NullBuffer::from_iter([true, true, false, true])),
+        )
+        .unwrap();
+
+        let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_array)]).unwrap();
+
+        let mut buf = Vec::new();
+        {
+            let mut writer = LineDelimitedWriter::new(&mut buf);
+            writer.write_batches(&[&batch]).unwrap();
+        }
+
+        assert_json_eq(
+            &buf,
+            r#"{"l":["a","b","c"]}
+{"l":["a",null]}
+{}
+{"l":["c"]}
+"#,
+        );
+    }
+
     #[test]
     fn write_timestamps() {
         let ts_string = "2018-11-13T17:11:10.011375885995";
@@ -1192,7 +1279,7 @@ mod tests {
 
         assert_json_eq(
             &buf,
-            r#"{"duration_sec":"PT120S","duration_msec":"PT0.120S","duration_usec":"PT0.000120S","duration_nsec":"PT0.000000120S","name":"a"}
+            r#"{"duration_sec":"PT120S","duration_msec":"PT0.12S","duration_usec":"PT0.00012S","duration_nsec":"PT0.00000012S","name":"a"}
 {"name":"b"}
 "#,
         );
diff --git a/arrow-row/src/variable.rs b/arrow-row/src/variable.rs
index 4451c5287..466214e75 100644
--- a/arrow-row/src/variable.rs
+++ b/arrow-row/src/variable.rs
@@ -85,7 +85,7 @@ pub fn encode<'a, I: Iterator<Item = Option<&'a [u8]>>>(
 
 pub fn encode_one(out: &mut [u8], val: Option<&[u8]>, opts: SortOptions) -> usize {
     match val {
-        Some(val) if val.is_empty() => {
+        Some([]) => {
             out[0] = match opts.descending {
                 true => !EMPTY_SENTINEL,
                 false => EMPTY_SENTINEL,
diff --git a/arrow-schema/src/datatype.rs b/arrow-schema/src/datatype.rs
index a5bd66b50..079b855ce 100644
--- a/arrow-schema/src/datatype.rs
+++ b/arrow-schema/src/datatype.rs
@@ -608,6 +608,24 @@ impl DataType {
     pub fn new_list(data_type: DataType, nullable: bool) -> Self {
         DataType::List(Arc::new(Field::new_list_field(data_type, nullable)))
     }
+
+    /// Create a [`DataType::LargeList`] with elements of the specified type
+    /// and nullability, and conventionally named inner [`Field`] (`"item"`).
+    ///
+    /// To specify field level metadata, construct the inner [`Field`]
+    /// directly via [`Field::new`] or [`Field::new_list_field`].
+    pub fn new_large_list(data_type: DataType, nullable: bool) -> Self {
+        DataType::LargeList(Arc::new(Field::new_list_field(data_type, nullable)))
+    }
+
+    /// Create a [`DataType::FixedSizeList`] with elements of the specified type, size
+    /// and nullability, and conventionally named inner [`Field`] (`"item"`).
+    ///
+    /// To specify field level metadata, construct the inner [`Field`]
+    /// directly via [`Field::new`] or [`Field::new_list_field`].
+    pub fn new_fixed_size_list(data_type: DataType, size: i32, nullable: bool) -> Self {
+        DataType::FixedSizeList(Arc::new(Field::new_list_field(data_type, nullable)), size)
+    }
 }
 
 /// The maximum precision for [DataType::Decimal128] values
diff --git a/arrow-schema/src/field.rs b/arrow-schema/src/field.rs
index eebaf0e3c..5cb2399f3 100644
--- a/arrow-schema/src/field.rs
+++ b/arrow-schema/src/field.rs
@@ -217,6 +217,21 @@ impl Field {
         Self::new(name, DataType::LargeList(value.into()), nullable)
     }
 
+    /// Create a new [`Field`] with [`DataType::FixedSizeList`]
+    ///
+    /// - `name`: the name of the [`DataType::FixedSizeList`] field
+    /// - `value`: the description of each list element
+    /// - `size`: the size of the fixed size list
+    /// - `nullable`: if the [`DataType::FixedSizeList`] array is nullable
+    pub fn new_fixed_size_list(
+        name: impl Into<String>,
+        value: impl Into<FieldRef>,
+        size: i32,
+        nullable: bool,
+    ) -> Self {
+        Self::new(name, DataType::FixedSizeList(value.into(), size), nullable)
+    }
+
     /// Create a new [`Field`] with [`DataType::Map`]
     ///
     /// - `name`: the name of the [`DataType::Map`] field
diff --git a/arrow-schema/src/fields.rs b/arrow-schema/src/fields.rs
index 400f42c59..59b7e76c7 100644
--- a/arrow-schema/src/fields.rs
+++ b/arrow-schema/src/fields.rs
@@ -444,11 +444,7 @@ mod tests {
                 Field::new("floats", DataType::Struct(floats.clone()), true),
                 true,
             ),
-            Field::new(
-                "f",
-                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int32, false)), 3),
-                false,
-            ),
+            Field::new_fixed_size_list("f", Field::new("item", DataType::Int32, false), 3, false),
             Field::new_map(
                 "g",
                 "entries",
diff --git a/arrow-schema/src/schema.rs b/arrow-schema/src/schema.rs
index e547e5df3..ede158fcf 100644
--- a/arrow-schema/src/schema.rs
+++ b/arrow-schema/src/schema.rs
@@ -140,6 +140,12 @@ impl From<Fields> for SchemaBuilder {
     }
 }
 
+impl From<&Schema> for SchemaBuilder {
+    fn from(value: &Schema) -> Self {
+        Self::from(value.clone())
+    }
+}
+
 impl From<Schema> for SchemaBuilder {
     fn from(value: Schema) -> Self {
         Self {
diff --git a/arrow-select/src/filter.rs b/arrow-select/src/filter.rs
index ce51ecb58..c365d0b84 100644
--- a/arrow-select/src/filter.rs
+++ b/arrow-select/src/filter.rs
@@ -1261,8 +1261,7 @@ mod tests {
             .add_buffer(Buffer::from_slice_ref([0, 1, 2, 3, 4, 5, 6, 7, 8]))
             .build()
             .unwrap();
-        let list_data_type =
-            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int32, false)), 3);
+        let list_data_type = DataType::new_fixed_size_list(DataType::Int32, 3, false);
         let list_data = ArrayData::builder(list_data_type)
             .len(3)
             .add_child_data(value_data)
@@ -1318,8 +1317,7 @@ mod tests {
         bit_util::set_bit(&mut null_bits, 3);
         bit_util::set_bit(&mut null_bits, 4);
 
-        let list_data_type =
-            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Int32, false)), 2);
+        let list_data_type = DataType::new_fixed_size_list(DataType::Int32, 2, false);
         let list_data = ArrayData::builder(list_data_type)
             .len(5)
             .add_child_data(value_data)
diff --git a/arrow-select/src/take.rs b/arrow-select/src/take.rs
index 172e61dca..d9a639da8 100644
--- a/arrow-select/src/take.rs
+++ b/arrow-select/src/take.rs
@@ -420,7 +420,11 @@ fn take_bytes<T: ByteArrayType, IndexType: ArrowPrimitiveType>(
         nulls = Some(null_buf.into())
     }
 
-    T::Offset::from_usize(values.len()).expect("offset overflow");
+    T::Offset::from_usize(values.len()).ok_or(ArrowError::ComputeError(format!(
+        "Offset overflow for {}BinaryArray: {}",
+        T::Offset::PREFIX,
+        values.len()
+    )))?;
 
     let array_data = ArrayData::builder(T::DATA_TYPE)
         .len(data_len)
diff --git a/arrow-string/Cargo.toml b/arrow-string/Cargo.toml
index 1ae7af8bd..bdfa68111 100644
--- a/arrow-string/Cargo.toml
+++ b/arrow-string/Cargo.toml
@@ -42,3 +42,4 @@ arrow-select = { workspace = true }
 regex = { version = "1.7.0", default-features = false, features = ["std", "unicode", "perf"] }
 regex-syntax = { version = "0.8.0", default-features = false, features = ["unicode"] }
 num = { version = "0.4", default-features = false, features = ["std"] }
+memchr = "2.7.1"
diff --git a/arrow-string/src/predicate.rs b/arrow-string/src/predicate.rs
index fe288f9de..54ecf42e3 100644
--- a/arrow-string/src/predicate.rs
+++ b/arrow-string/src/predicate.rs
@@ -17,6 +17,7 @@
 
 use arrow_array::{BooleanArray, GenericStringArray, OffsetSizeTrait};
 use arrow_schema::ArrowError;
+use memchr::memchr2;
 use regex::{Regex, RegexBuilder};
 
 /// A string based predicate
@@ -39,19 +40,19 @@ pub enum Predicate<'a> {
 impl<'a> Predicate<'a> {
     /// Create a predicate for the given like pattern
     pub fn like(pattern: &'a str) -> Result<Self, ArrowError> {
-        if !pattern.contains(is_like_pattern) {
+        if !contains_like_pattern(pattern) {
             Ok(Self::Eq(pattern))
         } else if pattern.ends_with('%')
             && !pattern.ends_with("\\%")
-            && !pattern[..pattern.len() - 1].contains(is_like_pattern)
+            && !contains_like_pattern(&pattern[..pattern.len() - 1])
         {
             Ok(Self::StartsWith(&pattern[..pattern.len() - 1]))
-        } else if pattern.starts_with('%') && !pattern[1..].contains(is_like_pattern) {
+        } else if pattern.starts_with('%') && !contains_like_pattern(&pattern[1..]) {
             Ok(Self::EndsWith(&pattern[1..]))
         } else if pattern.starts_with('%')
             && pattern.ends_with('%')
             && !pattern.ends_with("\\%")
-            && !pattern[1..pattern.len() - 1].contains(is_like_pattern)
+            && !contains_like_pattern(&pattern[1..pattern.len() - 1])
         {
             Ok(Self::Contains(&pattern[1..pattern.len() - 1]))
         } else {
@@ -62,14 +63,14 @@ impl<'a> Predicate<'a> {
     /// Create a predicate for the given ilike pattern
     pub fn ilike(pattern: &'a str, is_ascii: bool) -> Result<Self, ArrowError> {
         if is_ascii && pattern.is_ascii() {
-            if !pattern.contains(is_like_pattern) {
+            if !contains_like_pattern(pattern) {
                 return Ok(Self::IEqAscii(pattern));
             } else if pattern.ends_with('%')
                 && !pattern.ends_with("\\%")
-                && !pattern[..pattern.len() - 1].contains(is_like_pattern)
+                && !contains_like_pattern(&pattern[..pattern.len() - 1])
             {
                 return Ok(Self::IStartsWithAscii(&pattern[..pattern.len() - 1]));
-            } else if pattern.starts_with('%') && !pattern[1..].contains(is_like_pattern) {
+            } else if pattern.starts_with('%') && !contains_like_pattern(&pattern[1..]) {
                 return Ok(Self::IEndsWithAscii(&pattern[1..]));
             }
         }
@@ -188,6 +189,10 @@ fn is_like_pattern(c: char) -> bool {
     c == '%' || c == '_'
 }
 
+fn contains_like_pattern(pattern: &str) -> bool {
+    memchr2(b'%', b'_', pattern.as_bytes()).is_some()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/object_store/src/azure/builder.rs b/object_store/src/azure/builder.rs
index 905fa5216..530095ff7 100644
--- a/object_store/src/azure/builder.rs
+++ b/object_store/src/azure/builder.rs
@@ -474,6 +474,7 @@ impl MicrosoftAzureBuilder {
     /// - `azure://<container>/<path>` (custom)
     /// - `https://<account>.dfs.core.windows.net`
     /// - `https://<account>.blob.core.windows.net`
+    /// - `https://<account>.blob.core.windows.net/<container>`
     /// - `https://<account>.dfs.fabric.microsoft.com`
     /// - `https://<account>.dfs.fabric.microsoft.com/<container>`
     /// - `https://<account>.blob.fabric.microsoft.com`
@@ -589,6 +590,9 @@ impl MicrosoftAzureBuilder {
             "https" => match host.split_once('.') {
                 Some((a, "dfs.core.windows.net")) | Some((a, "blob.core.windows.net")) => {
                     self.account_name = Some(validate(a)?);
+                    if let Some(container) = parsed.path_segments().unwrap().next() {
+                        self.container_name = Some(validate(container)?);
+                    }
                 }
                 Some((a, "dfs.fabric.microsoft.com")) | Some((a, "blob.fabric.microsoft.com")) => {
                     self.account_name = Some(validate(a)?);
@@ -984,6 +988,14 @@ mod tests {
         assert_eq!(builder.account_name, Some("account".to_string()));
         assert!(!builder.use_fabric_endpoint.get().unwrap());
 
+        let mut builder = MicrosoftAzureBuilder::new();
+        builder
+            .parse_url("https://account.blob.core.windows.net/container")
+            .unwrap();
+        assert_eq!(builder.account_name, Some("account".to_string()));
+        assert_eq!(builder.container_name, Some("container".to_string()));
+        assert!(!builder.use_fabric_endpoint.get().unwrap());
+
         let mut builder = MicrosoftAzureBuilder::new();
         builder
             .parse_url("https://account.dfs.fabric.microsoft.com/")
diff --git a/parquet/regen.sh b/parquet/regen.sh
index 915396343..f2d815876 100755
--- a/parquet/regen.sh
+++ b/parquet/regen.sh
@@ -21,15 +21,19 @@ REVISION=46cc3a0647d301bb9579ca8dd2cc356caf2a72d2
 
 SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
 
-docker run -v $SOURCE_DIR:/thrift/src -it archlinux pacman -Sy --noconfirm thrift  && \
+docker run -v $SOURCE_DIR:/thrift -it archlinux /bin/bash -c "\
+  pacman -Sy --noconfirm wget thrift && \
   wget https://raw.githubusercontent.com/apache/parquet-format/$REVISION/src/main/thrift/parquet.thrift -O /tmp/parquet.thrift && \
   thrift --gen rs /tmp/parquet.thrift && \
-  echo "Removing TProcessor" && \
+  echo 'Removing TProcessor' && \
   sed -i '/use thrift::server::TProcessor;/d' parquet.rs && \
-  echo "Replacing TSerializable" && \
+  echo 'Replacing TSerializable' && \
   sed -i 's/impl TSerializable for/impl crate::thrift::TSerializable for/g' parquet.rs && \
-  echo "Rewriting write_to_out_protocol" && \
+  echo 'Rewriting write_to_out_protocol' && \
   sed -i 's/fn write_to_out_protocol(&self, o_prot: &mut dyn TOutputProtocol)/fn write_to_out_protocol<T: TOutputProtocol>(\&self, o_prot: \&mut T)/g' parquet.rs && \
-  echo "Rewriting read_from_in_protocol" && \
+  echo 'Rewriting read_from_in_protocol' && \
   sed -i 's/fn read_from_in_protocol(i_prot: &mut dyn TInputProtocol)/fn read_from_in_protocol<T: TInputProtocol>(i_prot: \&mut T)/g' parquet.rs && \
-  mv parquet.rs src/format.rs
+  echo 'Rewriting return value expectations' && \
+  sed -i 's/Ok(ret.expect(\"return value should have been constructed\"))/ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, \"return value should have been constructed\")))/g' parquet.rs && \
+  mv parquet.rs /thrift/src/format.rs
+  "
diff --git a/parquet/src/file/serialized_reader.rs b/parquet/src/file/serialized_reader.rs
index 7e06cde37..5c47e5a83 100644
--- a/parquet/src/file/serialized_reader.rs
+++ b/parquet/src/file/serialized_reader.rs
@@ -1249,6 +1249,19 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_file_reader_invalid_metadata() {
+        let data = [
+            255, 172, 1, 0, 50, 82, 65, 73, 1, 0, 0, 0, 169, 168, 168, 162, 87, 255, 16, 0, 0, 0,
+            80, 65, 82, 49,
+        ];
+        let ret = SerializedFileReader::new(Bytes::copy_from_slice(&data));
+        assert_eq!(
+            ret.err().unwrap().to_string(),
+            "Parquet error: Could not parse metadata: bad data"
+        );
+    }
+
     #[test]
     // Use java parquet-tools get below pageIndex info
     // !```
diff --git a/parquet/src/format.rs b/parquet/src/format.rs
index 4700b05dc..4e1aa0b65 100644
--- a/parquet/src/format.rs
+++ b/parquet/src/format.rs
@@ -1369,7 +1369,7 @@ impl crate::thrift::TSerializable for TimeUnit {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {
@@ -1851,7 +1851,7 @@ impl crate::thrift::TSerializable for LogicalType {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {
@@ -2625,7 +2625,7 @@ impl crate::thrift::TSerializable for BloomFilterAlgorithm {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {
@@ -2738,7 +2738,7 @@ impl crate::thrift::TSerializable for BloomFilterHash {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {
@@ -2850,7 +2850,7 @@ impl crate::thrift::TSerializable for BloomFilterCompression {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {
@@ -3846,7 +3846,7 @@ impl crate::thrift::TSerializable for ColumnCryptoMetaData {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {
@@ -4300,7 +4300,7 @@ impl crate::thrift::TSerializable for ColumnOrder {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {
@@ -4873,7 +4873,7 @@ impl crate::thrift::TSerializable for EncryptionAlgorithm {
         )
       )
     } else {
-      Ok(ret.expect("return value should have been constructed"))
+      ret.ok_or_else(|| thrift::Error::Protocol(ProtocolError::new(ProtocolErrorKind::InvalidData, "return value should have been constructed")))
     }
   }
   fn write_to_out_protocol<T: TOutputProtocol>(&self, o_prot: &mut T) -> thrift::Result<()> {


Test patch:
diff --git a/arrow-integration-testing/Cargo.toml b/arrow-integration-testing/Cargo.toml
index c29860f09..032b99f4f 100644
--- a/arrow-integration-testing/Cargo.toml
+++ b/arrow-integration-testing/Cargo.toml
@@ -46,7 +46,7 @@ prost = { version = "0.12", default-features = false }
 serde = { version = "1.0", default-features = false, features = ["rc", "derive"] }
 serde_json = { version = "1.0", default-features = false, features = ["std"] }
 tokio = { version = "1.0", default-features = false }
-tonic = { version = "0.10", default-features = false }
+tonic = { version = "0.11", default-features = false }
 tracing-subscriber = { version = "0.3.1", default-features = false, features = ["fmt"], optional = true }
 num = { version = "0.4", default-features = false, features = ["std"] }
 flate2 = { version = "1", default-features = false, features = ["rust_backend"] }