Repository: apache/arrow-rs
Pull request #5881 (2024-06-13 15:07:22)

Update for new clippy rules
# Which issue does this PR close?

<!--
We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
-->

Closes #.

# Rationale for this change
 Apply new clippy rules for rust 1.79, should unblock CI now
<!--
Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes.
-->

# What changes are included in this PR?

<!--
There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
-->

# Are there any user-facing changes?


<!--
If there are user-facing changes then we may require documentation to be updated before approving the PR.
-->

<!---
If there are any breaking changes to public APIs, please add the `breaking change` label.
-->



Patch:
diff --git a/arrow-data/src/data.rs b/arrow-data/src/data.rs
index 5ee966394..12fe4968d 100644
--- a/arrow-data/src/data.rs
+++ b/arrow-data/src/data.rs
@@ -24,9 +24,9 @@ use arrow_buffer::{
     bit_util, i256, ArrowNativeType, Buffer, IntervalDayTime, IntervalMonthDayNano, MutableBuffer,
 };
 use arrow_schema::{ArrowError, DataType, UnionMode};
+use std::mem;
 use std::ops::Range;
 use std::sync::Arc;
-use std::{mem, usize};
 
 use crate::{equal, validate_binary_view, validate_string_view};
 
diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs
index ef08a6130..f74a86e01 100644
--- a/arrow-ipc/src/writer.rs
+++ b/arrow-ipc/src/writer.rs
@@ -1742,7 +1742,7 @@ mod tests {
         let array1 = NullArray::new(32);
         let array2 = Int32Array::from(vec![1; 32]);
         let array3 = NullArray::new(32);
-        let array4 = Float64Array::from(vec![std::f64::NAN; 32]);
+        let array4 = Float64Array::from(vec![f64::NAN; 32]);
         let batch = RecordBatch::try_new(
             Arc::new(schema.clone()),
             vec![
diff --git a/arrow-json/src/reader/mod.rs b/arrow-json/src/reader/mod.rs
index 9a113ee2b..0a50cfac6 100644
--- a/arrow-json/src/reader/mod.rs
+++ b/arrow-json/src/reader/mod.rs
@@ -2282,8 +2282,8 @@ mod tests {
     fn test_coercing_primitive_into_string_decoder() {
         let buf = &format!(
             r#"[{{"a": 1, "b": "A", "c": "T"}}, {{"a": 2, "b": "BB", "c": "F"}}, {{"a": {}, "b": 123, "c": false}}, {{"a": {}, "b": 789, "c": true}}]"#,
-            (std::i32::MAX as i64 + 10),
-            std::i64::MAX - 10
+            (i32::MAX as i64 + 10),
+            i64::MAX - 10
         );
         let schema = Schema::new(vec![
             Field::new("a", DataType::Float64, true),
@@ -2306,8 +2306,8 @@ mod tests {
                     Arc::new(Float64Array::from(vec![
                         1.0,
                         2.0,
-                        (std::i32::MAX as i64 + 10) as f64,
-                        (std::i64::MAX - 10) as f64
+                        (i32::MAX as i64 + 10) as f64,
+                        (i64::MAX - 10) as f64
                     ])),
                     Arc::new(StringArray::from(vec!["A", "BB", "123", "789"])),
                     Arc::new(StringArray::from(vec!["T", "F", "false", "true"])),
diff --git a/arrow-select/src/filter.rs b/arrow-select/src/filter.rs
index 8e06b07f5..65ccbe1e0 100644
--- a/arrow-select/src/filter.rs
+++ b/arrow-select/src/filter.rs
@@ -552,7 +552,10 @@ fn filter_native<T: ArrowNativeType>(values: &[T], predicate: &FilterPredicate)
 }
 
 /// `filter` implementation for primitive arrays
-fn filter_primitive<T>(array: &PrimitiveArray<T>, predicate: &FilterPredicate) -> PrimitiveArray<T>
+pub(crate) fn filter_primitive<T>(
+    array: &PrimitiveArray<T>,
+    predicate: &FilterPredicate,
+) -> PrimitiveArray<T>
 where
     T: ArrowPrimitiveType,
 {
diff --git a/arrow-select/src/take.rs b/arrow-select/src/take.rs
index b8d59142d..d6892eb0a 100644
--- a/arrow-select/src/take.rs
+++ b/arrow-select/src/take.rs
@@ -31,6 +31,8 @@ use arrow_schema::{ArrowError, DataType, FieldRef, UnionMode};
 
 use num::{One, Zero};
 
+use crate::filter::{filter_primitive, FilterBuilder};
+
 /// Take elements by index from [Array], creating a new [Array] from those indexes.
 ///
 /// ```text
@@ -240,6 +242,44 @@ fn take_impl<IndexType: ArrowPrimitiveType>(
             let array = UnionArray::try_new(fields.clone(), type_ids, None, children)?;
             Ok(Arc::new(array))
         }
+        DataType::Union(fields, UnionMode::Dense) => {
+            let values = values.as_any().downcast_ref::<UnionArray>().unwrap();
+
+            let type_ids = <PrimitiveArray<Int8Type>>::new(take_native(values.type_ids(), indices), None);
+            let offsets = <PrimitiveArray<Int32Type>>::new(take_native(values.offsets().unwrap(), indices), None);
+
+            let children = fields.iter()
+                .map(|(field_type_id, _)| {
+                    let mask = BooleanArray::from_unary(&type_ids, |value_type_id| value_type_id == field_type_id);
+                    let predicate = FilterBuilder::new(&mask).build();
+
+                    let indices = filter_primitive(&offsets, &predicate);
+
+                    let values = values.child(field_type_id);
+
+                    take_impl(values, &indices)
+                })
+                .collect::<Result<_, _>>()?;
+
+            let mut child_offsets = [0; 128];
+
+            let offsets = type_ids.values()
+                .iter()
+                .map(|&i| {
+                    let offset = child_offsets[i as usize];
+
+                    child_offsets[i as usize] += 1;
+
+                    offset
+                })
+                .collect();
+
+            let (_, type_ids, _) = type_ids.into_parts();
+
+            let array = UnionArray::try_new(fields.clone(), type_ids, Some(offsets), children)?;
+
+            Ok(Arc::new(array))
+        }
         t => unimplemented!("Take not supported for data type {:?}", t)
     }
 }
@@ -2146,7 +2186,7 @@ mod tests {
     }
 
     #[test]
-    fn test_take_union() {
+    fn test_take_union_sparse() {
         let structs = create_test_struct(vec![
             Some((Some(true), Some(42))),
             Some((Some(false), Some(28))),
@@ -2183,4 +2223,90 @@ mod tests {
         let expected = vec![Some("a"), None, None, Some("a"), Some("c"), Some("d")];
         assert_eq!(expected, actual);
     }
+
+    #[test]
+    fn test_take_union_dense() {
+        let type_ids = vec![0, 1, 1, 0, 0, 1, 0];
+        let offsets = vec![0, 0, 1, 1, 2, 2, 3];
+        let ints = vec![10, 20, 30, 40];
+        let strings = vec![Some("a"), None, Some("c"), Some("d")];
+
+        let indices = vec![0, 3, 1, 0, 2, 4];
+
+        let taken_type_ids = vec![0, 0, 1, 0, 1, 0];
+        let taken_offsets = vec![0, 1, 0, 2, 1, 3];
+        let taken_ints = vec![10, 20, 10, 30];
+        let taken_strings = vec![Some("a"), None];
+
+        let type_ids = <ScalarBuffer<i8>>::from(type_ids);
+        let offsets = <ScalarBuffer<i32>>::from(offsets);
+        let ints = UInt32Array::from(ints);
+        let strings = StringArray::from(strings);
+
+        let union_fields = [
+            (
+                0,
+                Arc::new(Field::new("f1", ints.data_type().clone(), true)),
+            ),
+            (
+                1,
+                Arc::new(Field::new("f2", strings.data_type().clone(), true)),
+            ),
+        ]
+        .into_iter()
+        .collect();
+
+        let array = UnionArray::try_new(
+            union_fields,
+            type_ids,
+            Some(offsets),
+            vec![Arc::new(ints), Arc::new(strings)],
+        )
+        .unwrap();
+
+        let index = UInt32Array::from(indices);
+
+        let actual = take(&array, &index, None).unwrap();
+        let actual = actual.as_any().downcast_ref::<UnionArray>().unwrap();
+
+        assert_eq!(actual.offsets(), Some(&ScalarBuffer::from(taken_offsets)));
+        assert_eq!(actual.type_ids(), &ScalarBuffer::from(taken_type_ids));
+        assert_eq!(
+            UInt32Array::from(actual.child(0).to_data()),
+            UInt32Array::from(taken_ints)
+        );
+        assert_eq!(
+            StringArray::from(actual.child(1).to_data()),
+            StringArray::from(taken_strings)
+        );
+    }
+
+    #[test]
+    fn test_take_union_dense_using_builder() {
+        let mut builder = UnionBuilder::new_dense();
+
+        builder.append::<Int32Type>("a", 1).unwrap();
+        builder.append::<Float64Type>("b", 3.0).unwrap();
+        builder.append::<Int32Type>("a", 4).unwrap();
+        builder.append::<Int32Type>("a", 5).unwrap();
+        builder.append::<Float64Type>("b", 2.0).unwrap();
+
+        let union = builder.build().unwrap();
+
+        let indices = UInt32Array::from(vec![2, 0, 1, 2]);
+
+        let mut builder = UnionBuilder::new_dense();
+
+        builder.append::<Int32Type>("a", 4).unwrap();
+        builder.append::<Int32Type>("a", 1).unwrap();
+        builder.append::<Float64Type>("b", 3.0).unwrap();
+        builder.append::<Int32Type>("a", 4).unwrap();
+
+        let taken = builder.build().unwrap();
+
+        assert_eq!(
+            taken.to_data(),
+            take(&union, &indices, None).unwrap().to_data()
+        );
+    }
 }
diff --git a/parquet/src/column/reader.rs b/parquet/src/column/reader.rs
index 53d56dd6b..b40ca2b78 100644
--- a/parquet/src/column/reader.rs
+++ b/parquet/src/column/reader.rs
@@ -672,8 +672,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
     test!(
         test_read_plain_v2_int32,
@@ -684,8 +684,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
 
     test!(
@@ -697,8 +697,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         17,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
     test!(
         test_read_plain_v2_int32_uneven,
@@ -709,8 +709,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         17,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
 
     test!(
@@ -722,8 +722,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         512,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
     test!(
         test_read_plain_v2_int32_multi_page,
@@ -734,8 +734,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         512,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
 
     // test cases when column descriptor has MAX_DEF_LEVEL = 0 and MAX_REP_LEVEL = 0
@@ -748,8 +748,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
     test!(
         test_read_plain_v2_int32_required_non_repeated,
@@ -760,8 +760,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i32::MIN,
-        std::i32::MAX
+        i32::MIN,
+        i32::MAX
     );
 
     test!(
@@ -773,8 +773,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
     test!(
         test_read_plain_v2_int64,
@@ -785,8 +785,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
 
     test!(
@@ -798,8 +798,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         17,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
     test!(
         test_read_plain_v2_int64_uneven,
@@ -810,8 +810,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         17,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
 
     test!(
@@ -823,8 +823,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         512,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
     test!(
         test_read_plain_v2_int64_multi_page,
@@ -835,8 +835,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         512,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
 
     // test cases when column descriptor has MAX_DEF_LEVEL = 0 and MAX_REP_LEVEL = 0
@@ -849,8 +849,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
     test!(
         test_read_plain_v2_int64_required_non_repeated,
@@ -861,8 +861,8 @@ mod tests {
         NUM_PAGES,
         NUM_LEVELS,
         16,
-        std::i64::MIN,
-        std::i64::MAX
+        i64::MIN,
+        i64::MAX
     );
 
     test!(
@@ -1037,8 +1037,8 @@ mod tests {
             num_pages,
             num_levels,
             batch_size,
-            std::i32::MIN,
-            std::i32::MAX,
+            i32::MIN,
+            i32::MAX,
             false,
         );
     }
diff --git a/parquet/src/encodings/decoding.rs b/parquet/src/encodings/decoding.rs
index a1924f72e..b3dd13523 100644
--- a/parquet/src/encodings/decoding.rs
+++ b/parquet/src/encodings/decoding.rs
@@ -1624,14 +1624,14 @@ mod tests {
     #[test]
     fn test_delta_bit_packed_int64_min_max() {
         let block_data = vec![
-            i64::min_value(),
-            i64::max_value(),
-            i64::min_value(),
-            i64::max_value(),
-            i64::min_value(),
-            i64::max_value(),
-            i64::min_value(),
-            i64::max_value(),
+            i64::MIN,
+            i64::MAX,
+            i64::MIN,
+            i64::MAX,
+            i64::MIN,
+            i64::MAX,
+            i64::MIN,
+            i64::MAX,
         ];
         test_delta_bit_packed_decode::<Int64Type>(vec![block_data]);
     }


Test patch:
diff --git a/parquet_derive_test/src/lib.rs b/parquet_derive_test/src/lib.rs
index 3743c6b55..b8e6c27f5 100644
--- a/parquet_derive_test/src/lib.rs
+++ b/parquet_derive_test/src/lib.rs
@@ -141,9 +141,9 @@ mod tests {
             maybe_usize: Some(4456),
             isize: -365,
             float: 3.5,
-            double: std::f64::NAN,
+            double: f64::NAN,
             maybe_float: None,
-            maybe_double: Some(std::f64::MAX),
+            maybe_double: Some(f64::MAX),
             borrowed_maybe_a_string: &maybe_a_string,
             borrowed_maybe_a_str: &maybe_a_str,
             now: chrono::Utc::now().naive_local(),
@@ -180,7 +180,7 @@ mod tests {
             u64: 4563424,
             isize: -365,
             float: 3.5,
-            double: std::f64::NAN,
+            double: f64::NAN,
             now: chrono::Utc::now().naive_local(),
             date: chrono::naive::NaiveDate::from_ymd_opt(2015, 3, 14).unwrap(),
             uuid: uuid::Uuid::new_v4(),