Repository: jorgecarleitao/arrow2
Pull request #1063 (2022-06-10 05:09:53)

Added support to write parquet `_metadata` sidecar
This PR exposes parquet2's API to write partitioned parquet files' metadata on a sidecar, `_common_metadata` and `_metadata`. See https://github.com/jorgecarleitao/parquet2/issues/146 for context within `parquet2` and [pyarrow documentation](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_to_dataset.html). Thanks to @kylebarron for the idea and thorough explanation!


codecov[bot]: # [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) Report
> Merging [#1063](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (0da8601) into [main](https://codecov.io/gh/jorgecarleitao/arrow2/commit/b0c601e52db43633688fcf801e60564c8edda7d5?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (b0c601e) will **decrease** coverage by `0.01%`.
> The diff coverage is `0.00%`.

```diff
@@            Coverage Diff             @@
##             main    #1063      +/-   ##
==========================================
- Coverage   81.32%   81.30%   -0.02%     
==========================================
  Files         363      363              
  Lines       34685    34692       +7     
==========================================
  Hits        28206    28206              
- Misses       6479     6486       +7     
```


| [Impacted Files](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) | Coverage Δ | |
|---|---|---|
| [src/io/parquet/write/file.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL3BhcnF1ZXQvd3JpdGUvZmlsZS5ycw==) | `76.47% <0.00%> (-4.78%)` | :arrow_down: |
| [src/io/parquet/write/mod.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL3BhcnF1ZXQvd3JpdGUvbW9kLnJz) | `62.88% <ø> (ø)` | |
| [src/io/parquet/read/deserialize/null.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL3BhcnF1ZXQvcmVhZC9kZXNlcmlhbGl6ZS9udWxsLnJz) | `0.00% <0.00%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Last update [b0c601e...0da8601](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1063?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).


Patch:
diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md
index a39fd47da8..85ffbce583 100644
--- a/guide/src/SUMMARY.md
+++ b/guide/src/SUMMARY.md
@@ -20,3 +20,4 @@
     - [Write Avro](./io/avro_write.md)
     - [Read JSON](./io/json_read.md)
     - [Write JSON](./io/json_write.md)
+    - [Read and Write ODBC](./io/odbc.md)
diff --git a/src/compute/cast/mod.rs b/src/compute/cast/mod.rs
index f236473e88..2cae7a2192 100644
--- a/src/compute/cast/mod.rs
+++ b/src/compute/cast/mod.rs
@@ -79,16 +79,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool {
     }
 
     match (from_type, to_type) {
-        (
-            Null,
-            Boolean | Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 | Float32 | Date32 | Time32(_)
-            | Int64 | UInt64 | Float64 | Date64 | List(_) | Dictionary(..),
-        )
-        | (
-            Boolean | Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 | Float32 | Date32 | Time32(_)
-            | Int64 | UInt64 | Float64 | Date64 | List(_) | Dictionary(..),
-            Null,
-        ) => true,
+        (Null, _) | (_, Null) => true,
         (Struct(_), _) => false,
         (_, Struct(_)) => false,
         (List(list_from), List(list_to)) => {
@@ -379,16 +370,7 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu
 
     let as_options = options.with_wrapped(true);
     match (from_type, to_type) {
-        (
-            Null,
-            Boolean | Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 | Float32 | Date32 | Time32(_)
-            | Int64 | UInt64 | Float64 | Date64 | List(_) | Dictionary(..),
-        )
-        | (
-            Boolean | Int8 | UInt8 | Int16 | UInt16 | Int32 | UInt32 | Float32 | Date32 | Time32(_)
-            | Int64 | UInt64 | Float64 | Date64 | List(_) | Dictionary(..),
-            Null,
-        ) => Ok(new_null_array(to_type.clone(), array.len())),
+        (Null, _) | (_, Null) => Ok(new_null_array(to_type.clone(), array.len())),
         (Struct(_), _) => Err(Error::NotYetImplemented(
             "Cannot cast from struct to other types".to_string(),
         )),
diff --git a/src/io/parquet/write/file.rs b/src/io/parquet/write/file.rs
index 42153a93a2..e93aa9fa67 100644
--- a/src/io/parquet/write/file.rs
+++ b/src/io/parquet/write/file.rs
@@ -8,7 +8,7 @@ use parquet2::write::WriteOptions as FileWriteOptions;
 use crate::datatypes::Schema;
 use crate::error::{Error, Result};
 
-use super::{schema::schema_to_metadata_key, to_parquet_schema, WriteOptions};
+use super::{schema::schema_to_metadata_key, to_parquet_schema, ThriftFileMetaData, WriteOptions};
 
 /// Attaches [`Schema`] to `key_value_metadata`
 pub fn add_arrow_schema(
@@ -87,4 +87,11 @@ impl<W: Write> FileWriter<W> {
     pub fn into_inner(self) -> W {
         self.writer.into_inner()
     }
+
+    /// Returns the underlying writer and [`ThriftFileMetaData`]
+    /// # Panics
+    /// This function panics if [`Self::end`] has not yet been called
+    pub fn into_inner_and_metadata(self) -> (W, ThriftFileMetaData) {
+        self.writer.into_inner_and_metadata()
+    }
 }
diff --git a/src/io/parquet/write/mod.rs b/src/io/parquet/write/mod.rs
index 72a90ebb7c..5dfd8ae28b 100644
--- a/src/io/parquet/write/mod.rs
+++ b/src/io/parquet/write/mod.rs
@@ -24,10 +24,13 @@ pub use parquet2::{
     compression::{BrotliLevel, CompressionLevel, CompressionOptions, GzipLevel, ZstdLevel},
     encoding::Encoding,
     fallible_streaming_iterator,
-    metadata::{Descriptor, KeyValue, SchemaDescriptor},
+    metadata::{Descriptor, FileMetaData, KeyValue, SchemaDescriptor, ThriftFileMetaData},
     page::{CompressedDataPage, CompressedPage, EncodedPage},
     schema::types::{FieldInfo, ParquetType, PhysicalType as ParquetPhysicalType},
-    write::{compress, Compressor, DynIter, DynStreamingIterator, RowGroupIter, Version},
+    write::{
+        compress, write_metadata_sidecar, Compressor, DynIter, DynStreamingIterator, RowGroupIter,
+        Version,
+    },
     FallibleStreamingIterator,
 };
 


Test patch:
diff --git a/tests/it/compute/comparison.rs b/tests/it/compute/comparison.rs
index 35d86be578..1e887d7806 100644
--- a/tests/it/compute/comparison.rs
+++ b/tests/it/compute/comparison.rs
@@ -1,7 +1,7 @@
 use arrow2::array::*;
-use arrow2::compute::comparison::{self, boolean::*};
-use arrow2::datatypes::{DataType::*, IntervalUnit};
-use arrow2::datatypes::{IntegerType, TimeUnit};
+use arrow2::bitmap::Bitmap;
+use arrow2::compute::comparison::{self, boolean::*, primitive};
+use arrow2::datatypes::{DataType::*, IntegerType, IntervalUnit, TimeUnit};
 use arrow2::scalar::new_scalar;
 
 #[test]
@@ -68,263 +68,315 @@ fn consistency() {
     });
 }
 
-// disable wrapping inside literal vectors used for test data and assertions
-#[rustfmt::skip::macros(vec)]
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use arrow2::bitmap::Bitmap;
-
-    macro_rules! cmp_bool {
-        ($KERNEL:ident, $A_VEC:expr, $B_VEC:expr, $EXPECTED:expr) => {
-            let a = BooleanArray::from_slice($A_VEC);
-            let b = BooleanArray::from_slice($B_VEC);
-            let c = $KERNEL(&a, &b);
-            assert_eq!(BooleanArray::from_slice($EXPECTED), c);
-        };
-    }
-
-    macro_rules! cmp_bool_options {
-        ($KERNEL:ident, $A_VEC:expr, $B_VEC:expr, $EXPECTED:expr) => {
-            let a = BooleanArray::from($A_VEC);
-            let b = BooleanArray::from($B_VEC);
-            let c = $KERNEL(&a, &b);
-            assert_eq!(BooleanArray::from($EXPECTED), c);
-        };
-    }
-
-    macro_rules! cmp_bool_scalar {
-        ($KERNEL:ident, $A_VEC:expr, $B:literal, $EXPECTED:expr) => {
-            let a = BooleanArray::from_slice($A_VEC);
-            let c = $KERNEL(&a, $B);
-            assert_eq!(BooleanArray::from_slice($EXPECTED), c);
-        };
-    }
-
-    #[test]
-    fn test_eq() {
-        cmp_bool!(
-            eq,
-            &[true, false, true, false],
-            &[true, true, false, false],
-            &[true, false, false, true]
-        );
-    }
-
-    #[test]
-    fn test_eq_scalar() {
-        cmp_bool_scalar!(eq_scalar, &[false, true], true, &[false, true]);
-    }
-
-    #[test]
-    fn test_eq_with_slice() {
-        let a = BooleanArray::from_slice(&[true, true, false]);
-        let b = BooleanArray::from_slice(&[true, true, true, true, false]);
-        let c = b.slice(2, 3);
-        let d = eq(&c, &a);
-        assert_eq!(d, BooleanArray::from_slice(&[true, true, true]));
-    }
-
-    #[test]
-    fn test_neq() {
-        cmp_bool!(
-            neq,
-            &[true, false, true, false],
-            &[true, true, false, false],
-            &[false, true, true, false]
-        );
-    }
-
-    #[test]
-    fn test_neq_scalar() {
-        cmp_bool_scalar!(neq_scalar, &[false, true], true, &[true, false]);
-    }
-
-    #[test]
-    fn test_lt() {
-        cmp_bool!(
-            lt,
-            &[true, false, true, false],
-            &[true, true, false, false],
-            &[false, true, false, false]
-        );
-    }
-
-    #[test]
-    fn test_lt_scalar_true() {
-        cmp_bool_scalar!(lt_scalar, &[false, true], true, &[true, false]);
-    }
-
-    #[test]
-    fn test_lt_scalar_false() {
-        cmp_bool_scalar!(lt_scalar, &[false, true], false, &[false, false]);
-    }
-
-    #[test]
-    fn test_lt_eq_scalar_true() {
-        cmp_bool_scalar!(lt_eq_scalar, &[false, true], true, &[true, true]);
-    }
-
-    #[test]
-    fn test_lt_eq_scalar_false() {
-        cmp_bool_scalar!(lt_eq_scalar, &[false, true], false, &[true, false]);
-    }
-
-    #[test]
-    fn test_gt_scalar_true() {
-        cmp_bool_scalar!(gt_scalar, &[false, true], true, &[false, false]);
-    }
-
-    #[test]
-    fn test_gt_scalar_false() {
-        cmp_bool_scalar!(gt_scalar, &[false, true], false, &[false, true]);
-    }
-
-    #[test]
-    fn test_gt_eq_scalar_true() {
-        cmp_bool_scalar!(gt_eq_scalar, &[false, true], true, &[false, true]);
-    }
-
-    #[test]
-    fn test_gt_eq_scalar_false() {
-        cmp_bool_scalar!(gt_eq_scalar, &[false, true], false, &[true, true]);
-    }
-
-    #[test]
-    fn test_lt_eq_scalar_true_1() {
-        cmp_bool_scalar!(
-            lt_eq_scalar,
-            &[false, true, true, true, true, true, true, true, false],
-            true,
-            &[true, true, true, true, true, true, true, true, true]
-        );
-    }
-
-    #[test]
-    fn eq_nulls() {
-        cmp_bool_options!(
-            eq,
-            &[
-                None,
-                None,
-                None,
-                Some(false),
-                Some(false),
-                Some(false),
-                Some(true),
-                Some(true),
-                Some(true)
-            ],
-            &[
-                None,
-                Some(false),
-                Some(true),
-                None,
-                Some(false),
-                Some(true),
-                None,
-                Some(false),
-                Some(true)
-            ],
-            &[
-                None,
-                None,
-                None,
-                None,
-                Some(true),
-                Some(false),
-                None,
-                Some(false),
-                Some(true)
-            ]
-        );
-    }
-
-    fn check_mask(mask: &BooleanArray, expected: &[bool]) {
-        assert!(mask.validity().is_none());
-        let mask = mask.values_iter().collect::<Vec<_>>();
-        assert_eq!(mask, expected);
-    }
-
-    #[test]
-    fn compare_no_propagating_nulls_eq() {
-        // single validity
-        let a = Utf8Array::<i32>::from_iter([Some("a"), None, Some("c")]);
-        let b = Utf8Array::<i32>::from_iter([Some("a"), Some("c"), Some("c")]);
-
-        let out = comparison::utf8::eq_and_validity(&a, &b);
-        check_mask(&out, &[true, false, true]);
-        let out = comparison::utf8::eq_and_validity(&b, &a);
-        check_mask(&out, &[true, false, true]);
-
-        // both have validities
-        let b = Utf8Array::<i32>::from_iter([Some("a"), None, None]);
-        let out = comparison::utf8::eq_and_validity(&a, &b);
-        check_mask(&out, &[true, true, false]);
-
-        // scalar
-        let out = comparison::utf8::eq_scalar_and_validity(&a, "a");
-        check_mask(&out, &[true, false, false]);
-
-        // now we add a mask while we know that underlying values are equal
-        let a = Utf8Array::<i32>::from_iter([Some("a"), Some("b"), Some("c")]);
-
-        // now mask with a null
-        let mask = Bitmap::from_iter([false, true, true]);
-        let a_masked = a.with_validity(Some(mask));
-        let out = comparison::utf8::eq_and_validity(&a, &a_masked);
-        check_mask(&out, &[false, true, true]);
-
-        // other types
-        let a = Int32Array::from_iter([Some(1), Some(2), Some(3)]);
-        let b = Int32Array::from_iter([Some(1), Some(2), None]);
-        let out = comparison::primitive::eq_and_validity(&a, &b);
-        check_mask(&out, &[true, true, false]);
-
-        let a = BooleanArray::from_iter([Some(true), Some(false), Some(false)]);
-        let b = BooleanArray::from_iter([Some(true), Some(true), None]);
-        let out = comparison::boolean::eq_and_validity(&a, &b);
-        check_mask(&out, &[true, false, false]);
-    }
-
-    #[test]
-    fn compare_no_propagating_nulls_neq() {
-        // single validity
-        let a = Utf8Array::<i32>::from_iter([Some("a"), None, Some("c")]);
-        let b = Utf8Array::<i32>::from_iter([Some("foo"), Some("c"), Some("c")]);
-
-        let out = comparison::utf8::neq_and_validity(&a, &b);
-        check_mask(&out, &[true, true, false]);
-        let out = comparison::utf8::neq_and_validity(&b, &a);
-        check_mask(&out, &[true, true, false]);
-
-        // both have validities
-        let b = Utf8Array::<i32>::from_iter([Some("a"), None, None]);
-        let out = comparison::utf8::neq_and_validity(&a, &b);
-        check_mask(&out, &[false, false, true]);
-
-        // scalar
-        let out = comparison::utf8::neq_scalar_and_validity(&a, "a");
-        check_mask(&out, &[false, true, true]);
-
-        // now we add a mask while we know that underlying values are equal
-        let a = Utf8Array::<i32>::from_iter([Some("a"), Some("b"), Some("c")]);
-
-        // now mask with a null
-        let mask = Bitmap::from_iter([false, true, true]);
-        let a_masked = a.with_validity(Some(mask));
-        let out = comparison::utf8::neq_and_validity(&a, &a_masked);
-        check_mask(&out, &[true, false, false]);
-
-        // other types
-        let a = Int32Array::from_iter([Some(1), Some(2), Some(3)]);
-        let b = Int32Array::from_iter([Some(1), Some(2), None]);
-        let out = comparison::primitive::neq_and_validity(&a, &b);
-        check_mask(&out, &[false, false, true]);
-
-        let a = BooleanArray::from_iter([Some(true), Some(false), Some(false)]);
-        let b = BooleanArray::from_iter([Some(true), Some(true), None]);
-        let out = comparison::boolean::neq_and_validity(&a, &b);
-        check_mask(&out, &[false, true, true]);
-    }
+macro_rules! cmp_bool {
+    ($KERNEL:ident, $A_VEC:expr, $B_VEC:expr, $EXPECTED:expr) => {
+        let a = BooleanArray::from_slice($A_VEC);
+        let b = BooleanArray::from_slice($B_VEC);
+        let c = $KERNEL(&a, &b);
+        assert_eq!(BooleanArray::from_slice($EXPECTED), c);
+    };
+}
+
+macro_rules! cmp_bool_options {
+    ($KERNEL:ident, $A_VEC:expr, $B_VEC:expr, $EXPECTED:expr) => {
+        let a = BooleanArray::from($A_VEC);
+        let b = BooleanArray::from($B_VEC);
+        let c = $KERNEL(&a, &b);
+        assert_eq!(BooleanArray::from($EXPECTED), c);
+    };
+}
+
+macro_rules! cmp_bool_scalar {
+    ($KERNEL:ident, $A_VEC:expr, $B:literal, $EXPECTED:expr) => {
+        let a = BooleanArray::from_slice($A_VEC);
+        let c = $KERNEL(&a, $B);
+        assert_eq!(BooleanArray::from_slice($EXPECTED), c);
+    };
+}
+
+#[test]
+fn test_eq() {
+    cmp_bool!(
+        eq,
+        &[true, false, true, false],
+        &[true, true, false, false],
+        &[true, false, false, true]
+    );
+}
+
+#[test]
+fn test_eq_scalar() {
+    cmp_bool_scalar!(eq_scalar, &[false, true], true, &[false, true]);
+}
+
+#[test]
+fn test_eq_with_slice() {
+    let a = BooleanArray::from_slice(&[true, true, false]);
+    let b = BooleanArray::from_slice(&[true, true, true, true, false]);
+    let c = b.slice(2, 3);
+    let d = eq(&c, &a);
+    assert_eq!(d, BooleanArray::from_slice(&[true, true, true]));
+}
+
+#[test]
+fn test_neq() {
+    cmp_bool!(
+        neq,
+        &[true, false, true, false],
+        &[true, true, false, false],
+        &[false, true, true, false]
+    );
+}
+
+#[test]
+fn test_neq_scalar() {
+    cmp_bool_scalar!(neq_scalar, &[false, true], true, &[true, false]);
+}
+
+#[test]
+fn test_lt() {
+    cmp_bool!(
+        lt,
+        &[true, false, true, false],
+        &[true, true, false, false],
+        &[false, true, false, false]
+    );
+}
+
+#[test]
+fn test_lt_scalar_true() {
+    cmp_bool_scalar!(lt_scalar, &[false, true], true, &[true, false]);
+}
+
+#[test]
+fn test_lt_scalar_false() {
+    cmp_bool_scalar!(lt_scalar, &[false, true], false, &[false, false]);
+}
+
+#[test]
+fn test_lt_eq_scalar_true() {
+    cmp_bool_scalar!(lt_eq_scalar, &[false, true], true, &[true, true]);
+}
+
+#[test]
+fn test_lt_eq_scalar_false() {
+    cmp_bool_scalar!(lt_eq_scalar, &[false, true], false, &[true, false]);
+}
+
+#[test]
+fn test_gt_scalar_true() {
+    cmp_bool_scalar!(gt_scalar, &[false, true], true, &[false, false]);
+}
+
+#[test]
+fn test_gt_scalar_false() {
+    cmp_bool_scalar!(gt_scalar, &[false, true], false, &[false, true]);
+}
+
+#[test]
+fn test_gt_eq_scalar_true() {
+    cmp_bool_scalar!(gt_eq_scalar, &[false, true], true, &[false, true]);
+}
+
+#[test]
+fn test_gt_eq_scalar_false() {
+    cmp_bool_scalar!(gt_eq_scalar, &[false, true], false, &[true, true]);
+}
+
+#[test]
+fn test_lt_eq_scalar_true_1() {
+    cmp_bool_scalar!(
+        lt_eq_scalar,
+        &[false, true, true, true, true, true, true, true, false],
+        true,
+        &[true, true, true, true, true, true, true, true, true]
+    );
+}
+
+#[test]
+fn eq_nulls() {
+    cmp_bool_options!(
+        eq,
+        &[
+            None,
+            None,
+            None,
+            Some(false),
+            Some(false),
+            Some(false),
+            Some(true),
+            Some(true),
+            Some(true)
+        ],
+        &[
+            None,
+            Some(false),
+            Some(true),
+            None,
+            Some(false),
+            Some(true),
+            None,
+            Some(false),
+            Some(true)
+        ],
+        &[
+            None,
+            None,
+            None,
+            None,
+            Some(true),
+            Some(false),
+            None,
+            Some(false),
+            Some(true)
+        ]
+    );
+}
+
+fn check_mask(mask: &BooleanArray, expected: &[bool]) {
+    assert!(mask.validity().is_none());
+    let mask = mask.values_iter().collect::<Vec<_>>();
+    assert_eq!(mask, expected);
+}
+
+#[test]
+fn compare_no_propagating_nulls_eq() {
+    // single validity
+    let a = Utf8Array::<i32>::from_iter([Some("a"), None, Some("c")]);
+    let b = Utf8Array::<i32>::from_iter([Some("a"), Some("c"), Some("c")]);
+
+    let out = comparison::utf8::eq_and_validity(&a, &b);
+    check_mask(&out, &[true, false, true]);
+    let out = comparison::utf8::eq_and_validity(&b, &a);
+    check_mask(&out, &[true, false, true]);
+
+    // both have validities
+    let b = Utf8Array::<i32>::from_iter([Some("a"), None, None]);
+    let out = comparison::utf8::eq_and_validity(&a, &b);
+    check_mask(&out, &[true, true, false]);
+
+    // scalar
+    let out = comparison::utf8::eq_scalar_and_validity(&a, "a");
+    check_mask(&out, &[true, false, false]);
+
+    // now we add a mask while we know that underlying values are equal
+    let a = Utf8Array::<i32>::from_iter([Some("a"), Some("b"), Some("c")]);
+
+    // now mask with a null
+    let mask = Bitmap::from_iter([false, true, true]);
+    let a_masked = a.with_validity(Some(mask));
+    let out = comparison::utf8::eq_and_validity(&a, &a_masked);
+    check_mask(&out, &[false, true, true]);
+
+    // other types
+    let a = Int32Array::from_iter([Some(1), Some(2), Some(3)]);
+    let b = Int32Array::from_iter([Some(1), Some(2), None]);
+    let out = comparison::primitive::eq_and_validity(&a, &b);
+    check_mask(&out, &[true, true, false]);
+
+    let a = BooleanArray::from_iter([Some(true), Some(false), Some(false)]);
+    let b = BooleanArray::from_iter([Some(true), Some(true), None]);
+    let out = comparison::boolean::eq_and_validity(&a, &b);
+    check_mask(&out, &[true, false, false]);
+}
+
+#[test]
+fn compare_no_propagating_nulls_neq() {
+    // single validity
+    let a = Utf8Array::<i32>::from_iter([Some("a"), None, Some("c")]);
+    let b = Utf8Array::<i32>::from_iter([Some("foo"), Some("c"), Some("c")]);
+
+    let out = comparison::utf8::neq_and_validity(&a, &b);
+    check_mask(&out, &[true, true, false]);
+    let out = comparison::utf8::neq_and_validity(&b, &a);
+    check_mask(&out, &[true, true, false]);
+
+    // both have validities
+    let b = Utf8Array::<i32>::from_iter([Some("a"), None, None]);
+    let out = comparison::utf8::neq_and_validity(&a, &b);
+    check_mask(&out, &[false, false, true]);
+
+    // scalar
+    let out = comparison::utf8::neq_scalar_and_validity(&a, "a");
+    check_mask(&out, &[false, true, true]);
+
+    // now we add a mask while we know that underlying values are equal
+    let a = Utf8Array::<i32>::from_iter([Some("a"), Some("b"), Some("c")]);
+
+    // now mask with a null
+    let mask = Bitmap::from_iter([false, true, true]);
+    let a_masked = a.with_validity(Some(mask));
+    let out = comparison::utf8::neq_and_validity(&a, &a_masked);
+    check_mask(&out, &[true, false, false]);
+
+    // other types
+    let a = Int32Array::from_iter([Some(1), Some(2), Some(3)]);
+    let b = Int32Array::from_iter([Some(1), Some(2), None]);
+    let out = comparison::primitive::neq_and_validity(&a, &b);
+    check_mask(&out, &[false, false, true]);
+
+    let a = BooleanArray::from_iter([Some(true), Some(false), Some(false)]);
+    let b = BooleanArray::from_iter([Some(true), Some(true), None]);
+    let out = comparison::boolean::neq_and_validity(&a, &b);
+    check_mask(&out, &[false, true, true]);
+}
+
+#[test]
+fn primitive_eq() {
+    let a = Int32Array::from([Some(0), Some(1), Some(3), Some(2), None]);
+    let b = Int32Array::from([Some(0), Some(3), Some(1), None, Some(3)]);
+
+    let a = primitive::eq(&a, &b);
+    assert_eq!(
+        a,
+        BooleanArray::from([Some(true), Some(false), Some(false), None, None])
+    )
+}
+
+#[test]
+fn primitive_lt() {
+    let a = Int32Array::from([Some(0), Some(1), Some(3), Some(2), None]);
+    let b = Int32Array::from([Some(0), Some(3), Some(1), None, Some(3)]);
+
+    let a = primitive::lt(&a, &b);
+    assert_eq!(
+        a,
+        BooleanArray::from([Some(false), Some(true), Some(false), None, None])
+    )
+}
+
+#[test]
+fn primitive_lt_eq() {
+    let a = Int32Array::from([Some(0), Some(1), Some(3), Some(2), None]);
+    let b = Int32Array::from([Some(0), Some(3), Some(1), None, Some(3)]);
+
+    let a = primitive::lt_eq(&a, &b);
+    assert_eq!(
+        a,
+        BooleanArray::from([Some(true), Some(true), Some(false), None, None])
+    )
+}
+
+#[test]
+fn primitive_gt() {
+    let a = Int32Array::from([Some(0), Some(1), Some(3), Some(2), None]);
+    let b = Int32Array::from([Some(0), Some(3), Some(1), None, Some(3)]);
+
+    let a = primitive::gt(&a, &b);
+    assert_eq!(
+        a,
+        BooleanArray::from([Some(false), Some(false), Some(true), None, None])
+    )
+}
+
+#[test]
+fn primitive_gt_eq() {
+    let a = Int32Array::from([Some(0), Some(1), Some(3), Some(2), None]);
+    let b = Int32Array::from([Some(0), Some(3), Some(1), None, Some(3)]);
+
+    let a = primitive::gt_eq(&a, &b);
+    assert_eq!(
+        a,
+        BooleanArray::from([Some(true), Some(false), Some(true), None, None])
+    )
 }