Repository: jorgecarleitao/arrow2
Pull request #239 (2021-07-30 07:48:46)

Added support to parquet IO on wasm32
This PR adds a new cargo feature, `io_parquet_compression`, that activates the compressions in parquet.

The corollary is that this allows compiling this crate against wasm32 with parquet io. AFAIK this effectively makes arrow2 to be usable to interact with parquet directly on a browser.

As an example, this seems to allow building a client-side web application that reads or writes (uncompressed) parquet files.

For compression support, [some work is needed on the `zstd`](https://github.com/gyscos/zstd-rs/issues/48).

jorgecarleitao: cc @domoritz since if I remember, you were interested in the wasm aspect.
codecov[bot]: # [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) Report
> Merging [#239](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (cec0989) into [main](https://codecov.io/gh/jorgecarleitao/arrow2/commit/3cd8cffc73a41d5741a5965a67c2370f56878e61?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (3cd8cff) will **increase** coverage by `0.01%`.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239/graphs/tree.svg?width=650&height=150&src=pr&token=AgyTF60R3D&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao)](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao)

```diff
@@            Coverage Diff             @@
##             main     #239      +/-   ##
==========================================
+ Coverage   76.82%   76.83%   +0.01%     
==========================================
  Files         229      229              
  Lines       19586    19613      +27     
==========================================
+ Hits        15046    15069      +23     
- Misses       4540     4544       +4     
```


| [Impacted Files](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) | Coverage Δ | |
|---|---|---|
| [src/bitmap/immutable.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2JpdG1hcC9pbW11dGFibGUucnM=) | `85.56% <0.00%> (-1.28%)` | :arrow_down: |
| [src/bitmap/utils/slice\_iterator.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2JpdG1hcC91dGlscy9zbGljZV9pdGVyYXRvci5ycw==) | `96.42% <0.00%> (+0.15%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Last update [3cd8cff...cec0989](https://codecov.io/gh/jorgecarleitao/arrow2/pull/239?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).

domoritz: Yes, this change is awesome.

I have to rewrite my arrow-wasm repo against arrow2 but ran into some issue where some types were not exported. I'll try again sometime and consolidate the things I find in an issue. 

Patch:
diff --git a/Cargo.toml b/Cargo.toml
index 55eeab15e4..74ac46afd3 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -58,7 +58,7 @@ futures = { version = "0.3", optional = true }
 # for faster hashing
 ahash = { version = "0.7", optional = true }
 
-parquet2 = { version = "0.1", optional = true }
+parquet2 = { version = "0.1", optional = true, default_features = false, features = ["stream"] }
 
 [dev-dependencies]
 rand = "0.8"
@@ -68,12 +68,33 @@ doc-comment = "0.3"
 crossbeam-channel = "0.5.1"
 
 [features]
-default = ["io_csv", "io_json", "io_ipc", "io_ipc_compression", "io_json_integration", "io_print", "io_parquet", "regex", "merge_sort", "ahash", "benchmarks", "compute"]
+default = [
+    "io_csv",
+    "io_json",
+    "io_ipc",
+    "io_ipc_compression",
+    "io_json_integration",
+    "io_print",
+    "io_parquet",
+    "io_parquet_compression",
+    "regex",
+    "merge_sort",
+    "ahash",
+    "benchmarks",
+    "compute",
+]
 merge_sort = ["itertools"]
 io_csv = ["csv", "lazy_static", "regex"]
 io_json = ["serde", "serde_derive", "serde_json", "indexmap"]
 io_ipc = ["flatbuffers"]
 io_ipc_compression = ["lz4", "zstd"]
+io_parquet_compression = [
+    "parquet2/zstd",
+    "parquet2/snappy",
+    "parquet2/gzip",
+    "parquet2/lz4",
+    "parquet2/brotli",
+]
 io_json_integration = ["io_json", "hex"]
 io_print = ["prettytable-rs"]
 # the compute kernels. Disabling this significantly reduces compile time.
@@ -88,6 +109,8 @@ skip_feature_sets = [
     ["benchmarks"],
     ["merge_sort"],
     ["io_json_integration"],
+    # this does not change the public API
+    ["io_parquet_compression"],
     ["simd"],
 ]
 
diff --git a/examples/arithmetics.rs b/examples/arithmetics.rs
new file mode 100644
index 0000000000..3123b95152
--- /dev/null
+++ b/examples/arithmetics.rs
@@ -0,0 +1,84 @@
+use arrow2::array::{Array, PrimitiveArray};
+use arrow2::compute::arithmetics::*;
+use arrow2::compute::arity::{binary, unary};
+use arrow2::datatypes::DataType;
+use arrow2::error::Result;
+
+fn main() -> Result<()> {
+    // say we have two arrays
+    let array0 = PrimitiveArray::<i64>::from(&[Some(1), Some(2), Some(3)]);
+    let array1 = PrimitiveArray::<i64>::from(&[Some(4), None, Some(6)]);
+
+    // we can add them as follows:
+    let added = arithmetic_primitive(&array0, Operator::Add, &array1)?;
+    assert_eq!(
+        added,
+        PrimitiveArray::<i64>::from(&[Some(5), None, Some(9)])
+    );
+
+    // subtract:
+    let subtracted = arithmetic_primitive(&array0, Operator::Subtract, &array1)?;
+    assert_eq!(
+        subtracted,
+        PrimitiveArray::<i64>::from(&[Some(-3), None, Some(-3)])
+    );
+
+    // add a scalar:
+    let plus10 = arithmetic_primitive_scalar(&array0, Operator::Add, &10)?;
+    assert_eq!(
+        plus10,
+        PrimitiveArray::<i64>::from(&[Some(11), Some(12), Some(13)])
+    );
+
+    // when the array is a trait object, there is a similar API
+    let array0 = &array0 as &dyn Array;
+    let array1 = &array1 as &dyn Array;
+
+    // check whether the logical types support addition (they could be any `Array`).
+    assert!(can_arithmetic(
+        array0.data_type(),
+        Operator::Add,
+        array1.data_type()
+    ));
+
+    // add them
+    let added = arithmetic(array0, Operator::Add, array1).unwrap();
+    assert_eq!(
+        PrimitiveArray::<i64>::from(&[Some(5), None, Some(9)]),
+        added.as_ref(),
+    );
+
+    // a more exotic implementation: arbitrary binary operations
+    // this is compiled to SIMD when intrinsics exist.
+    let array0 = PrimitiveArray::<i64>::from(&[Some(1), Some(2), Some(3)]);
+    let array1 = PrimitiveArray::<i64>::from(&[Some(4), None, Some(6)]);
+
+    let op = |x: i64, y: i64| x.pow(2) + y.pow(2);
+    let r = binary(&array0, &array1, DataType::Int64, op)?;
+    assert_eq!(
+        r,
+        PrimitiveArray::<i64>::from(&[Some(1 + 16), None, Some(9 + 36)])
+    );
+
+    // arbitrary unary operations
+    // this is compiled to SIMD when intrinsics exist.
+    let array0 = PrimitiveArray::<f64>::from(&[Some(4.0), None, Some(6.0)]);
+    let r = unary(
+        &array0,
+        |x| x.cos().powi(2) + x.sin().powi(2),
+        DataType::Float64,
+    );
+    assert!((r.values()[0] - 1.0).abs() < 0.0001);
+    assert!(r.is_null(1));
+    assert!((r.values()[2] - 1.0).abs() < 0.0001);
+
+    // finally, a transformation that changes types:
+    let array0 = PrimitiveArray::<f64>::from(&[Some(4.4), None, Some(4.6)]);
+    let rounded = unary(&array0, |x| x.round() as i64, DataType::Int64);
+    assert_eq!(
+        rounded,
+        PrimitiveArray::<i64>::from(&[Some(4), None, Some(5)])
+    );
+
+    Ok(())
+}
diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md
index a73ef112ec..cdf5aec37e 100644
--- a/guide/src/SUMMARY.md
+++ b/guide/src/SUMMARY.md
@@ -4,6 +4,7 @@
 - [The arrow format](./arrow.md)
 - [Low-level API](./low_level.md)
 - [High-level API](./high_level.md)
+- [Compute](./compute.md)
 - [Metadata](./metadata.md)
 - [Foreign interfaces](./ffi.md)
 - [IO](./io/README.md)
diff --git a/guide/src/compute.md b/guide/src/compute.md
new file mode 100644
index 0000000000..68948149de
--- /dev/null
+++ b/guide/src/compute.md
@@ -0,0 +1,23 @@
+# Compute API
+
+When compiled with the feature `compute`, this crate offers a wide range of functions to perform both vertical (e.g. add two arrays) and horizontal (compute the sum of an array) operations.
+
+```rust
+{{#include ../../examples/arithmetics.rs}}
+```
+
+An overview of the implemented functionality.
+
+* arithmetics, checked, saturating, etc.
+* `sum`, `min` and `max`
+* `unary`, `binary`, etc.
+* `comparison`
+* `cast`
+* `take`, `filter`, `concat`
+* `sort`, `hash`, `merge-sort`
+* `if-then-else`
+* `nullif`
+* `lenght` (of string)
+* `hour`, `year` (of temporal logical types)
+* `regex`
+* (list) `contains`
diff --git a/src/compute/arithmetics/mod.rs b/src/compute/arithmetics/mod.rs
index ce21ed3c14..496f02d72f 100644
--- a/src/compute/arithmetics/mod.rs
+++ b/src/compute/arithmetics/mod.rs
@@ -228,7 +228,8 @@ pub enum Operator {
 }
 
 /// Perform arithmetic operations on two primitive arrays based on the Operator enum
-fn arithmetic_primitive<T>(
+//
+pub fn arithmetic_primitive<T>(
     lhs: &PrimitiveArray<T>,
     op: Operator,
     rhs: &PrimitiveArray<T>,
diff --git a/src/compute/arity.rs b/src/compute/arity.rs
index 0e99c488e1..e69d8162a2 100644
--- a/src/compute/arity.rs
+++ b/src/compute/arity.rs
@@ -128,7 +128,8 @@ where
 /// Applies a binary operations to two primitive arrays. This is the fastest
 /// way to perform an operation on two primitive array when the benefits of a
 /// vectorized operation outweighs the cost of branching nulls and non-nulls.
-///
+/// # Errors
+/// This function errors iff the arrays have a different length.
 /// # Implementation
 /// This will apply the function for all values, including those on null slots.
 /// This implies that the operation must be infallible for any value of the
diff --git a/src/compute/merge_sort/mod.rs b/src/compute/merge_sort/mod.rs
index 3dad0afdcb..e3c2d4fedd 100644
--- a/src/compute/merge_sort/mod.rs
+++ b/src/compute/merge_sort/mod.rs
@@ -77,7 +77,7 @@ use crate::error::Result;
 /// This representation is useful when building arrays in memory as it allows to memcopy slices of arrays.
 /// This is particularly useful in merge-sort because sorted arrays (passed to the merge-sort) are more likely
 /// to have contiguous blocks of sorted elements (than by random).
-type MergeSlice = (usize, usize, usize);
+pub type MergeSlice = (usize, usize, usize);
 
 /// Takes N arrays together through `slices` under the assumption that the slices have
 /// a total coverage of the arrays.
@@ -234,7 +234,7 @@ fn recursive_merge_sort(slices: &[&[MergeSlice]], comparator: &Comparator) -> Ve
 
 // An iterator adapter that merge-sorts two iterators of `MergeSlice` into a single `MergeSlice`
 // such that the resulting `MergeSlice`s are ordered according to `comparator`.
-struct MergeSortSlices<'a, L, R>
+pub struct MergeSortSlices<'a, L, R>
 where
     L: Iterator<Item = &'a MergeSlice>,
     R: Iterator<Item = &'a MergeSlice>,
@@ -426,7 +426,11 @@ where
 /// Given two iterators of slices representing two sets of sorted [`Array`]s, and a `comparator` bound to those [`Array`]s,
 /// returns a new iterator of slices denoting how to `take` slices from each of the arrays such that the resulting
 /// array is sorted according to `comparator`
-fn merge_sort_slices<'a, L: Iterator<Item = &'a MergeSlice>, R: Iterator<Item = &'a MergeSlice>>(
+pub fn merge_sort_slices<
+    'a,
+    L: Iterator<Item = &'a MergeSlice>,
+    R: Iterator<Item = &'a MergeSlice>,
+>(
     lhs: L,
     rhs: R,
     comparator: &'a Comparator,
@@ -439,7 +443,7 @@ type Comparator<'a> = Box<dyn Fn(usize, usize, usize, usize) -> Ordering + 'a>;
 type IsValid<'a> = Box<dyn Fn(usize) -> bool + 'a>;
 
 /// returns a comparison function between any two arrays of each pair of arrays, according to `SortOptions`.
-fn build_comparator<'a>(
+pub fn build_comparator<'a>(
     pairs: &'a [(&'a [&'a dyn Array], &SortOptions)],
 ) -> Result<Comparator<'a>> {
     // prepare the comparison function of _values_ between all pairs of arrays
diff --git a/src/io/ipc/read/reader.rs b/src/io/ipc/read/reader.rs
index 8860426bae..648c332a39 100644
--- a/src/io/ipc/read/reader.rs
+++ b/src/io/ipc/read/reader.rs
@@ -26,6 +26,7 @@ use crate::record_batch::{RecordBatch, RecordBatchReader};
 use super::super::{convert, gen};
 use super::super::{ARROW_MAGIC, CONTINUATION_MARKER};
 use super::common::*;
+use flatbuffers::VerifierOptions;
 
 type ArrayRef = Arc<dyn Array>;
 
@@ -95,7 +96,17 @@ pub fn read_file_metadata<R: Read + Seek>(reader: &mut R) -> Result<FileMetadata
     reader.seek(SeekFrom::End(-10 - footer_len as i64))?;
     reader.read_exact(&mut footer_data)?;
 
-    let footer = gen::File::root_as_footer(&footer_data[..])
+    // set flatbuffer verification options to the same settings as the C++ arrow implementation.
+    // Heuristic: tables in a Arrow flatbuffers buffer must take at least 1 bit
+    // each in average (ARROW-11559).
+    // Especially, the only recursive table (the `Field` table in Schema.fbs)
+    // must have a non-empty `type` member.
+    let verifier_options = VerifierOptions {
+        max_depth: 128,
+        max_tables: footer_len as usize * 8,
+        ..Default::default()
+    };
+    let footer = gen::File::root_as_footer_with_opts(&verifier_options, &footer_data[..])
         .map_err(|err| ArrowError::Ipc(format!("Unable to get root as footer: {:?}", err)))?;
 
     let blocks = footer.recordBatches().ok_or_else(|| {


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index fbef76bacc..4655dd4afc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -271,7 +271,7 @@ jobs:
           export CARGO_HOME="/github/home/.cargo"
           export CARGO_TARGET_DIR="/github/home/target"
           # no need 
-          cargo build --no-default-features --features=merge_sort,io_ipc,io_csv,io_json --target wasm32-unknown-unknown
+          cargo build --no-default-features --features=merge_sort,io_ipc,io_csv,io_json,io_parquet --target wasm32-unknown-unknown
 
   linux-simd-test:
     name: SIMD