Repository: jorgecarleitao/arrow2
Pull request #1143 (2022-07-07 13:38:56)

fix casting dictionary keys
Updating arrow2 had some failed test in polars. It turned out that casting dictionary keys was a no-op. This PR fixes that and adds a test.

codecov[bot]: # [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) Report
> Merging [#1143](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (eeddfac) into [main](https://codecov.io/gh/jorgecarleitao/arrow2/commit/98e49133b2e56e51e30335830485b3cf768eb5a2?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (98e4913) will **increase** coverage by `0.00%`.
> The diff coverage is `42.85%`.

```diff
@@           Coverage Diff           @@
##             main    #1143   +/-   ##
=======================================
  Coverage   83.59%   83.60%           
=======================================
  Files         366      366           
  Lines       35901    35903    +2     
=======================================
+ Hits        30013    30018    +5     
+ Misses       5888     5885    -3     
```


| [Impacted Files](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) | Coverage Δ | |
|---|---|---|
| [src/compute/cast/dictionary\_to.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2NvbXB1dGUvY2FzdC9kaWN0aW9uYXJ5X3RvLnJz) | `26.27% <42.85%> (+3.85%)` | :arrow_up: |
| [src/io/ipc/read/reader.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL2lwYy9yZWFkL3JlYWRlci5ycw==) | `95.98% <0.00%> (-0.67%)` | :arrow_down: |
| [src/io/ipc/read/schema.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL2lwYy9yZWFkL3NjaGVtYS5ycw==) | `94.56% <0.00%> (-0.32%)` | :arrow_down: |
| [src/array/utf8/mod.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2FycmF5L3V0ZjgvbW9kLnJz) | `86.58% <0.00%> (+0.95%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Last update [98e4913...eeddfac](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1143?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).


Patch:
diff --git a/Cargo.toml b/Cargo.toml
index 4bc21d5948..b9d6749aca 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,7 +64,7 @@ base64 = { version = "0.13.0", optional = true }
 futures = { version = "0.3", optional = true }
 
 # for faster hashing
-ahash = { version = "0.7", optional = true }
+ahash = { version = "0.7" }
 
 # parquet support
 parquet2 = { version = "0.14.0", optional = true, default_features = false }
@@ -175,7 +175,7 @@ compute_comparison = ["compute_take", "compute_boolean"]
 compute_concatenate = []
 compute_contains = []
 compute_filter = []
-compute_hash = ["multiversion", "ahash"]
+compute_hash = ["multiversion"]
 compute_if_then_else = []
 compute_length = []
 compute_like = ["regex"]
diff --git a/src/array/growable/dictionary.rs b/src/array/growable/dictionary.rs
index 0b3c432525..930f4268f2 100644
--- a/src/array/growable/dictionary.rs
+++ b/src/array/growable/dictionary.rs
@@ -88,11 +88,22 @@ impl<'a, T: DictionaryKey> GrowableDictionary<'a, T> {
         let validity = std::mem::take(&mut self.key_validity);
         let key_values = std::mem::take(&mut self.key_values);
 
+        #[cfg(debug_assertions)]
+        {
+            crate::array::specification::check_indexes(&key_values, self.values.len()).unwrap();
+        }
         let keys =
-            PrimitiveArray::<T>::try_new(T::PRIMITIVE.into(), key_values.into(), validity.into())
-                .unwrap();
-
-        DictionaryArray::<T>::try_new(self.data_type.clone(), keys, self.values.clone()).unwrap()
+            PrimitiveArray::<T>::new(T::PRIMITIVE.into(), key_values.into(), validity.into());
+
+        // Safety - the invariant of this struct ensures that this is up-held
+        unsafe {
+            DictionaryArray::<T>::try_new_unchecked(
+                self.data_type.clone(),
+                keys,
+                self.values.clone(),
+            )
+            .unwrap()
+        }
     }
 }
 
@@ -141,12 +152,7 @@ impl<'a, T: DictionaryKey> Growable<'a> for GrowableDictionary<'a, T> {
 
 impl<'a, T: DictionaryKey> From<GrowableDictionary<'a, T>> for DictionaryArray<T> {
     #[inline]
-    fn from(val: GrowableDictionary<'a, T>) -> Self {
-        let data_type = T::PRIMITIVE.into();
-        let keys =
-            PrimitiveArray::<T>::try_new(data_type, val.key_values.into(), val.key_validity.into())
-                .unwrap();
-
-        DictionaryArray::<T>::try_new(val.data_type.clone(), keys, val.values).unwrap()
+    fn from(mut val: GrowableDictionary<'a, T>) -> Self {
+        val.to()
     }
 }
diff --git a/src/array/growable/list.rs b/src/array/growable/list.rs
index 41db9ae789..0e1e7ceb5f 100644
--- a/src/array/growable/list.rs
+++ b/src/array/growable/list.rs
@@ -104,12 +104,20 @@ impl<'a, O: Offset> GrowableList<'a, O> {
         let offsets = std::mem::take(&mut self.offsets);
         let values = self.values.as_box();
 
-        ListArray::<O>::new(
-            self.arrays[0].data_type().clone(),
-            offsets.into(),
-            values,
-            validity.into(),
-        )
+        #[cfg(debug_assertions)]
+        {
+            crate::array::specification::try_check_offsets(&offsets, values.len()).unwrap();
+        }
+
+        // Safety - the invariant of this struct ensures that this is up-held
+        unsafe {
+            ListArray::<O>::new_unchecked(
+                self.arrays[0].data_type().clone(),
+                offsets.into(),
+                values,
+                validity.into(),
+            )
+        }
     }
 }
 
@@ -135,15 +143,7 @@ impl<'a, O: Offset> Growable<'a> for GrowableList<'a, O> {
 }
 
 impl<'a, O: Offset> From<GrowableList<'a, O>> for ListArray<O> {
-    fn from(val: GrowableList<'a, O>) -> Self {
-        let mut values = val.values;
-        let values = values.as_box();
-
-        ListArray::<O>::new(
-            val.arrays[0].data_type().clone(),
-            val.offsets.into(),
-            values,
-            val.validity.into(),
-        )
+    fn from(mut val: GrowableList<'a, O>) -> Self {
+        val.to()
     }
 }
diff --git a/src/array/growable/utf8.rs b/src/array/growable/utf8.rs
index c50c6fd4db..adfdba2cd5 100644
--- a/src/array/growable/utf8.rs
+++ b/src/array/growable/utf8.rs
@@ -55,13 +55,19 @@ impl<'a, O: Offset> GrowableUtf8<'a, O> {
         let offsets = std::mem::take(&mut self.offsets);
         let values = std::mem::take(&mut self.values);
 
+        #[cfg(debug_assertions)]
+        {
+            crate::array::specification::try_check_offsets_and_utf8(&offsets, &values).unwrap();
+        }
+
         unsafe {
-            Utf8Array::<O>::from_data_unchecked(
+            Utf8Array::<O>::try_new_unchecked(
                 self.arrays[0].data_type().clone(),
                 offsets.into(),
                 values.into(),
                 validity.into(),
             )
+            .unwrap()
         }
     }
 }
@@ -99,14 +105,7 @@ impl<'a, O: Offset> Growable<'a> for GrowableUtf8<'a, O> {
 }
 
 impl<'a, O: Offset> From<GrowableUtf8<'a, O>> for Utf8Array<O> {
-    fn from(val: GrowableUtf8<'a, O>) -> Self {
-        unsafe {
-            Utf8Array::<O>::from_data_unchecked(
-                val.arrays[0].data_type().clone(),
-                val.offsets.into(),
-                val.values.into(),
-                val.validity.into(),
-            )
-        }
+    fn from(mut val: GrowableUtf8<'a, O>) -> Self {
+        val.to()
     }
 }
diff --git a/src/array/union/mod.rs b/src/array/union/mod.rs
index 6d605da064..3bda591568 100644
--- a/src/array/union/mod.rs
+++ b/src/array/union/mod.rs
@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use ahash::AHashMap;
 
 use crate::{
     bitmap::Bitmap,
@@ -31,7 +31,7 @@ type UnionComponents<'a> = (&'a [Field], Option<&'a [i32]>, UnionMode);
 pub struct UnionArray {
     types: Buffer<i8>,
     // None represents when there is no typeid
-    fields_hash: Option<HashMap<i8, FieldEntry>>,
+    fields_hash: Option<AHashMap<i8, FieldEntry>>,
     fields: Vec<Box<dyn Array>>,
     offsets: Option<Buffer<i32>>,
     data_type: DataType,
diff --git a/src/compute/cast/dictionary_to.rs b/src/compute/cast/dictionary_to.rs
index 01d3325cf9..101669f644 100644
--- a/src/compute/cast/dictionary_to.rs
+++ b/src/compute/cast/dictionary_to.rs
@@ -7,7 +7,7 @@ use crate::{
 };
 
 macro_rules! key_cast {
-    ($keys:expr, $values:expr, $array:expr, $to_keys_type:expr, $to_type:ty) => {{
+    ($keys:expr, $values:expr, $array:expr, $to_keys_type:expr, $to_type:ty, $to_datatype:expr) => {{
         let cast_keys = primitive_to_primitive::<_, $to_type>($keys, $to_keys_type);
 
         // Failure to cast keys (because they don't fit in the
@@ -15,7 +15,11 @@ macro_rules! key_cast {
         if cast_keys.null_count() > $keys.null_count() {
             return Err(Error::Overflow);
         }
-        DictionaryArray::try_new($array.data_type().clone(), $keys.clone(), $values.clone())
+        // Safety: this is safe because given a type `T` that fits in a `usize`, casting it to type `P` either overflows or also fits in a `usize`
+        unsafe {
+             DictionaryArray::try_new_unchecked($to_datatype, cast_keys, $values.clone())
+        }
+            .map(|x| x.boxed())
     }};
 }
 
@@ -88,8 +92,8 @@ where
             Box::new(values.data_type().clone()),
             is_ordered,
         );
-        // some of the values may not fit in `usize` and thus this needs to be checked
-        DictionaryArray::try_new(data_type, casted_keys, values.clone())
+        // Safety: this is safe because given a type `T` that fits in a `usize`, casting it to type `P` either overflows or also fits in a `usize`
+        unsafe { DictionaryArray::try_new_unchecked(data_type, casted_keys, values.clone()) }
     }
 }
 
@@ -134,11 +138,13 @@ pub(super) fn dictionary_cast_dyn<K: DictionaryKey + num_traits::NumCast>(
             let values = cast(values.as_ref(), to_values_type, options)?;
 
             // create the appropriate array type
-            let data_type = (*to_keys_type).into();
+            let to_key_type = (*to_keys_type).into();
+
+            // Safety:
+            // we return an error on overflow so the integers remain within bounds
             match_integer_type!(to_keys_type, |$T| {
-                key_cast!(keys, values, array, &data_type, $T)
+                key_cast!(keys, values, array, &to_key_type, $T, to_type.clone())
             })
-            .map(|x| x.boxed())
         }
         _ => unpack_dictionary::<K>(keys, values.as_ref(), to_type, options),
     }
diff --git a/src/compute/like.rs b/src/compute/like.rs
index 4509617398..cc7517e823 100644
--- a/src/compute/like.rs
+++ b/src/compute/like.rs
@@ -1,6 +1,6 @@
 //! Contains "like" operators such as [`like_utf8`] and [`like_utf8_scalar`].
-use std::collections::HashMap;
 
+use ahash::AHashMap;
 use regex::bytes::Regex as BytesRegex;
 use regex::Regex;
 
@@ -35,7 +35,7 @@ fn a_like_utf8<O: Offset, F: Fn(bool) -> bool>(
 
     let validity = combine_validities(lhs.validity(), rhs.validity());
 
-    let mut map = HashMap::new();
+    let mut map = AHashMap::new();
 
     let values =
         Bitmap::try_from_trusted_len_iter(lhs.iter().zip(rhs.iter()).map(|(lhs, rhs)| {
@@ -179,7 +179,7 @@ fn a_like_binary<O: Offset, F: Fn(bool) -> bool>(
 
     let validity = combine_validities(lhs.validity(), rhs.validity());
 
-    let mut map = HashMap::new();
+    let mut map = AHashMap::new();
 
     let values =
         Bitmap::try_from_trusted_len_iter(lhs.iter().zip(rhs.iter()).map(|(lhs, rhs)| {
diff --git a/src/compute/merge_sort/mod.rs b/src/compute/merge_sort/mod.rs
index 9c48a29dd0..2776d58266 100644
--- a/src/compute/merge_sort/mod.rs
+++ b/src/compute/merge_sort/mod.rs
@@ -56,8 +56,9 @@
 //! To serialize slices, e.g. for checkpointing or transfer via Arrow's IPC, you can store
 //! them as 3 non-null primitive arrays (e.g. `PrimitiveArray<i64>`).
 
+use ahash::AHashMap;
+use std::cmp::Ordering;
 use std::iter::once;
-use std::{cmp::Ordering, collections::HashMap};
 
 use itertools::Itertools;
 
@@ -498,7 +499,7 @@ pub fn build_comparator_impl<'a>(
                 .collect::<Result<Vec<_>>>()?;
             Ok(((lhs_index, rhs_index), multi_column_comparator))
         })
-        .collect::<Result<HashMap<(usize, usize), Vec<(IsValid, IsValid, DynComparator)>>>>()?;
+        .collect::<Result<AHashMap<(usize, usize), Vec<(IsValid, IsValid, DynComparator)>>>>()?;
 
     // prepare a comparison function taking into account _nulls_ and sort options
     let cmp = move |left_index, left_row, right_index, right_row| {
diff --git a/src/compute/regex_match.rs b/src/compute/regex_match.rs
index 001bd0f5d2..41cacb293b 100644
--- a/src/compute/regex_match.rs
+++ b/src/compute/regex_match.rs
@@ -1,7 +1,6 @@
 //! Contains regex matching operators [`regex_match`] and [`regex_match_scalar`].
 
-use std::collections::HashMap;
-
+use ahash::AHashMap;
 use regex::Regex;
 
 use super::utils::combine_validities;
@@ -18,7 +17,7 @@ pub fn regex_match<O: Offset>(values: &Utf8Array<O>, regex: &Utf8Array<O>) -> Re
         ));
     }
 
-    let mut map = HashMap::new();
+    let mut map = AHashMap::new();
     let validity = combine_validities(values.validity(), regex.validity());
 
     let iterator = values.iter().zip(regex.iter()).map(|(haystack, regex)| {
diff --git a/src/io/avro/mod.rs b/src/io/avro/mod.rs
index 852915d900..5addae95e4 100644
--- a/src/io/avro/mod.rs
+++ b/src/io/avro/mod.rs
@@ -48,7 +48,7 @@ macro_rules! avro_decode {
 
 macro_rules! read_header {
     ($reader:ident $($_await:tt)*) => {{
-        let mut items = HashMap::new();
+        let mut items = ahash::AHashMap::new();
 
         loop {
             let len = zigzag_i64($reader)$($_await)*? as usize;
diff --git a/src/io/avro/read/header.rs b/src/io/avro/read/header.rs
index 66bec96cc0..c985ce2508 100644
--- a/src/io/avro/read/header.rs
+++ b/src/io/avro/read/header.rs
@@ -1,5 +1,4 @@
-use std::collections::HashMap;
-
+use ahash::AHashMap;
 use avro_schema::Schema;
 use serde_json;
 
@@ -9,7 +8,7 @@ use super::Compression;
 
 /// Deserializes the Avro header into an Avro [`Schema`] and optional [`Compression`].
 pub(crate) fn deserialize_header(
-    header: HashMap<String, Vec<u8>>,
+    header: AHashMap<String, Vec<u8>>,
 ) -> Result<(Schema, Option<Compression>)> {
     let schema = header
         .get("avro.schema")
diff --git a/src/io/avro/read/util.rs b/src/io/avro/read/util.rs
index 92cbe3db2b..b14870a390 100644
--- a/src/io/avro/read/util.rs
+++ b/src/io/avro/read/util.rs
@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use ahash::AHashMap;
 use std::io::Read;
 
 use avro_schema::Schema;
@@ -29,7 +29,7 @@ fn _read_binary<R: Read>(reader: &mut R) -> Result<Vec<u8>> {
     Ok(buf)
 }
 
-fn read_header<R: Read>(reader: &mut R) -> Result<HashMap<String, Vec<u8>>> {
+fn read_header<R: Read>(reader: &mut R) -> Result<AHashMap<String, Vec<u8>>> {
     read_header!(reader)
 }
 
diff --git a/src/io/avro/read_async/metadata.rs b/src/io/avro/read_async/metadata.rs
index 4609509376..213428e0f0 100644
--- a/src/io/avro/read_async/metadata.rs
+++ b/src/io/avro/read_async/metadata.rs
@@ -1,6 +1,5 @@
 //! Async Avro
-use std::collections::HashMap;
-
+use ahash::AHashMap;
 use avro_schema::{Record, Schema as AvroSchema};
 use futures::AsyncRead;
 use futures::AsyncReadExt;
@@ -56,6 +55,6 @@ async fn _read_binary<R: AsyncRead + Unpin + Send>(reader: &mut R) -> Result<Vec
 
 async fn read_header<R: AsyncRead + Unpin + Send>(
     reader: &mut R,
-) -> Result<HashMap<String, Vec<u8>>> {
+) -> Result<AHashMap<String, Vec<u8>>> {
     read_header!(reader.await)
 }
diff --git a/src/io/avro/write/header.rs b/src/io/avro/write/header.rs
index 1d25f2f596..370a0532b5 100644
--- a/src/io/avro/write/header.rs
+++ b/src/io/avro/write/header.rs
@@ -1,5 +1,4 @@
-use std::collections::HashMap;
-
+use ahash::AHashMap;
 use avro_schema::Schema;
 use serde_json;
 
@@ -11,10 +10,10 @@ use super::Compression;
 pub(crate) fn serialize_header(
     schema: &Schema,
     compression: Option<Compression>,
-) -> Result<HashMap<String, Vec<u8>>> {
+) -> Result<AHashMap<String, Vec<u8>>> {
     let schema = serde_json::to_string(schema).map_err(|e| Error::ExternalFormat(e.to_string()))?;
 
-    let mut header = HashMap::<String, Vec<u8>>::default();
+    let mut header = AHashMap::<String, Vec<u8>>::default();
 
     header.insert("avro.schema".to_string(), schema.into_bytes());
     if let Some(compression) = compression {
diff --git a/src/io/csv/read/infer_schema.rs b/src/io/csv/read/infer_schema.rs
index 83462bc1e8..4aa3eaf53e 100644
--- a/src/io/csv/read/infer_schema.rs
+++ b/src/io/csv/read/infer_schema.rs
@@ -1,7 +1,5 @@
-use std::{
-    collections::HashSet,
-    io::{Read, Seek},
-};
+use ahash::AHashSet;
+use std::io::{Read, Seek};
 
 use crate::datatypes::{DataType, Field};
 use crate::error::Result;
@@ -34,7 +32,7 @@ pub fn infer_schema<R: Read + Seek, F: Fn(&[u8]) -> DataType>(
 
     let header_length = headers.len();
     // keep track of inferred field types
-    let mut column_types: Vec<HashSet<DataType>> = vec![HashSet::new(); header_length];
+    let mut column_types: Vec<AHashSet<DataType>> = vec![AHashSet::new(); header_length];
 
     let mut records_count = 0;
 
diff --git a/src/io/csv/read_async/infer_schema.rs b/src/io/csv/read_async/infer_schema.rs
index b22f2a613d..2a2a806159 100644
--- a/src/io/csv/read_async/infer_schema.rs
+++ b/src/io/csv/read_async/infer_schema.rs
@@ -1,6 +1,5 @@
-use std::collections::HashSet;
-
 use super::{AsyncReader, ByteRecord};
+use ahash::AHashSet;
 
 use crate::datatypes::{DataType, Field};
 use crate::error::Result;
@@ -41,7 +40,7 @@ where
 
     let header_length = headers.len();
     // keep track of inferred field types
-    let mut column_types: Vec<HashSet<DataType>> = vec![HashSet::new(); header_length];
+    let mut column_types: Vec<AHashSet<DataType>> = vec![AHashSet::new(); header_length];
 
     let mut records_count = 0;
 
diff --git a/src/io/csv/utils.rs b/src/io/csv/utils.rs
index 266801d940..436c7d192f 100644
--- a/src/io/csv/utils.rs
+++ b/src/io/csv/utils.rs
@@ -1,6 +1,5 @@
-use std::collections::HashSet;
-
 use crate::datatypes::{DataType, Field, TimeUnit};
+use ahash::AHashSet;
 
 pub(super) const RFC3339: &str = "%Y-%m-%dT%H:%M:%S%.f%:z";
 
@@ -78,7 +77,7 @@ pub fn infer(bytes: &[u8]) -> DataType {
     }
 }
 
-fn merge_fields(field_name: &str, possibilities: &mut HashSet<DataType>) -> Field {
+fn merge_fields(field_name: &str, possibilities: &mut AHashSet<DataType>) -> Field {
     // determine data type based on possible types
     // if there are incompatible types, use DataType::Utf8
     let data_type = match possibilities.len() {
@@ -101,7 +100,7 @@ fn merge_fields(field_name: &str, possibilities: &mut HashSet<DataType>) -> Fiel
 
 pub(crate) fn merge_schema(
     headers: &[String],
-    column_types: &mut [HashSet<DataType>],
+    column_types: &mut [AHashSet<DataType>],
 ) -> Vec<Field> {
     headers
         .iter()
diff --git a/src/io/ipc/read/common.rs b/src/io/ipc/read/common.rs
index fe445f2079..5638a5289e 100644
--- a/src/io/ipc/read/common.rs
+++ b/src/io/ipc/read/common.rs
@@ -1,4 +1,5 @@
-use std::collections::{HashMap, HashSet, VecDeque};
+use ahash::AHashMap;
+use std::collections::VecDeque;
 use std::io::{Read, Seek};
 
 use arrow_format;
@@ -322,40 +323,47 @@ mod tests {
 pub fn prepare_projection(
     fields: &[Field],
     mut projection: Vec<usize>,
-) -> (Vec<usize>, HashMap<usize, usize>, Vec<Field>) {
-    assert_eq!(
-        projection.iter().collect::<HashSet<_>>().len(),
-        projection.len(),
-        "The projection on IPC must not contain duplicates"
-    );
-
+) -> (Vec<usize>, AHashMap<usize, usize>, Vec<Field>) {
     let fields = projection.iter().map(|x| fields[*x].clone()).collect();
 
     // todo: find way to do this more efficiently
     let mut indices = (0..projection.len()).collect::<Vec<_>>();
     indices.sort_unstable_by_key(|&i| &projection[i]);
     let map = indices.iter().copied().enumerate().fold(
-        HashMap::default(),
+        AHashMap::default(),
         |mut acc, (index, new_index)| {
-            if !acc.contains_key(&new_index) {
-                acc.insert(index, new_index);
-            };
+            acc.insert(index, new_index);
             acc
         },
     );
     projection.sort_unstable();
 
+    // check unique
+    if !projection.is_empty() {
+        let mut previous = projection[0];
+
+        for &i in &projection[1..] {
+            assert!(
+                previous < i,
+                "The projection on IPC must not contain duplicates"
+            );
+            previous = i;
+        }
+    }
+
     (projection, map, fields)
 }
 
 pub fn apply_projection(
     chunk: Chunk<Box<dyn Array>>,
-    map: &HashMap<usize, usize>,
+    map: &AHashMap<usize, usize>,
 ) -> Chunk<Box<dyn Array>> {
     // re-order according to projection
-    let mut arrays = chunk.into_arrays();
-    map.iter().for_each(|(old, new)| {
-        arrays.swap(*old, *new);
-    });
-    Chunk::new(arrays)
+    let arrays = chunk.into_arrays();
+    let mut new_arrays = arrays.clone();
+
+    map.iter()
+        .for_each(|(old, new)| new_arrays[*new] = arrays[*old].clone());
+
+    Chunk::new(new_arrays)
 }
diff --git a/src/io/ipc/read/file_async.rs b/src/io/ipc/read/file_async.rs
index 0d80b80ac6..4b36a44026 100644
--- a/src/io/ipc/read/file_async.rs
+++ b/src/io/ipc/read/file_async.rs
@@ -1,5 +1,5 @@
 //! Async reader for Arrow IPC files
-use std::collections::HashMap;
+use ahash::AHashMap;
 use std::io::SeekFrom;
 
 use arrow_format::ipc::{planus::ReadAsRoot, Block, MessageHeaderRef};
@@ -73,7 +73,7 @@ impl<'a> FileStream<'a> {
         mut reader: R,
         mut dictionaries: Option<Dictionaries>,
         metadata: FileMetadata,
-        projection: Option<(Vec<usize>, HashMap<usize, usize>)>,
+        projection: Option<(Vec<usize>, AHashMap<usize, usize>)>,
         limit: Option<usize>,
     ) -> BoxStream<'a, Result<Chunk<Box<dyn Array>>>>
     where
diff --git a/src/io/ipc/read/mod.rs b/src/io/ipc/read/mod.rs
index 5ffe6426e2..f623e0aef5 100644
--- a/src/io/ipc/read/mod.rs
+++ b/src/io/ipc/read/mod.rs
@@ -4,7 +4,7 @@
 //! which provides arbitrary access to any of its messages, and the
 //! [`StreamReader`](stream::StreamReader), which only supports reading
 //! data in the order it was written in.
-use std::collections::HashMap;
+use ahash::AHashMap;
 
 use crate::array::Array;
 
@@ -35,7 +35,7 @@ pub use schema::deserialize_schema;
 pub use stream::{read_stream_metadata, StreamMetadata, StreamReader, StreamState};
 
 /// how dictionaries are tracked in this crate
-pub type Dictionaries = HashMap<i64, Box<dyn Array>>;
+pub type Dictionaries = AHashMap<i64, Box<dyn Array>>;
 
 pub(crate) type Node<'a> = arrow_format::ipc::FieldNodeRef<'a>;
 pub(crate) type IpcBuffer<'a> = arrow_format::ipc::BufferRef<'a>;
diff --git a/src/io/ipc/read/reader.rs b/src/io/ipc/read/reader.rs
index b18ca4cead..ab72adec96 100644
--- a/src/io/ipc/read/reader.rs
+++ b/src/io/ipc/read/reader.rs
@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use ahash::AHashMap;
 use std::convert::TryInto;
 use std::io::{Read, Seek, SeekFrom};
 
@@ -119,7 +119,7 @@ pub fn read_file_dictionaries<R: Read + Seek>(
     let blocks = if let Some(blocks) = metadata.dictionaries.as_deref() {
         blocks
     } else {
-        return Ok(HashMap::new());
+        return Ok(AHashMap::new());
     };
     // use a temporary smaller scratch for the messages
     let mut message_scratch = Default::default();
@@ -326,7 +326,7 @@ pub struct FileReader<R: Read + Seek> {
     // the dictionaries are going to be read
     dictionaries: Option<Dictionaries>,
     current_block: usize,
-    projection: Option<(Vec<usize>, HashMap<usize, usize>, Schema)>,
+    projection: Option<(Vec<usize>, AHashMap<usize, usize>, Schema)>,
     remaining: usize,
     data_scratch: Vec<u8>,
     message_scratch: Vec<u8>,
diff --git a/src/io/ipc/read/stream.rs b/src/io/ipc/read/stream.rs
index b3f3a986d6..cea6e08825 100644
--- a/src/io/ipc/read/stream.rs
+++ b/src/io/ipc/read/stream.rs
@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use ahash::AHashMap;
 use std::io::Read;
 
 use arrow_format;
@@ -96,7 +96,7 @@ fn read_next<R: Read>(
     dictionaries: &mut Dictionaries,
     message_buffer: &mut Vec<u8>,
     data_buffer: &mut Vec<u8>,
-    projection: &Option<(Vec<usize>, HashMap<usize, usize>, Schema)>,
+    projection: &Option<(Vec<usize>, AHashMap<usize, usize>, Schema)>,
     scratch: &mut Vec<u8>,
 ) -> Result<Option<StreamState>> {
     // determine metadata length
@@ -241,7 +241,7 @@ pub struct StreamReader<R: Read> {
     finished: bool,
     data_buffer: Vec<u8>,
     message_buffer: Vec<u8>,
-    projection: Option<(Vec<usize>, HashMap<usize, usize>, Schema)>,
+    projection: Option<(Vec<usize>, AHashMap<usize, usize>, Schema)>,
     scratch: Vec<u8>,
 }
 
diff --git a/src/io/json_integration/read/array.rs b/src/io/json_integration/read/array.rs
index 2d7d0ec39a..0c44cc2432 100644
--- a/src/io/json_integration/read/array.rs
+++ b/src/io/json_integration/read/array.rs
@@ -1,5 +1,4 @@
-use std::collections::HashMap;
-
+use ahash::AHashMap;
 use num_traits::NumCast;
 use serde_json::Value;
 
@@ -192,7 +191,7 @@ fn to_list<O: Offset>(
     json_col: &ArrowJsonColumn,
     data_type: DataType,
     field: &IpcField,
-    dictionaries: &HashMap<i64, ArrowJsonDictionaryBatch>,
+    dictionaries: &AHashMap<i64, ArrowJsonDictionaryBatch>,
 ) -> Result<Box<dyn Array>> {
     let validity = to_validity(&json_col.validity);
 
@@ -214,7 +213,7 @@ fn to_map(
     json_col: &ArrowJsonColumn,
     data_type: DataType,
     field: &IpcField,
-    dictionaries: &HashMap<i64, ArrowJsonDictionaryBatch>,
+    dictionaries: &AHashMap<i64, ArrowJsonDictionaryBatch>,
 ) -> Result<Box<dyn Array>> {
     let validity = to_validity(&json_col.validity);
 
@@ -234,7 +233,7 @@ fn to_dictionary<K: DictionaryKey + NumCast>(
     data_type: DataType,
     field: &IpcField,
     json_col: &ArrowJsonColumn,
-    dictionaries: &HashMap<i64, ArrowJsonDictionaryBatch>,
+    dictionaries: &AHashMap<i64, ArrowJsonDictionaryBatch>,
 ) -> Result<Box<dyn Array>> {
     // find dictionary
     let dict_id = field.dictionary_id.unwrap();
@@ -260,7 +259,7 @@ pub fn to_array(
     data_type: DataType,
     field: &IpcField,
     json_col: &ArrowJsonColumn,
-    dictionaries: &HashMap<i64, ArrowJsonDictionaryBatch>,
+    dictionaries: &AHashMap<i64, ArrowJsonDictionaryBatch>,
 ) -> Result<Box<dyn Array>> {
     use PhysicalType::*;
     match data_type.to_physical_type() {
@@ -409,7 +408,7 @@ pub fn deserialize_chunk(
     schema: &Schema,
     ipc_fields: &[IpcField],
     json_batch: &ArrowJsonBatch,
-    json_dictionaries: &HashMap<i64, ArrowJsonDictionaryBatch>,
+    json_dictionaries: &AHashMap<i64, ArrowJsonDictionaryBatch>,
 ) -> Result<Chunk<Box<dyn Array>>> {
     let arrays = schema
         .fields
diff --git a/src/io/parquet/write/sink.rs b/src/io/parquet/write/sink.rs
index 160ac81d7f..9ba2d4aea9 100644
--- a/src/io/parquet/write/sink.rs
+++ b/src/io/parquet/write/sink.rs
@@ -1,4 +1,5 @@
-use std::{collections::HashMap, pin::Pin, task::Poll};
+use ahash::AHashMap;
+use std::{pin::Pin, task::Poll};
 
 use futures::{future::BoxFuture, AsyncWrite, FutureExt, Sink, TryFutureExt};
 use parquet2::metadata::KeyValue;
@@ -62,7 +63,7 @@ pub struct FileSink<'a, W: AsyncWrite + Send + Unpin> {
     schema: Schema,
     parquet_schema: SchemaDescriptor,
     /// Key-value metadata that will be written to the file on close.
-    pub metadata: HashMap<String, Option<String>>,
+    pub metadata: AHashMap<String, Option<String>>,
 }
 
 impl<'a, W> FileSink<'a, W>
@@ -105,7 +106,7 @@ where
             schema,
             encodings,
             parquet_schema,
-            metadata: HashMap::default(),
+            metadata: AHashMap::default(),
         })
     }
 
diff --git a/src/lib.rs b/src/lib.rs
index 03963841b5..01fe444d5c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -37,3 +37,6 @@ mod docs;
 
 // re-exported because we return `Either` in our public API
 pub use either::Either;
+
+// re-exported to construct dictionaries
+pub use ahash::AHashMap;


Test patch:
diff --git a/arrow-parquet-integration-testing/src/main.rs b/arrow-parquet-integration-testing/src/main.rs
index f62fba5e09..d79da94318 100644
--- a/arrow-parquet-integration-testing/src/main.rs
+++ b/arrow-parquet-integration-testing/src/main.rs
@@ -1,9 +1,10 @@
 use std::fs::File;
-use std::{collections::HashMap, io::Read};
+use std::{io::Read};
 
 use arrow2::array::Array;
 use arrow2::io::ipc::IpcField;
 use arrow2::{
+    AHashMap,
     chunk::Chunk,
     datatypes::{DataType, Schema},
     error::Result,
@@ -40,7 +41,7 @@ pub fn read_gzip_json(
     let (schema, ipc_fields) = read::deserialize_schema(&schema)?;
 
     // read dictionaries
-    let mut dictionaries = HashMap::new();
+    let mut dictionaries = AHashMap::new();
     if let Some(dicts) = arrow_json.dictionaries {
         for json_dict in dicts {
             // TODO: convert to a concrete Arrow type
diff --git a/integration-testing/src/lib.rs b/integration-testing/src/lib.rs
index 0a3b778e38..e36ad7eef0 100644
--- a/integration-testing/src/lib.rs
+++ b/integration-testing/src/lib.rs
@@ -22,11 +22,11 @@ use arrow2::io::ipc::IpcField;
 use serde_json::Value;
 
 use arrow2::chunk::Chunk;
+use arrow2::AHashMap;
 use arrow2::datatypes::*;
 use arrow2::error::Result;
 use arrow2::io::json_integration::{read, ArrowJsonBatch, ArrowJsonDictionaryBatch};
 
-use std::collections::HashMap;
 use std::fs::File;
 use std::io::BufReader;
 
@@ -43,7 +43,7 @@ pub struct ArrowFile {
     pub fields: Vec<IpcField>,
     // we can evolve this into a concrete Arrow type
     // this is temporarily not being read from
-    pub _dictionaries: HashMap<i64, ArrowJsonDictionaryBatch>,
+    pub _dictionaries: AHashMap<i64, ArrowJsonDictionaryBatch>,
     pub chunks: Vec<Chunk<Box<dyn Array>>>,
 }
 
@@ -54,7 +54,7 @@ pub fn read_json_file(json_name: &str) -> Result<ArrowFile> {
 
     let (schema, fields) = read::deserialize_schema(&arrow_json["schema"])?;
     // read dictionaries
-    let mut dictionaries = HashMap::new();
+    let mut dictionaries = AHashMap::new();
     if let Some(dicts) = arrow_json.get("dictionaries") {
         for d in dicts
             .as_array()
diff --git a/tests/it/compute/cast.rs b/tests/it/compute/cast.rs
index 79ce42c990..9e9888926d 100644
--- a/tests/it/compute/cast.rs
+++ b/tests/it/compute/cast.rs
@@ -805,3 +805,27 @@ fn utf8_to_date64() {
 
     assert_eq!(&expected, c);
 }
+
+#[test]
+fn dict_keys() {
+    let mut array = MutableDictionaryArray::<u8, MutableUtf8Array<i32>>::new();
+    array
+        .try_extend([Some("one"), None, Some("three"), Some("one")])
+        .unwrap();
+    let array: DictionaryArray<u8> = array.into();
+
+    let result = cast(
+        &array,
+        &DataType::Dictionary(IntegerType::Int64, Box::new(DataType::Utf8), false),
+        CastOptions::default(),
+    )
+    .expect("cast failed");
+
+    let mut expected = MutableDictionaryArray::<i64, MutableUtf8Array<i32>>::new();
+    expected
+        .try_extend([Some("one"), None, Some("three"), Some("one")])
+        .unwrap();
+    let expected: DictionaryArray<i64> = expected.into();
+
+    assert_eq!(expected, result.as_ref());
+}
diff --git a/tests/it/io/ipc/common.rs b/tests/it/io/ipc/common.rs
index deaf44ec6a..6254c8f5bb 100644
--- a/tests/it/io/ipc/common.rs
+++ b/tests/it/io/ipc/common.rs
@@ -1,4 +1,5 @@
-use std::{collections::HashMap, fs::File, io::Read};
+use ahash::AHashMap;
+use std::{fs::File, io::Read};
 
 use arrow2::{
     array::Array, chunk::Chunk, datatypes::Schema, error::Result,
@@ -29,7 +30,7 @@ pub fn read_gzip_json(version: &str, file_name: &str) -> Result<IpcRead> {
     let (schema, ipc_fields) = read::deserialize_schema(&schema)?;
 
     // read dictionaries
-    let mut dictionaries = HashMap::new();
+    let mut dictionaries = AHashMap::new();
     if let Some(dicts) = arrow_json.dictionaries {
         for json_dict in dicts {
             // TODO: convert to a concrete Arrow type
diff --git a/tests/it/io/parquet/write_async.rs b/tests/it/io/parquet/write_async.rs
index 89d76efea9..5644caf67f 100644
--- a/tests/it/io/parquet/write_async.rs
+++ b/tests/it/io/parquet/write_async.rs
@@ -1,5 +1,4 @@
-use std::collections::HashMap;
-
+use ahash::AHashMap;
 use arrow2::{
     array::{Float32Array, Int32Array},
     chunk::Chunk,
@@ -46,7 +45,7 @@ async fn test_parquet_async_roundtrip() {
 
     buffer.set_position(0);
     let metadata = read_metadata_async(&mut buffer).await.unwrap();
-    let kv = HashMap::<String, Option<String>>::from_iter(
+    let kv = AHashMap::<String, Option<String>>::from_iter(
         metadata
             .key_value_metadata()
             .to_owned()