Repository: jorgecarleitao/arrow2
Pull request #1127 (2022-06-29 16:26:46)

Avoid zeroed allocation in reading avro

codecov[bot]: # [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) Report
> Merging [#1127](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (31715ed) into [main](https://codecov.io/gh/jorgecarleitao/arrow2/commit/81ab424913daad775ce63be62f7b57b145313267?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) (81ab424) will **decrease** coverage by `0.00%`.
> The diff coverage is `92.30%`.

```diff
@@            Coverage Diff             @@
##             main    #1127      +/-   ##
==========================================
- Coverage   83.37%   83.37%   -0.01%     
==========================================
  Files         367      367              
  Lines       35574    35579       +5     
==========================================
+ Hits        29661    29664       +3     
- Misses       5913     5915       +2     
```


| [Impacted Files](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao) | Coverage Δ | |
|---|---|---|
| [src/io/avro/read\_async/block.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL2F2cm8vcmVhZF9hc3luYy9ibG9jay5ycw==) | `87.75% <80.00%> (-1.38%)` | :arrow_down: |
| [src/io/avro/read/block.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL2F2cm8vcmVhZC9ibG9jay5ycw==) | `69.09% <100.00%> (ø)` | |
| [src/io/avro/read/util.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL2F2cm8vcmVhZC91dGlsLnJz) | `96.29% <100.00%> (+0.14%)` | :arrow_up: |
| [src/io/avro/read\_async/metadata.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2lvL2F2cm8vcmVhZF9hc3luYy9tZXRhZGF0YS5ycw==) | `96.87% <100.00%> (+0.10%)` | :arrow_up: |
| [src/chunk.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2NodW5rLnJz) | `83.33% <0.00%> (-7.15%)` | :arrow_down: |
| [src/bitmap/immutable.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2JpdG1hcC9pbW11dGFibGUucnM=) | `81.48% <0.00%> (+0.61%)` | :arrow_up: |
| [src/bitmap/utils/slice\_iterator.rs](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao#diff-c3JjL2JpdG1hcC91dGlscy9zbGljZV9pdGVyYXRvci5ycw==) | `98.78% <0.00%> (+1.21%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Last update [81ab424...31715ed](https://codecov.io/gh/jorgecarleitao/arrow2/pull/1127?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Jorge+Leitao).


Patch:
diff --git a/src/io/avro/read/block.rs b/src/io/avro/read/block.rs
index c2e74ec883..62cc5a88ef 100644
--- a/src/io/avro/read/block.rs
+++ b/src/io/avro/read/block.rs
@@ -40,8 +40,8 @@ fn read_block<R: Read>(
     };
 
     block.data.clear();
-    block.data.resize(bytes, 0);
-    reader.read_exact(&mut block.data)?;
+    block.data.try_reserve(bytes)?;
+    reader.take(bytes as u64).read_to_end(&mut block.data)?;
 
     let mut marker = [0u8; 16];
     reader.read_exact(&mut marker)?;
diff --git a/src/io/avro/read/util.rs b/src/io/avro/read/util.rs
index 7bda9d84fa..92cbe3db2b 100644
--- a/src/io/avro/read/util.rs
+++ b/src/io/avro/read/util.rs
@@ -23,8 +23,9 @@ fn decode_variable<R: Read>(reader: &mut R) -> Result<u64> {
 
 fn _read_binary<R: Read>(reader: &mut R) -> Result<Vec<u8>> {
     let len: usize = zigzag_i64(reader)? as usize;
-    let mut buf = vec![0u8; len];
-    reader.read_exact(&mut buf)?;
+    let mut buf = vec![];
+    buf.try_reserve(len)?;
+    reader.take(len as u64).read_to_end(&mut buf)?;
     Ok(buf)
 }
 
diff --git a/src/io/avro/read_async/block.rs b/src/io/avro/read_async/block.rs
index 89069cad83..ff92d6a97e 100644
--- a/src/io/avro/read_async/block.rs
+++ b/src/io/avro/read_async/block.rs
@@ -42,8 +42,11 @@ async fn read_block<R: AsyncRead + Unpin + Send>(
     };
 
     block.data.clear();
-    block.data.resize(bytes, 0);
-    reader.read_exact(&mut block.data).await?;
+    block.data.try_reserve(bytes)?;
+    reader
+        .take(bytes as u64)
+        .read_to_end(&mut block.data)
+        .await?;
 
     let mut marker = [0u8; 16];
     reader.read_exact(&mut marker).await?;
diff --git a/src/io/avro/read_async/metadata.rs b/src/io/avro/read_async/metadata.rs
index 7d8df6888f..4609509376 100644
--- a/src/io/avro/read_async/metadata.rs
+++ b/src/io/avro/read_async/metadata.rs
@@ -48,8 +48,9 @@ async fn read_file_marker<R: AsyncRead + Unpin + Send>(reader: &mut R) -> Result
 
 async fn _read_binary<R: AsyncRead + Unpin + Send>(reader: &mut R) -> Result<Vec<u8>> {
     let len: usize = zigzag_i64(reader).await? as usize;
-    let mut buf = vec![0u8; len];
-    reader.read_exact(&mut buf).await?;
+    let mut buf = vec![];
+    buf.try_reserve(len)?;
+    reader.take(len as u64).read_to_end(&mut buf).await?;
     Ok(buf)
 }
 
diff --git a/src/io/csv/write/mod.rs b/src/io/csv/write/mod.rs
index 1f124ef676..62f228ef14 100644
--- a/src/io/csv/write/mod.rs
+++ b/src/io/csv/write/mod.rs
@@ -27,16 +27,16 @@ fn new_serializers<'a, A: AsRef<dyn Array>>(
 /// The vector is guaranteed to have `columns.len()` entries.
 /// Each `row` is guaranteed to have `columns.array().len()` fields.
 pub fn serialize<A: AsRef<dyn Array>>(
-    columns: &Chunk<A>,
+    chunk: &Chunk<A>,
     options: &SerializeOptions,
 ) -> Result<Vec<Vec<u8>>> {
-    let mut serializers = new_serializers(columns, options)?;
+    let mut serializers = new_serializers(chunk, options)?;
 
-    let mut rows = Vec::with_capacity(columns.len());
+    let mut rows = Vec::with_capacity(chunk.len());
     let mut row = vec![];
 
     // this is where the (expensive) transposition happens: the outer loop is on rows, the inner on columns
-    (0..columns.len()).try_for_each(|_| {
+    (0..chunk.len()).try_for_each(|_| {
         serializers
             .iter_mut()
             // `unwrap` is infalible because `array.len()` equals `Chunk::len`
@@ -49,8 +49,7 @@ pub fn serialize<A: AsRef<dyn Array>>(
             // replace last delimiter with new line
             let last_byte = row.len() - 1;
             row[last_byte] = b'\n';
-            rows.push(row.clone());
-            row.clear();
+            rows.push(std::mem::take(&mut row));
         }
         Result::Ok(())
     })?;


Test patch:
diff --git a/tests/it/io/csv/write.rs b/tests/it/io/csv/write.rs
index 88b502adfb..44bec50497 100644
--- a/tests/it/io/csv/write.rs
+++ b/tests/it/io/csv/write.rs
@@ -423,3 +423,23 @@ fn write_escaping_resize_local_buf() {
         test_generic(chunk,  &format!("\"{}\"\n", payload));
     }
 }
+
+#[test]
+fn serialize_vec() -> Result<()> {
+    let columns = data();
+
+    let options = SerializeOptions::default();
+
+    let data = serialize(&columns, &options)?;
+
+    // check
+    assert_eq!(
+        vec![
+            b"a b,123.564532,3,true,,00:20:34,d\n".to_vec(),
+            b"c,,2,false,2019-04-18 10:54:47.378,06:51:20,a b\n".to_vec(),
+            b"d,-556132.25,1,,2019-04-18 02:45:55.555,23:46:03,c\n".to_vec(),
+        ],
+        data
+    );
+    Ok(())
+}