Repository: apache/arrow-rs
Pull request #6466 (2024-09-26 15:26:48)

Add `ParquetMetaDataBuilder`
# Which issue does this PR close?

Closes https://github.com/apache/arrow-rs/issues/6465

# Rationale for this change
 
At the moment it is 
1. awkward to modify ParquetMetaData (you have to re-create it from its constitutent fields)
2. and there is no way to avoid clone'ing when trying to modify it

See https://github.com/apache/arrow-rs/issues/6465 for more rationale


# What changes are included in this PR?
1. Add `ParquetMetaDataBuilder`
3. Add documentation and examples
4. Update code that modifies `ParquetMetaData` to use `ParquetMetadataBuilder` (which requires fewer clones) so might be marginally faster

<!--
There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
-->

# Are there any user-facing changes?


<!--
If there are user-facing changes then we may require documentation to be updated before approving the PR.
-->

<!---
If there are any breaking changes to public APIs, please add the `breaking change` label.
-->


alamb: > If you are planning to eventually deprecate the non-builder `ParquetMetaData::new*`, then [this would also need to be switched ](https://github.com/apache/arrow-rs/blob/5804f21d7043f55e6e8775c17a3d14f6f413fc27/parquet/src/file/metadata/reader.rs#L600)to the builder.

That is a good point -- I wasn't planing to deprecate the functions, though we could argume that deprecating `new_with_page_index` might be a good idea 🤔 
alamb: > > If you are planning to eventually deprecate the non-builder `ParquetMetaData::new*`, then [this would also need to be switched ](https://github.com/apache/arrow-rs/blob/5804f21d7043f55e6e8775c17a3d14f6f413fc27/parquet/src/file/metadata/reader.rs#L600)to the builder.
> 
> That is a good point -- I wasn't planing to deprecate the functions, though we could argume that deprecating `new_with_page_index` might be a good idea 🤔

I deprecated `new_with_page_index` in b6c63a8bad

Patch:
diff --git a/parquet/src/arrow/async_reader/mod.rs b/parquet/src/arrow/async_reader/mod.rs
index 5e8bdbc02..b521ac010 100644
--- a/parquet/src/arrow/async_reader/mod.rs
+++ b/parquet/src/arrow/async_reader/mod.rs
@@ -1556,13 +1556,16 @@ mod tests {
             index_reader::read_offset_indexes(&data, metadata.row_group(0).columns())
                 .expect("reading offset index");
 
-        let row_group_meta = metadata.row_group(0).clone();
-        let metadata = ParquetMetaData::new_with_page_index(
-            metadata.file_metadata().clone(),
-            vec![row_group_meta],
-            None,
-            Some(vec![offset_index.clone()]),
-        );
+        let mut metadata_builder = metadata.into_builder();
+        let mut row_groups = metadata_builder.take_row_groups();
+        row_groups.truncate(1);
+        let row_group_meta = row_groups.pop().unwrap();
+
+        let metadata = metadata_builder
+            .add_row_group(row_group_meta)
+            .set_column_index(None)
+            .set_offset_index(Some(vec![offset_index.clone()]))
+            .build();
 
         let metadata = Arc::new(metadata);
 
diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs
index 05bc261a6..30b17b6a2 100644
--- a/parquet/src/file/metadata/mod.rs
+++ b/parquet/src/file/metadata/mod.rs
@@ -156,6 +156,7 @@ pub type ParquetOffsetIndex = Vec<Vec<OffsetIndexMetaData>>;
 /// defined by [`parquet.thrift`].
 ///
 /// # Overview
+/// The fields of this structure are:
 /// * [`FileMetaData`]: Information about the overall file (such as the schema) (See [`Self::file_metadata`])
 /// * [`RowGroupMetaData`]: Information about each Row Group (see [`Self::row_groups`])
 /// * [`ParquetColumnIndex`] and [`ParquetOffsetIndex`]: Optional "Page Index" structures (see [`Self::column_index`] and [`Self::offset_index`])
@@ -163,6 +164,8 @@ pub type ParquetOffsetIndex = Vec<Vec<OffsetIndexMetaData>>;
 /// This structure is read by the various readers in this crate or can be read
 /// directly from a file using the [`ParquetMetaDataReader`] struct.
 ///
+/// See the [`ParquetMetaDataBuilder`] to create and modify this structure.
+///
 /// [`parquet.thrift`]: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift
 #[derive(Debug, Clone, PartialEq)]
 pub struct ParquetMetaData {
@@ -190,18 +193,23 @@ impl ParquetMetaData {
 
     /// Creates Parquet metadata from file metadata, a list of row
     /// group metadata, and the column index structures.
+    #[deprecated(note = "Use ParquetMetaDataBuilder")]
     pub fn new_with_page_index(
         file_metadata: FileMetaData,
         row_groups: Vec<RowGroupMetaData>,
         column_index: Option<ParquetColumnIndex>,
         offset_index: Option<ParquetOffsetIndex>,
     ) -> Self {
-        ParquetMetaData {
-            file_metadata,
-            row_groups,
-            column_index,
-            offset_index,
-        }
+        ParquetMetaDataBuilder::new(file_metadata)
+            .set_row_groups(row_groups)
+            .set_column_index(column_index)
+            .set_offset_index(offset_index)
+            .build()
+    }
+
+    /// Convert this ParquetMetaData into a [`ParquetMetaDataBuilder`]
+    pub fn into_builder(self) -> ParquetMetaDataBuilder {
+        self.into()
     }
 
     /// Returns file metadata as reference.
@@ -290,6 +298,127 @@ impl ParquetMetaData {
     }
 }
 
+/// A builder for creating / manipulating [`ParquetMetaData`]
+///
+/// # Example creating a new [`ParquetMetaData`]
+///
+///```no_run
+/// # use parquet::file::metadata::{FileMetaData, ParquetMetaData, ParquetMetaDataBuilder, RowGroupMetaData, RowGroupMetaDataBuilder};
+/// # fn get_file_metadata() -> FileMetaData { unimplemented!(); }
+/// // Create a new builder given the file metadata
+/// let file_metadata = get_file_metadata();
+/// // Create a row group
+/// let row_group = RowGroupMetaData::builder(file_metadata.schema_descr_ptr())
+///    .set_num_rows(100)
+///    // ... (A real row group needs more than just the number of rows)
+///    .build()
+///    .unwrap();
+/// // Create the final metadata
+/// let metadata: ParquetMetaData = ParquetMetaDataBuilder::new(file_metadata)
+///   .add_row_group(row_group)
+///   .build();
+/// ```
+///
+/// # Example modifying an existing [`ParquetMetaData`]
+/// ```no_run
+/// # use parquet::file::metadata::ParquetMetaData;
+/// # fn load_metadata() -> ParquetMetaData { unimplemented!(); }
+/// // Modify the metadata so only the last RowGroup remains
+/// let metadata: ParquetMetaData = load_metadata();
+/// let mut builder = metadata.into_builder();
+///
+/// // Take existing row groups to modify
+/// let mut row_groups = builder.take_row_groups();
+/// let last_row_group = row_groups.pop().unwrap();
+///
+/// let metadata = builder
+///   .add_row_group(last_row_group)
+///   .build();
+/// ```
+pub struct ParquetMetaDataBuilder(ParquetMetaData);
+
+impl ParquetMetaDataBuilder {
+    /// Create a new builder from a file metadata, with no row groups
+    pub fn new(file_meta_data: FileMetaData) -> Self {
+        Self(ParquetMetaData::new(file_meta_data, vec![]))
+    }
+
+    /// Create a new builder from an existing ParquetMetaData
+    pub fn new_from_metadata(metadata: ParquetMetaData) -> Self {
+        Self(metadata)
+    }
+
+    /// Adds a row group to the metadata
+    pub fn add_row_group(mut self, row_group: RowGroupMetaData) -> Self {
+        self.0.row_groups.push(row_group);
+        self
+    }
+
+    /// Sets all the row groups to the specified list
+    pub fn set_row_groups(mut self, row_groups: Vec<RowGroupMetaData>) -> Self {
+        self.0.row_groups = row_groups;
+        self
+    }
+
+    /// Takes ownership of the row groups in this builder, and clears the list
+    /// of row groups.
+    ///
+    /// This can be used for more efficient creation of a new ParquetMetaData
+    /// from an existing one.
+    pub fn take_row_groups(&mut self) -> Vec<RowGroupMetaData> {
+        std::mem::take(&mut self.0.row_groups)
+    }
+
+    /// Return a reference to the current row groups
+    pub fn row_groups(&self) -> &[RowGroupMetaData] {
+        &self.0.row_groups
+    }
+
+    /// Sets the column index
+    pub fn set_column_index(mut self, column_index: Option<ParquetColumnIndex>) -> Self {
+        self.0.column_index = column_index;
+        self
+    }
+
+    /// Returns the current column index from the builder, replacing it with `None`
+    pub fn take_column_index(&mut self) -> Option<ParquetColumnIndex> {
+        std::mem::take(&mut self.0.column_index)
+    }
+
+    /// Return a reference to the current column index, if any
+    pub fn column_index(&self) -> Option<&ParquetColumnIndex> {
+        self.0.column_index.as_ref()
+    }
+
+    /// Sets the offset index
+    pub fn set_offset_index(mut self, offset_index: Option<ParquetOffsetIndex>) -> Self {
+        self.0.offset_index = offset_index;
+        self
+    }
+
+    /// Returns the current offset index from the builder, replacing it with `None`
+    pub fn take_offset_index(&mut self) -> Option<ParquetOffsetIndex> {
+        std::mem::take(&mut self.0.offset_index)
+    }
+
+    /// Return a reference to the current offset index, if any
+    pub fn offset_index(&self) -> Option<&ParquetOffsetIndex> {
+        self.0.offset_index.as_ref()
+    }
+
+    /// Creates a new ParquetMetaData from the builder
+    pub fn build(self) -> ParquetMetaData {
+        let Self(metadata) = self;
+        metadata
+    }
+}
+
+impl From<ParquetMetaData> for ParquetMetaDataBuilder {
+    fn from(meta_data: ParquetMetaData) -> Self {
+        Self(meta_data)
+    }
+}
+
 pub type KeyValue = crate::format::KeyValue;
 
 /// Reference counted pointer for [`FileMetaData`].
@@ -566,12 +695,27 @@ impl RowGroupMetaDataBuilder {
         self
     }
 
+    /// Takes ownership of the the column metadata in this builder, and clears
+    /// the list of columns.
+    ///
+    /// This can be used for more efficient creation of a new RowGroupMetaData
+    /// from an existing one.
+    pub fn take_columns(&mut self) -> Vec<ColumnChunkMetaData> {
+        std::mem::take(&mut self.0.columns)
+    }
+
     /// Sets column metadata for this row group.
     pub fn set_column_metadata(mut self, value: Vec<ColumnChunkMetaData>) -> Self {
         self.0.columns = value;
         self
     }
 
+    /// Adds a column metadata to this row group
+    pub fn add_column_metadata(mut self, value: ColumnChunkMetaData) -> Self {
+        self.0.columns.push(value);
+        self
+    }
+
     /// Sets ordinal for this row group.
     pub fn set_ordinal(mut self, value: i16) -> Self {
         self.0.ordinal = Some(value);
@@ -1672,7 +1816,9 @@ mod tests {
             .unwrap();
         let row_group_meta_with_stats = vec![row_group_meta_with_stats];
 
-        let parquet_meta = ParquetMetaData::new(file_metadata.clone(), row_group_meta_with_stats);
+        let parquet_meta = ParquetMetaDataBuilder::new(file_metadata.clone())
+            .set_row_groups(row_group_meta_with_stats)
+            .build();
         let base_expected_size = 2312;
 
         assert_eq!(parquet_meta.memory_size(), base_expected_size);
@@ -1692,14 +1838,13 @@ mod tests {
         offset_index.append_unencoded_byte_array_data_bytes(Some(10));
         let offset_index = offset_index.build_to_thrift();
 
-        let parquet_meta = ParquetMetaData::new_with_page_index(
-            file_metadata,
-            row_group_meta,
-            Some(vec![vec![Index::BOOLEAN(native_index)]]),
-            Some(vec![vec![
+        let parquet_meta = ParquetMetaDataBuilder::new(file_metadata)
+            .set_row_groups(row_group_meta)
+            .set_column_index(Some(vec![vec![Index::BOOLEAN(native_index)]]))
+            .set_offset_index(Some(vec![vec![
                 OffsetIndexMetaData::try_new(offset_index).unwrap()
-            ]]),
-        );
+            ]]))
+            .build();
 
         let bigger_expected_size = 2816;
         // more set fields means more memory usage
diff --git a/parquet/src/file/serialized_reader.rs b/parquet/src/file/serialized_reader.rs
index b253b73a4..ddb7d0df9 100644
--- a/parquet/src/file/serialized_reader.rs
+++ b/parquet/src/file/serialized_reader.rs
@@ -191,11 +191,13 @@ impl<R: 'static + ChunkReader> SerializedFileReader<R> {
     /// Creates file reader from a Parquet file with read options.
     /// Returns error if Parquet file does not exist or is corrupt.
     pub fn new_with_options(chunk_reader: R, options: ReadOptions) -> Result<Self> {
-        let metadata = ParquetMetaDataReader::new().parse_and_finish(&chunk_reader)?;
+        let mut metadata_builder = ParquetMetaDataReader::new()
+            .parse_and_finish(&chunk_reader)?
+            .into_builder();
         let mut predicates = options.predicates;
-        let row_groups = metadata.row_groups().to_vec();
-        let mut filtered_row_groups = Vec::<RowGroupMetaData>::new();
-        for (i, rg_meta) in row_groups.into_iter().enumerate() {
+
+        // Filter row groups based on the predicates
+        for (i, rg_meta) in metadata_builder.take_row_groups().into_iter().enumerate() {
             let mut keep = true;
             for predicate in &mut predicates {
                 if !predicate(&rg_meta, i) {
@@ -204,7 +206,7 @@ impl<R: 'static + ChunkReader> SerializedFileReader<R> {
                 }
             }
             if keep {
-                filtered_row_groups.push(rg_meta);
+                metadata_builder = metadata_builder.add_row_group(rg_meta);
             }
         }
 
@@ -212,33 +214,22 @@ impl<R: 'static + ChunkReader> SerializedFileReader<R> {
             let mut columns_indexes = vec![];
             let mut offset_indexes = vec![];
 
-            for rg in &mut filtered_row_groups {
+            for rg in metadata_builder.row_groups().iter() {
                 let column_index = index_reader::read_columns_indexes(&chunk_reader, rg.columns())?;
                 let offset_index = index_reader::read_offset_indexes(&chunk_reader, rg.columns())?;
                 columns_indexes.push(column_index);
                 offset_indexes.push(offset_index);
             }
-
-            Ok(Self {
-                chunk_reader: Arc::new(chunk_reader),
-                metadata: Arc::new(ParquetMetaData::new_with_page_index(
-                    metadata.file_metadata().clone(),
-                    filtered_row_groups,
-                    Some(columns_indexes),
-                    Some(offset_indexes),
-                )),
-                props: Arc::new(options.props),
-            })
-        } else {
-            Ok(Self {
-                chunk_reader: Arc::new(chunk_reader),
-                metadata: Arc::new(ParquetMetaData::new(
-                    metadata.file_metadata().clone(),
-                    filtered_row_groups,
-                )),
-                props: Arc::new(options.props),
-            })
+            metadata_builder = metadata_builder
+                .set_column_index(Some(columns_indexes))
+                .set_offset_index(Some(offset_indexes));
         }
+
+        Ok(Self {
+            chunk_reader: Arc::new(chunk_reader),
+            metadata: Arc::new(metadata_builder.build()),
+            props: Arc::new(options.props),
+        })
     }
 }
 
diff --git a/parquet/src/file/statistics.rs b/parquet/src/file/statistics.rs
index 854900f1e..50ed06436 100644
--- a/parquet/src/file/statistics.rs
+++ b/parquet/src/file/statistics.rs
@@ -248,20 +248,21 @@ pub fn from_thrift(
 pub fn to_thrift(stats: Option<&Statistics>) -> Option<TStatistics> {
     let stats = stats?;
 
-    // record null counts if greater than zero.
-    //
-    // TODO: This should be Some(0) if there are no nulls.
-    // see https://github.com/apache/arrow-rs/pull/6216/files
+    // record null count if it can fit in i64
     let null_count = stats
         .null_count_opt()
-        .map(|value| value as i64)
-        .filter(|&x| x > 0);
+        .and_then(|value| i64::try_from(value).ok());
+
+    // record distinct count if it can fit in i64
+    let distinct_count = stats
+        .distinct_count_opt()
+        .and_then(|value| i64::try_from(value).ok());
 
     let mut thrift_stats = TStatistics {
         max: None,
         min: None,
         null_count,
-        distinct_count: stats.distinct_count_opt().map(|value| value as i64),
+        distinct_count,
         max_value: None,
         min_value: None,
         is_max_value_exact: None,
@@ -1052,4 +1053,92 @@ mod tests {
             true,
         ));
     }
+
+    #[test]
+    fn test_count_encoding() {
+        statistics_count_test(None, None);
+        statistics_count_test(Some(0), Some(0));
+        statistics_count_test(Some(100), Some(2000));
+        statistics_count_test(Some(1), None);
+        statistics_count_test(None, Some(1));
+    }
+
+    #[test]
+    fn test_count_encoding_distinct_too_large() {
+        // statistics are stored using i64, so test trying to store larger values
+        let statistics = make_bool_stats(Some(u64::MAX), Some(100));
+        let thrift_stats = to_thrift(Some(&statistics)).unwrap();
+        assert_eq!(thrift_stats.distinct_count, None); // can't store u64 max --> null
+        assert_eq!(thrift_stats.null_count, Some(100));
+    }
+
+    #[test]
+    fn test_count_encoding_null_too_large() {
+        // statistics are stored using i64, so test trying to store larger values
+        let statistics = make_bool_stats(Some(100), Some(u64::MAX));
+        let thrift_stats = to_thrift(Some(&statistics)).unwrap();
+        assert_eq!(thrift_stats.distinct_count, Some(100));
+        assert_eq!(thrift_stats.null_count, None); // can' store u64 max --> null
+    }
+
+    #[test]
+    fn test_count_decoding_null_invalid() {
+        let tstatistics = TStatistics {
+            null_count: Some(-42),
+            ..Default::default()
+        };
+        let err = from_thrift(Type::BOOLEAN, Some(tstatistics)).unwrap_err();
+        assert_eq!(
+            err.to_string(),
+            "Parquet error: Statistics null count is negative -42"
+        );
+    }
+
+    /// Writes statistics to thrift and reads them back and ensures:
+    /// - The statistics are the same
+    /// - The statistics written to thrift are the same as the original statistics
+    fn statistics_count_test(distinct_count: Option<u64>, null_count: Option<u64>) {
+        let statistics = make_bool_stats(distinct_count, null_count);
+
+        let thrift_stats = to_thrift(Some(&statistics)).unwrap();
+        assert_eq!(thrift_stats.null_count.map(|c| c as u64), null_count);
+        assert_eq!(
+            thrift_stats.distinct_count.map(|c| c as u64),
+            distinct_count
+        );
+
+        let round_tripped = from_thrift(Type::BOOLEAN, Some(thrift_stats))
+            .unwrap()
+            .unwrap();
+        // TODO: remove branch when we no longer support assuming null_count==None in the thrift
+        // means null_count = Some(0)
+        if null_count.is_none() {
+            assert_ne!(round_tripped, statistics);
+            assert!(round_tripped.null_count_opt().is_some());
+            assert_eq!(round_tripped.null_count_opt(), Some(0));
+            assert_eq!(round_tripped.min_bytes_opt(), statistics.min_bytes_opt());
+            assert_eq!(round_tripped.max_bytes_opt(), statistics.max_bytes_opt());
+            assert_eq!(
+                round_tripped.distinct_count_opt(),
+                statistics.distinct_count_opt()
+            );
+        } else {
+            assert_eq!(round_tripped, statistics);
+        }
+    }
+
+    fn make_bool_stats(distinct_count: Option<u64>, null_count: Option<u64>) -> Statistics {
+        let min = Some(true);
+        let max = Some(false);
+        let is_min_max_deprecated = false;
+
+        // test is about the counts, so we aren't really testing the min/max values
+        Statistics::Boolean(ValueStatistics::new(
+            min,
+            max,
+            distinct_count,
+            null_count,
+            is_min_max_deprecated,
+        ))
+    }
 }


Test patch:
diff --git a/parquet/tests/arrow_writer_layout.rs b/parquet/tests/arrow_writer_layout.rs
index 3e0f6ce3a..9a66d13f8 100644
--- a/parquet/tests/arrow_writer_layout.rs
+++ b/parquet/tests/arrow_writer_layout.rs
@@ -189,7 +189,7 @@ fn test_primitive() {
                     pages: (0..8)
                         .map(|_| Page {
                             rows: 250,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 1000,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,
@@ -218,14 +218,14 @@ fn test_primitive() {
                     pages: vec![
                         Page {
                             rows: 250,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 258,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 1750,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 7000,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,
@@ -233,7 +233,7 @@ fn test_primitive() {
                     ],
                     dictionary_page: Some(Page {
                         rows: 250,
-                        page_header_size: 36,
+                        page_header_size: 38,
                         compressed_size: 1000,
                         encoding: Encoding::PLAIN,
                         page_type: PageType::DICTIONARY_PAGE,
@@ -260,42 +260,42 @@ fn test_primitive() {
                     pages: vec![
                         Page {
                             rows: 400,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 452,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 370,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 472,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 330,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 464,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 330,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 464,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 330,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 464,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 240,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 332,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
@@ -303,7 +303,7 @@ fn test_primitive() {
                     ],
                     dictionary_page: Some(Page {
                         rows: 2000,
-                        page_header_size: 36,
+                        page_header_size: 38,
                         compressed_size: 8000,
                         encoding: Encoding::PLAIN,
                         page_type: PageType::DICTIONARY_PAGE,
@@ -329,7 +329,7 @@ fn test_primitive() {
                     pages: (0..20)
                         .map(|_| Page {
                             rows: 100,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 400,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,
@@ -364,14 +364,14 @@ fn test_string() {
                     pages: (0..15)
                         .map(|_| Page {
                             rows: 130,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 1040,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,
                         })
                         .chain(std::iter::once(Page {
                             rows: 50,
-                            page_header_size: 35,
+                            page_header_size: 37,
                             compressed_size: 400,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,
@@ -400,21 +400,21 @@ fn test_string() {
                     pages: vec![
                         Page {
                             rows: 130,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 138,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 1250,
-                            page_header_size: 38,
+                            page_header_size: 40,
                             compressed_size: 10000,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 620,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 4960,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,
@@ -422,7 +422,7 @@ fn test_string() {
                     ],
                     dictionary_page: Some(Page {
                         rows: 130,
-                        page_header_size: 36,
+                        page_header_size: 38,
                         compressed_size: 1040,
                         encoding: Encoding::PLAIN,
                         page_type: PageType::DICTIONARY_PAGE,
@@ -449,42 +449,42 @@ fn test_string() {
                     pages: vec![
                         Page {
                             rows: 400,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 452,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 370,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 472,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 330,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 464,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 330,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 464,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 330,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 464,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
                         },
                         Page {
                             rows: 240,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 332,
                             encoding: Encoding::RLE_DICTIONARY,
                             page_type: PageType::DATA_PAGE,
@@ -492,7 +492,7 @@ fn test_string() {
                     ],
                     dictionary_page: Some(Page {
                         rows: 2000,
-                        page_header_size: 36,
+                        page_header_size: 38,
                         compressed_size: 16000,
                         encoding: Encoding::PLAIN,
                         page_type: PageType::DICTIONARY_PAGE,
@@ -532,7 +532,7 @@ fn test_list() {
                     pages: (0..10)
                         .map(|_| Page {
                             rows: 20,
-                            page_header_size: 36,
+                            page_header_size: 38,
                             compressed_size: 672,
                             encoding: Encoding::PLAIN,
                             page_type: PageType::DATA_PAGE,