Repository: parquet-go/parquet-go (go)

Issue:
disable min/max page values per column
We're writing html files to parquet. Right now, page statistics are being written with the "min" and "max" values, but that isn't useful for a block of multiple megabytes of html, but it does bloat the file size.

There's a similar ticket for Java.
https://issues.apache.org/jira/browse/PARQUET-1911

My thought would be to add a new struct tag option, `nostats`, `nobounds`, `skipbounds` or something like that, and skip the min/max calculations. Would you be opposed to supporting that?

Pull request:
add SkipBounds WriterOption to omit min/max bounds
Fixes #145 

Patch:
diff --git a/README.md b/README.md
index 7ac9553..f0f0d8f 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ dependency on and install with the following command:
 go get github.com/parquet-go/parquet-go
 ```
 
-Go 1.20 or later is required to use the package.
+Go 1.21 or later is required to use the package.
 
 ### Compatibility Guarantees
 
diff --git a/config.go b/config.go
index 84fd80e..63922d8 100644
--- a/config.go
+++ b/config.go
@@ -213,6 +213,7 @@ type WriterConfig struct {
 	BloomFilters         []BloomFilterColumn
 	Compression          compress.Codec
 	Sorting              SortingConfig
+	SkipPageBounds       [][]string
 }
 
 // DefaultWriterConfig returns a new WriterConfig value initialized with the
@@ -623,6 +624,15 @@ func SortingWriterConfig(options ...SortingOption) WriterOption {
 	return writerOption(func(config *WriterConfig) { config.Sorting.Apply(options...) })
 }
 
+// SkipPageBounds lists the path to a column that shouldn't have bounds written to the
+// footer of the parquet file. This is useful for data blobs, like a raw html file,
+// where the bounds are not meaningful.
+//
+// This option is additive, it may be used multiple times to skip multiple columns.
+func SkipPageBounds(path ...string) WriterOption {
+	return writerOption(func(config *WriterConfig) { config.SkipPageBounds = append(config.SkipPageBounds, path) })
+}
+
 // ColumnBufferCapacity creates a configuration option which defines the size of
 // row group column buffers.
 //
diff --git a/go.mod b/go.mod
index 63ca820..88b22fc 100644
--- a/go.mod
+++ b/go.mod
@@ -1,23 +1,24 @@
 module github.com/parquet-go/parquet-go
 
-go 1.20
+go 1.21
 
 require (
-	github.com/andybalholm/brotli v1.0.5
-	github.com/google/uuid v1.3.0
+	github.com/andybalholm/brotli v1.1.0
+	github.com/google/uuid v1.6.0
 	github.com/hexops/gotextdiff v1.0.3
-	github.com/klauspost/compress v1.17.8
+	github.com/klauspost/compress v1.17.9
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/pierrec/lz4/v4 v4.1.18
-	github.com/segmentio/encoding v0.3.6
-	github.com/stretchr/testify v1.8.4
-	golang.org/x/sys v0.10.0
-	google.golang.org/protobuf v1.33.0
+	github.com/pierrec/lz4/v4 v4.1.21
+	github.com/segmentio/encoding v0.4.0
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/sys v0.21.0
+	google.golang.org/protobuf v1.34.2
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
index bc72272..d135b9c 100644
--- a/go.sum
+++ b/go.sum
@@ -1,33 +1,56 @@
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
 github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
 github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCRwZQ=
 github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM=
+github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOVmy8=
+github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/writer.go b/writer.go
index 5ef301a..c02f4e2 100644
--- a/writer.go
+++ b/writer.go
@@ -11,6 +11,7 @@ import (
 	"math/bits"
 	"os"
 	"reflect"
+	"slices"
 	"sort"
 
 	"github.com/parquet-go/parquet-go/compress"
@@ -604,7 +605,10 @@ func newWriter(output io.Writer, config *WriterConfig) *writer {
 			bufferIndex:        int32(leaf.columnIndex),
 			bufferSize:         int32(float64(config.PageBufferSize) * 0.98),
 			writePageStats:     config.DataPageStatistics,
-			encodings:          make([]format.Encoding, 0, 3),
+			writePageBounds: !slices.ContainsFunc(config.SkipPageBounds, func(skip []string) bool {
+				return columnPath(skip).equal(leaf.path)
+			}),
+			encodings: make([]format.Encoding, 0, 3),
 			// Data pages in version 2 can omit compression when dictionary
 			// encoding is employed; only the dictionary page needs to be
 			// compressed, the data pages are encoded with the hybrid
@@ -1145,13 +1149,14 @@ type writerColumn struct {
 		encoder  thrift.Encoder
 	}
 
-	filter         []byte
-	numRows        int64
-	bufferIndex    int32
-	bufferSize     int32
-	writePageStats bool
-	isCompressed   bool
-	encodings      []format.Encoding
+	filter          []byte
+	numRows         int64
+	bufferIndex     int32
+	bufferSize      int32
+	writePageStats  bool
+	writePageBounds bool
+	isCompressed    bool
+	encodings       []format.Encoding
 
 	columnChunk *format.ColumnChunk
 	offsetIndex *format.OffsetIndex
@@ -1573,7 +1578,13 @@ func (c *writerColumn) recordPageStats(headerSize int32, header *format.PageHead
 	if page != nil {
 		numNulls := page.NumNulls()
 		numValues := page.NumValues()
-		minValue, maxValue, pageHasBounds := page.Bounds()
+
+		var minValue, maxValue Value
+		var pageHasBounds bool
+		if c.writePageBounds {
+			minValue, maxValue, pageHasBounds = page.Bounds()
+		}
+
 		c.columnIndex.IndexPage(numValues, numNulls, minValue, maxValue)
 		c.columnChunk.MetaData.NumValues += numValues
 		c.columnChunk.MetaData.Statistics.NullCount += numNulls


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6e881e1..b94a851 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         go:
-          - "1.20.x"
+          - "1.21.x"
         tags:
           - ""
           - purego
@@ -71,7 +71,7 @@ jobs:
       - name: Setup Go ${{ matrix.go }}
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: Validate formatting
         run: make format
@@ -84,7 +84,7 @@ jobs:
 
   #     - uses: actions/setup-go@v3
   #       with:
-  #         go-version: 1.20.x
+  #         go-version: 1.21.x
 
   #     - name: golangci-lint
   #       uses: golangci/golangci-lint-action@v3
diff --git a/writer_test.go b/writer_test.go
index 3c3b085..2bc961e 100644
--- a/writer_test.go
+++ b/writer_test.go
@@ -1239,3 +1239,37 @@ func TestColumnMaxValueAndMinValue(t *testing.T) {
 		t.Fatalf("wrong max value of row groups in parquet file: want=99 got=%s", string(statistics.MaxValue))
 	}
 }
+
+func TestColumnSkipPageBounds(t *testing.T) {
+	type testStruct struct {
+		A int `parquet:"a,plain"`
+	}
+
+	tests := make([]testStruct, 100)
+	for i := 0; i < 100; i++ {
+		tests[i] = testStruct{A: i + 1}
+	}
+	schema := parquet.SchemaOf(&testStruct{})
+	b := bytes.NewBuffer(nil)
+	config := parquet.DefaultWriterConfig()
+	config.PageBufferSize = 256
+	w := parquet.NewGenericWriter[testStruct](b, schema, config, parquet.SkipPageBounds("a"))
+	_, _ = w.Write(tests[0:50])
+	_, _ = w.Write(tests[50:100])
+	_ = w.Close()
+
+	f, err := parquet.OpenFile(bytes.NewReader(b.Bytes()), int64(b.Len()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(f.RowGroups()) != 1 {
+		t.Fatalf("wrong number of row groups in parquet file: want=1 got=%d", f.RowGroups())
+	}
+	statistics := f.Metadata().RowGroups[0].Columns[0].MetaData.Statistics
+	if string(statistics.MinValue) != "" {
+		t.Fatalf("wrong min value of row groups in parquet file: want='' got=%s", string(statistics.MinValue))
+	}
+	if string(statistics.MaxValue) != "" {
+		t.Fatalf("wrong max value of row groups in parquet file: want='' got=%s", string(statistics.MaxValue))
+	}
+}