Repository: segmentio/parquet-go
Pull request #448 (2022-12-08 22:56:12)

writer,bloom: make filter bits per value configurable
Previously we would use 10 bits per value for all bloom filters.
However, in our quest for more performance, we may try to tweak these
values as appropriate for the dataset being optimized.

Add a bitsPerValue argument to SplitBlockFilter so that callers are
required to pass this value, and remove bitsPerValue from the
BloomFilterColumn interface.

To update existing code to work with the new API with no change in
functionality, update:

    SplitBlockFilter("first_name")

to:

    SplitBlockFilter(10, "first_name")


Patch:
diff --git a/README.md b/README.md
index 4e76b36..be2976a 100644
--- a/README.md
+++ b/README.md
@@ -330,13 +330,14 @@ type RowType struct {
     LastName  string `parquet:"last_name"`
 }
 
+const filterBitsPerValue = 10
 writer := parquet.NewGenericWriter[RowType](output,
     parquet.BloomFilters(
         // Configures the write to generate split-block bloom filters for the
         // "first_name" and "last_name" columns of the parquet schema of rows
         // witten by the application.
-        parquet.SplitBlockFilter("first_name"),
-        parquet.SplitBlockFilter("last_name"),
+        parquet.SplitBlockFilter(filterBitsPerValue, "first_name"),
+        parquet.SplitBlockFilter(filterBitsPerValue, "last_name"),
     ),
 )
 ...
diff --git a/bloom.go b/bloom.go
index 4ea94ff..8d4509a 100644
--- a/bloom.go
+++ b/bloom.go
@@ -85,20 +85,35 @@ type BloomFilterColumn interface {
 
 	// Returns the size of the filter needed to encode values in the filter,
 	// assuming each value will be encoded with the given number of bits.
-	Size(numValues int64, bitsPerValue uint) int
+	Size(numValues int64) int
 }
 
 // SplitBlockFilter constructs a split block bloom filter object for the column
-// at the given path.
-func SplitBlockFilter(path ...string) BloomFilterColumn { return splitBlockFilter(path) }
+// at the given path, with the given bitsPerValue.
+//
+// If you are unsure what number of bitsPerValue to use, 10 is a reasonable
+// tradeoff between size and error rate for common datasets.
+//
+// For more information on the tradeoff between size and error rate, consult
+// this website: https://hur.st/bloomfilter/?n=4000&p=0.1&m=&k=1
+func SplitBlockFilter(bitsPerValue uint, path ...string) BloomFilterColumn {
+	return splitBlockFilter{
+		bitsPerValue: bitsPerValue,
+		path:         path,
+	}
+}
 
-type splitBlockFilter []string
+type splitBlockFilter struct {
+	bitsPerValue uint
+	path         []string
+}
 
-func (f splitBlockFilter) Path() []string              { return f }
+func (f splitBlockFilter) Path() []string              { return f.path }
 func (f splitBlockFilter) Hash() bloom.Hash            { return bloom.XXH64{} }
 func (f splitBlockFilter) Encoding() encoding.Encoding { return splitBlockEncoding{} }
-func (f splitBlockFilter) Size(numValues int64, bitsPerValue uint) int {
-	return bloom.BlockSize * bloom.NumSplitBlocksOf(numValues, bitsPerValue)
+
+func (f splitBlockFilter) Size(numValues int64) int {
+	return bloom.BlockSize * bloom.NumSplitBlocksOf(numValues, f.bitsPerValue)
 }
 
 // Creates a header from the given bloom filter.
diff --git a/writer.go b/writer.go
index 3da523d..395ce30 100644
--- a/writer.go
+++ b/writer.go
@@ -1049,8 +1049,7 @@ func (c *writerColumn) flushFilterPages() error {
 }
 
 func (c *writerColumn) resizeBloomFilter(numValues int64) {
-	const bitsPerValue = 10 // TODO: make this configurable
-	filterSize := c.columnFilter.Size(numValues, bitsPerValue)
+	filterSize := c.columnFilter.Size(numValues)
 	if cap(c.filter) < filterSize {
 		c.filter = make([]byte, filterSize)
 	} else {


Test patch:
diff --git a/bloom_test.go b/bloom_test.go
index 978e68c..90494db 100644
--- a/bloom_test.go
+++ b/bloom_test.go
@@ -15,7 +15,7 @@ func TestSplitBlockFilter(t *testing.T) {
 		return make(bloom.SplitBlockFilter, bloom.NumSplitBlocksOf(int64(numValues), 11))
 	}
 
-	enc := SplitBlockFilter("$").Encoding()
+	enc := SplitBlockFilter(10, "$").Encoding()
 
 	check := func(filter bloom.SplitBlockFilter, value Value) bool {
 		return filter.Check(value.hash(&bloom.XXH64{}))
@@ -157,7 +157,7 @@ func TestSplitBlockFilter(t *testing.T) {
 func BenchmarkSplitBlockFilter(b *testing.B) {
 	const N = 1000
 	f := make(bloom.SplitBlockFilter, bloom.NumSplitBlocksOf(N, 10)).Bytes()
-	e := SplitBlockFilter("$").Encoding()
+	e := SplitBlockFilter(10, "$").Encoding()
 
 	v := make([]int64, N)
 	r := rand.NewSource(10)
diff --git a/buffer_test.go b/buffer_test.go
index 9015f78..3cca6c9 100644
--- a/buffer_test.go
+++ b/buffer_test.go
@@ -370,9 +370,9 @@ func TestBufferGenerateBloomFilters(t *testing.T) {
 		buffer := parquet.NewBuffer()
 		writer := parquet.NewWriter(output,
 			parquet.BloomFilters(
-				parquet.SplitBlockFilter("X"),
-				parquet.SplitBlockFilter("Y"),
-				parquet.SplitBlockFilter("Z"),
+				parquet.SplitBlockFilter(10, "X"),
+				parquet.SplitBlockFilter(10, "Y"),
+				parquet.SplitBlockFilter(10, "Z"),
 			),
 		)
 		for i := range rows {
diff --git a/writer_test.go b/writer_test.go
index 2f4afcb..4b6b482 100644
--- a/writer_test.go
+++ b/writer_test.go
@@ -520,7 +520,7 @@ func TestWriterGenerateBloomFilters(t *testing.T) {
 		buffer := new(bytes.Buffer)
 		writer := parquet.NewWriter(buffer,
 			parquet.BloomFilters(
-				parquet.SplitBlockFilter("last_name"),
+				parquet.SplitBlockFilter(10, "last_name"),
 			),
 		)
 		for i := range rows {
@@ -584,7 +584,7 @@ func TestBloomFilterForDict(t *testing.T) {
 	w := parquet.NewWriter(
 		b,
 		schema,
-		parquet.BloomFilters(parquet.SplitBlockFilter("a")),
+		parquet.BloomFilters(parquet.SplitBlockFilter(10, "a")),
 	)
 
 	err := w.Write(&testStruct{A: "test"})
@@ -655,7 +655,7 @@ func TestWriterResetWithBloomFilters(t *testing.T) {
 
 	writer := parquet.NewWriter(new(bytes.Buffer),
 		parquet.BloomFilters(
-			parquet.SplitBlockFilter("value"),
+			parquet.SplitBlockFilter(10, "value"),
 		),
 	)