Repository: segmentio/parquet-go
Pull request #202 (2022-06-01 16:10:16)

remove unused WriterConfig.PageBufferPool field


Patch:
diff --git a/config.go b/config.go
index 9546dbd..079ec9a 100644
--- a/config.go
+++ b/config.go
@@ -136,7 +136,6 @@ type WriterConfig struct {
 	CreatedBy            string
 	ColumnPageBuffers    PageBufferPool
 	ColumnIndexSizeLimit int
-	PageBufferPool       PageBufferPool
 	PageBufferSize       int
 	WriteBufferSize      int
 	DataPageVersion      int
diff --git a/go.mod b/go.mod
index c149187..690e0d3 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/andybalholm/brotli v1.0.3
 	github.com/google/uuid v1.3.0
 	github.com/hexops/gotextdiff v1.0.3
-	github.com/klauspost/compress v1.15.2
+	github.com/klauspost/compress v1.15.5
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pierrec/lz4/v4 v4.1.9
 	github.com/segmentio/encoding v0.3.5
diff --git a/go.sum b/go.sum
index e5d4daa..dd857c4 100644
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw=
-github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.5 h1:qyCLMz2JCrKADihKOh9FxnW3houKeNsp2h5OEz0QSEA=
+github.com/klauspost/compress v1.15.5/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
diff --git a/schema.go b/schema.go
index e985f7b..d1b3b01 100644
--- a/schema.go
+++ b/schema.go
@@ -69,6 +69,10 @@ type Schema struct {
 // Invalid combination of struct tags and Go types, or repeating options will
 // cause the function to panic.
 //
+// As a special case, if the field tag is "-", the field is omitted from the schema
+// and the data will not be written into the parquet file(s).
+// Note that a field with name "-" can still be generated using the tag "-,".
+//
 // The schema name is the Go type name of the value.
 func SchemaOf(model interface{}) *Schema {
 	return schemaOf(dereference(reflect.TypeOf(model)))
@@ -314,10 +318,17 @@ func structFieldsOf(t reflect.Type) []reflect.StructField {
 
 func appendStructFields(t reflect.Type, fields []reflect.StructField, index []int, offset uintptr) []reflect.StructField {
 	for i, n := 0, t.NumField(); i < n; i++ {
+		f := t.Field(i)
+		if tag := f.Tag.Get("parquet"); tag != "" {
+			name, _ := split(tag)
+			if tag != "-," && name == "-" {
+				continue
+			}
+		}
+
 		fieldIndex := index[:len(index):len(index)]
 		fieldIndex = append(fieldIndex, i)
 
-		f := t.Field(i)
 		f.Offset += offset
 
 		if f.Anonymous {
@@ -391,7 +402,6 @@ func (f *structField) Value(base reflect.Value) reflect.Value {
 			return fieldByIndex(base, f.index)
 		}
 	}
-	return reflect.Value{}
 }
 
 func structFieldString(f reflect.StructField) string {


Test patch:
diff --git a/schema_test.go b/schema_test.go
index 50cb976..3a3231e 100644
--- a/schema_test.go
+++ b/schema_test.go
@@ -52,6 +52,35 @@ func TestSchemaOf(t *testing.T) {
 		required binary first_name (STRING);
 		required binary last_name (STRING);
 	}
+}`,
+		},
+
+		{
+			value: new(struct {
+				Inner struct {
+					FirstName          string `parquet:"first_name"`
+					ShouldNotBePresent string `parquet:"-"`
+				} `parquet:"inner,optional"`
+			}),
+			print: `message {
+	optional group inner {
+		required binary first_name (STRING);
+	}
+}`,
+		},
+
+		{
+			value: new(struct {
+				Inner struct {
+					FirstName    string `parquet:"first_name"`
+					MyNameIsDash string `parquet:"-,"`
+				} `parquet:"inner,optional"`
+			}),
+			print: `message {
+	optional group inner {
+		required binary first_name (STRING);
+		required binary - (STRING);
+	}
 }`,
 		},
 	}
diff --git a/writer_test.go b/writer_test.go
index 2fc53b0..7ea7e45 100644
--- a/writer_test.go
+++ b/writer_test.go
@@ -78,6 +78,13 @@ type timeseries struct {
 	Value     float64 `parquet:"value"`
 }
 
+type event struct {
+	Name     string  `parquet:"name,dict"`
+	Type     string  `parquet:"-"`
+	Value    float64 `parquet:"value"`
+	Category string  `parquet:"-"`
+}
+
 var writerTests = []struct {
 	scenario string
 	version  int
@@ -277,10 +284,10 @@ value 10: R:0 D:0 V:10.0
 		dump: `row group 0
 --------------------------------------------------------------------------------
 owner:              BINARY ZSTD DO:0 FPO:4 SZ:66/57/0.86 VC:2 ENC:DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
-ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:70 SZ:162/112/0.69 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:70 SZ:162/112/0.69 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 contacts:
-.name:              BINARY UNCOMPRESSED DO:0 FPO:232 SZ:116/116/1.00 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
-.phoneNumber:       BINARY ZSTD DO:0 FPO:348 SZ:113/95/0.84 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+.name:              BINARY UNCOMPRESSED DO:0 FPO:232 SZ:116/116/1.00 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
+.phoneNumber:       BINARY ZSTD DO:0 FPO:348 SZ:113/95/0.84 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 
     owner TV=2 RL=0 DL=0
     ----------------------------------------------------------------------------
@@ -359,10 +366,10 @@ value 3: R:0 D:0 V:<null>
 		dump: `row group 0
 --------------------------------------------------------------------------------
 owner:              BINARY ZSTD DO:0 FPO:4 SZ:71/62/0.87 VC:2 ENC:DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
-ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:75 SZ:156/106/0.68 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:75 SZ:156/106/0.68 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 contacts:
-.name:              BINARY UNCOMPRESSED DO:0 FPO:231 SZ:110/110/1.00 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
-.phoneNumber:       BINARY ZSTD DO:0 FPO:341 SZ:108/90/0.83 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+.name:              BINARY UNCOMPRESSED DO:0 FPO:231 SZ:110/110/1.00 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
+.phoneNumber:       BINARY ZSTD DO:0 FPO:341 SZ:108/90/0.83 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 
     owner TV=2 RL=0 DL=0
     ----------------------------------------------------------------------------
@@ -409,6 +416,40 @@ BINARY contacts.phoneNumber
 value 1: R:0 D:2 V:555 987 6543
 value 2: R:1 D:1 V:<null>
 value 3: R:0 D:0 V:<null>
+`,
+	},
+
+	{
+		scenario: "omit `-` fields",
+		version:  v1,
+		rows: []interface{}{
+			&event{Name: "customer1", Type: "request", Value: 42.0},
+			&event{Name: "customer2", Type: "access", Value: 1.0},
+		},
+		dump: `row group 0
+--------------------------------------------------------------------------------
+name:   BINARY UNCOMPRESSED DO:4 FPO:49 SZ:73/73/1.00 VC:2 ENC:PLAIN,RLE_DICTIONARY ST:[no stats for this column]
+value:  DOUBLE UNCOMPRESSED DO:0 FPO:77 SZ:39/39/1.00 VC:2 ENC:PLAIN ST:[no stats for this column]
+
+    name TV=2 RL=0 DL=0 DS: 2 DE:PLAIN
+    ----------------------------------------------------------------------------
+    page 0:                  DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[no stats for this column] CRC:[verified] SZ:5 VC:2
+
+    value TV=2 RL=0 DL=0
+    ----------------------------------------------------------------------------
+    page 0:                  DLE:RLE RLE:RLE VLE:PLAIN ST:[no stats for this column] CRC:[verified] SZ:16 VC:2
+
+BINARY name
+--------------------------------------------------------------------------------
+*** row group 1 of 1, values 1 to 2 ***
+value 1: R:0 D:0 V:customer1
+value 2: R:0 D:0 V:customer2
+
+DOUBLE value
+--------------------------------------------------------------------------------
+*** row group 1 of 1, values 1 to 2 ***
+value 1: R:0 D:0 V:42.0
+value 2: R:0 D:0 V:1.0
 `,
 	},
 }