Repository: segmentio/parquet-go (go)

Issue:
How to ignore a field when write parquet to file
Just want to ask is there any way to ignore a field when write to parquet file like what encoding/json package did when marshal/unmarshal:
```
// Field is ignored by this package.
Field int `json:"-"`

// Field appears in JSON as key "-".
Field int `json:"-,"`
```

ref: https://pkg.go.dev/encoding/json#Marshal

P/s: Thank, this project is great. 

Pull request:
Add the ability to omit fields in schema.
Fixes #185 

Also fixing some other failing tests.

Patch:
diff --git a/dictionary.go b/dictionary.go
index 5214c3a..fd852f5 100644
--- a/dictionary.go
+++ b/dictionary.go
@@ -1034,17 +1034,22 @@ type indexedPageValues struct {
 }
 
 func (r *indexedPageValues) ReadValues(values []Value) (n int, err error) {
-	var v Value
-	for n < len(values) && r.offset < len(r.page.values) {
-		v = r.page.typ.dict.Index(r.page.values[r.offset])
-		v.columnIndex = r.page.columnIndex
+	dict := r.page.typ.dict
+	pageValues := r.page.values
+	columnIndex := r.page.columnIndex
+
+	for n < len(values) && r.offset < len(pageValues) {
+		v := dict.Index(pageValues[r.offset])
+		v.columnIndex = columnIndex
 		values[n] = v
 		r.offset++
 		n++
 	}
-	if r.offset == len(r.page.values) {
+
+	if r.offset == len(pageValues) {
 		err = io.EOF
 	}
+
 	return n, err
 }
 
diff --git a/encoding.go b/encoding.go
index 580432e..40ce70f 100644
--- a/encoding.go
+++ b/encoding.go
@@ -84,8 +84,11 @@ var (
 )
 
 func isDictionaryEncoding(encoding encoding.Encoding) bool {
-	enc := encoding.Encoding()
-	return enc == format.PlainDictionary || enc == format.RLEDictionary
+	return isDictionaryFormat(encoding.Encoding())
+}
+
+func isDictionaryFormat(encoding format.Encoding) bool {
+	return encoding == format.PlainDictionary || encoding == format.RLEDictionary
 }
 
 // LookupEncoding returns the parquet encoding associated with the given code.
diff --git a/file.go b/file.go
index 7dd667f..eff129b 100644
--- a/file.go
+++ b/file.go
@@ -467,83 +467,23 @@ func (f *filePages) ReadPage() (Page, error) {
 		if err := f.decoder.Decode(header); err != nil {
 			return nil, err
 		}
-
-		if cap(f.dataPage.data) < int(header.CompressedPageSize) {
-			f.dataPage.data = make([]byte, header.CompressedPageSize)
-		} else {
-			f.dataPage.data = f.dataPage.data[:header.CompressedPageSize]
-		}
-
-		if cap(f.dataPage.values) < int(header.UncompressedPageSize) {
-			f.dataPage.values = make([]byte, 0, header.UncompressedPageSize)
-		}
-
-		if _, err := io.ReadFull(f.rbuf, f.dataPage.data); err != nil {
+		if err := f.readPage(header, f.dataPage, f.rbuf); err != nil {
 			return nil, err
 		}
 
-		if header.CRC != 0 {
-			headerChecksum := uint32(header.CRC)
-			bufferChecksum := crc32.ChecksumIEEE(f.dataPage.data)
-
-			if headerChecksum != bufferChecksum {
-				// The parquet specs indicate that corruption errors could be
-				// handled gracefully by skipping pages, tho this may not always
-				// be practical. Depending on how the pages are consumed,
-				// missing rows may cause unpredictable behaviors in algorithms.
-				//
-				// For now, we assume these errors to be fatal, but we may
-				// revisit later and improve error handling to be more resilient
-				// to data corruption.
-				return nil, fmt.Errorf("crc32 checksum mismatch in page %d of column %q: 0x%08X != 0x%08X: %w",
-					f.index,
-					f.columnPath(),
-					headerChecksum,
-					bufferChecksum,
-					ErrCorrupted,
-				)
-			}
-		}
-
-		var column = f.chunk.column
 		var page Page
 		var err error
 
 		switch header.Type {
 		case format.DataPageV2:
-			if header.DataPageHeaderV2 == nil {
-				err = ErrMissingPageHeader
-			} else {
-				page, err = column.decodeDataPageV2(DataPageHeaderV2{header.DataPageHeaderV2}, f.dataPage)
-			}
-
+			page, err = f.readDataPageV2(header)
 		case format.DataPage:
-			if header.DataPageHeader == nil {
-				err = ErrMissingPageHeader
-			} else {
-				page, err = column.decodeDataPageV1(DataPageHeaderV1{header.DataPageHeader}, f.dataPage)
-			}
-
+			page, err = f.readDataPageV1(header)
 		case format.DictionaryPage:
 			// Sometimes parquet files do not have the dictionary page offset
 			// recorded in the column metadata. We account for this by lazily
-			// checking whether the first page is a dictionary page.
-			if header.DictionaryPageHeader == nil {
-				err = ErrMissingPageHeader
-			} else if f.index > 0 {
-				err = ErrUnexpectedDictionaryPage
-			} else {
-				f.dictPage, _ = dictPagePool.Get().(*dictPage)
-				if f.dictPage == nil {
-					f.dictPage = new(dictPage)
-				}
-				f.dataPage.dictionary, err = column.decodeDictionary(
-					DictionaryPageHeader{header.DictionaryPageHeader},
-					f.dataPage,
-					f.dictPage,
-				)
-			}
-
+			// reading dictionary pages when we encounter them.
+			err = f.readDictionaryPage(header, f.dataPage)
 		default:
 			err = fmt.Errorf("cannot read values of type %s from page", header.Type)
 		}
@@ -575,6 +515,115 @@ func (f *filePages) ReadPage() (Page, error) {
 	}
 }
 
+func (f *filePages) readDictionary() error {
+	chunk := io.NewSectionReader(f.chunk.file, f.baseOffset, f.chunk.chunk.MetaData.TotalCompressedSize)
+	rbuf := acquireReadBuffer(chunk)
+	defer releaseReadBuffer(rbuf)
+
+	decoder := thrift.NewDecoder(f.protocol.NewReader(rbuf))
+	header := new(format.PageHeader)
+
+	if err := decoder.Decode(header); err != nil {
+		return err
+	}
+
+	page := acquireDataPage()
+	defer releaseDataPage(page)
+
+	if err := f.readPage(header, page, rbuf); err != nil {
+		return err
+	}
+
+	return f.readDictionaryPage(header, page)
+}
+
+func (f *filePages) readDictionaryPage(header *format.PageHeader, page *dataPage) (err error) {
+	if header.DictionaryPageHeader == nil {
+		return ErrMissingPageHeader
+	}
+	if f.index > 0 {
+		return ErrUnexpectedDictionaryPage
+	}
+	f.dictPage, _ = dictPagePool.Get().(*dictPage)
+	if f.dictPage == nil {
+		f.dictPage = new(dictPage)
+	}
+	f.dataPage.dictionary, err = f.chunk.column.decodeDictionary(
+		DictionaryPageHeader{header.DictionaryPageHeader},
+		page,
+		f.dictPage,
+	)
+	return err
+}
+
+func (f *filePages) readDataPageV1(header *format.PageHeader) (Page, error) {
+	if header.DataPageHeader == nil {
+		return nil, ErrMissingPageHeader
+	}
+	if isDictionaryFormat(header.DataPageHeader.Encoding) && f.dataPage.dictionary == nil {
+		if err := f.readDictionary(); err != nil {
+			return nil, err
+		}
+	}
+	return f.chunk.column.decodeDataPageV1(DataPageHeaderV1{header.DataPageHeader}, f.dataPage)
+}
+
+func (f *filePages) readDataPageV2(header *format.PageHeader) (Page, error) {
+	if header.DataPageHeaderV2 == nil {
+		return nil, ErrMissingPageHeader
+	}
+	if isDictionaryFormat(header.DataPageHeaderV2.Encoding) && f.dataPage.dictionary == nil {
+		// If the program seeked to a row passed the first page, the dictionary
+		// page may not have been seen, in which case we have to lazily load it
+		// from the beginning of column chunk.
+		if err := f.readDictionary(); err != nil {
+			return nil, err
+		}
+	}
+	return f.chunk.column.decodeDataPageV2(DataPageHeaderV2{header.DataPageHeaderV2}, f.dataPage)
+}
+
+func (f *filePages) readPage(header *format.PageHeader, page *dataPage, reader *bufio.Reader) error {
+	compressedPageSize, uncompressedPageSize := int(header.CompressedPageSize), int(header.UncompressedPageSize)
+
+	if cap(page.data) < compressedPageSize {
+		page.data = make([]byte, compressedPageSize)
+	} else {
+		page.data = page.data[:compressedPageSize]
+	}
+	if cap(page.values) < uncompressedPageSize {
+		page.values = make([]byte, 0, uncompressedPageSize)
+	}
+
+	if _, err := io.ReadFull(reader, page.data); err != nil {
+		return err
+	}
+
+	if header.CRC != 0 {
+		headerChecksum := uint32(header.CRC)
+		bufferChecksum := crc32.ChecksumIEEE(page.data)
+
+		if headerChecksum != bufferChecksum {
+			// The parquet specs indicate that corruption errors could be
+			// handled gracefully by skipping pages, tho this may not always
+			// be practical. Depending on how the pages are consumed,
+			// missing rows may cause unpredictable behaviors in algorithms.
+			//
+			// For now, we assume these errors to be fatal, but we may
+			// revisit later and improve error handling to be more resilient
+			// to data corruption.
+			return fmt.Errorf("crc32 checksum mismatch in page of column %q: want=0x%08X got=0x%08X: %w",
+				f.columnPath(),
+				headerChecksum,
+				bufferChecksum,
+				ErrCorrupted,
+			)
+		}
+	}
+
+	return nil
+}
+
 func (f *filePages) SeekToRow(rowIndex int64) (err error) {
 	if f.chunk == nil {
 		return io.ErrClosedPipe
diff --git a/go.mod b/go.mod
index c149187..690e0d3 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/andybalholm/brotli v1.0.3
 	github.com/google/uuid v1.3.0
 	github.com/hexops/gotextdiff v1.0.3
-	github.com/klauspost/compress v1.15.2
+	github.com/klauspost/compress v1.15.5
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pierrec/lz4/v4 v4.1.9
 	github.com/segmentio/encoding v0.3.5
diff --git a/go.sum b/go.sum
index e5d4daa..dd857c4 100644
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw=
-github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.5 h1:qyCLMz2JCrKADihKOh9FxnW3houKeNsp2h5OEz0QSEA=
+github.com/klauspost/compress v1.15.5/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
diff --git a/schema.go b/schema.go
index e985f7b..d1b3b01 100644
--- a/schema.go
+++ b/schema.go
@@ -69,6 +69,10 @@ type Schema struct {
 // Invalid combination of struct tags and Go types, or repeating options will
 // cause the function to panic.
 //
+// As a special case, if the field tag is "-", the field is omitted from the schema
+// and the data will not be written into the parquet file(s).
+// Note that a field with name "-" can still be generated using the tag "-,".
+//
 // The schema name is the Go type name of the value.
 func SchemaOf(model interface{}) *Schema {
 	return schemaOf(dereference(reflect.TypeOf(model)))
@@ -314,10 +318,17 @@ func structFieldsOf(t reflect.Type) []reflect.StructField {
 
 func appendStructFields(t reflect.Type, fields []reflect.StructField, index []int, offset uintptr) []reflect.StructField {
 	for i, n := 0, t.NumField(); i < n; i++ {
+		f := t.Field(i)
+		if tag := f.Tag.Get("parquet"); tag != "" {
+			name, _ := split(tag)
+			if tag != "-," && name == "-" {
+				continue
+			}
+		}
+
 		fieldIndex := index[:len(index):len(index)]
 		fieldIndex = append(fieldIndex, i)
 
-		f := t.Field(i)
 		f.Offset += offset
 
 		if f.Anonymous {
@@ -391,7 +402,6 @@ func (f *structField) Value(base reflect.Value) reflect.Value {
 			return fieldByIndex(base, f.index)
 		}
 	}
-	return reflect.Value{}
 }
 
 func structFieldString(f reflect.StructField) string {


Test patch:
diff --git a/reader_test.go b/reader_test.go
index 9449e52..a82c63a 100644
--- a/reader_test.go
+++ b/reader_test.go
@@ -352,3 +352,112 @@ func TestReaderSeekToRow(t *testing.T) {
 		}
 	}
 }
+
+func TestSeekToRowNoDict(t *testing.T) {
+	type rowType struct {
+		Name utf8string `parquet:","` // no dictionary encoding
+	}
+
+	// write samples to in-memory buffer
+	buf := new(bytes.Buffer)
+	schema := parquet.SchemaOf(new(rowType))
+	w := parquet.NewWriter(buf, schema)
+	sample := rowType{
+		Name: "foo1",
+	}
+	// write two rows
+	w.Write(sample)
+	sample.Name = "foo2"
+	w.Write(sample)
+	w.Close()
+
+	// create reader
+	r := parquet.NewReader(bytes.NewReader(buf.Bytes()))
+
+	// read second row
+	r.SeekToRow(1)
+	row := new(rowType)
+	err := r.Read(row)
+	if err != nil {
+		t.Fatalf("reading row: %v", err)
+	}
+	// fmt.Println(&sample, row)
+	if *row != sample {
+		t.Fatalf("read != write")
+	}
+}
+
+func TestSeekToRowReadAll(t *testing.T) {
+	type rowType struct {
+		Name utf8string `parquet:",dict"`
+	}
+
+	// write samples to in-memory buffer
+	buf := new(bytes.Buffer)
+	schema := parquet.SchemaOf(new(rowType))
+	w := parquet.NewWriter(buf, schema)
+	sample := rowType{
+		Name: "foo1",
+	}
+	// write two rows
+	w.Write(sample)
+	sample.Name = "foo2"
+	w.Write(sample)
+	w.Close()
+
+	// create reader
+	r := parquet.NewReader(bytes.NewReader(buf.Bytes()))
+
+	// read first row
+	r.SeekToRow(0)
+	row := new(rowType)
+	err := r.Read(row)
+	if err != nil {
+		t.Fatalf("reading row: %v", err)
+	}
+	// read second row
+	r.SeekToRow(1)
+	row = new(rowType)
+	err = r.Read(row)
+	if err != nil {
+		t.Fatalf("reading row: %v", err)
+	}
+	// fmt.Println(&sample, row)
+	if *row != sample {
+		t.Fatalf("read != write")
+	}
+}
+
+func TestSeekToRowDictReadSecond(t *testing.T) {
+	type rowType struct {
+		Name utf8string `parquet:",dict"`
+	}
+
+	// write samples to in-memory buffer
+	buf := new(bytes.Buffer)
+	schema := parquet.SchemaOf(new(rowType))
+	w := parquet.NewWriter(buf, schema)
+	sample := rowType{
+		Name: "foo1",
+	}
+	// write two rows
+	w.Write(sample)
+	sample.Name = "foo2"
+	w.Write(sample)
+	w.Close()
+
+	// create reader
+	r := parquet.NewReader(bytes.NewReader(buf.Bytes()))
+
+	// read second row
+	r.SeekToRow(1)
+	row := new(rowType)
+	err := r.Read(row)
+	if err != nil {
+		t.Fatalf("reading row: %v", err)
+	}
+	// fmt.Println(&sample, row)
+	if *row != sample {
+		t.Fatalf("read != write")
+	}
+}
diff --git a/schema_test.go b/schema_test.go
index 50cb976..3a3231e 100644
--- a/schema_test.go
+++ b/schema_test.go
@@ -52,6 +52,35 @@ func TestSchemaOf(t *testing.T) {
 		required binary first_name (STRING);
 		required binary last_name (STRING);
 	}
+}`,
+		},
+
+		{
+			value: new(struct {
+				Inner struct {
+					FirstName          string `parquet:"first_name"`
+					ShouldNotBePresent string `parquet:"-"`
+				} `parquet:"inner,optional"`
+			}),
+			print: `message {
+	optional group inner {
+		required binary first_name (STRING);
+	}
+}`,
+		},
+
+		{
+			value: new(struct {
+				Inner struct {
+					FirstName    string `parquet:"first_name"`
+					MyNameIsDash string `parquet:"-,"`
+				} `parquet:"inner,optional"`
+			}),
+			print: `message {
+	optional group inner {
+		required binary first_name (STRING);
+		required binary - (STRING);
+	}
 }`,
 		},
 	}
diff --git a/writer_test.go b/writer_test.go
index 2fc53b0..7ea7e45 100644
--- a/writer_test.go
+++ b/writer_test.go
@@ -78,6 +78,13 @@ type timeseries struct {
 	Value     float64 `parquet:"value"`
 }
 
+type event struct {
+	Name     string  `parquet:"name,dict"`
+	Type     string  `parquet:"-"`
+	Value    float64 `parquet:"value"`
+	Category string  `parquet:"-"`
+}
+
 var writerTests = []struct {
 	scenario string
 	version  int
@@ -277,10 +284,10 @@ value 10: R:0 D:0 V:10.0
 		dump: `row group 0
 --------------------------------------------------------------------------------
 owner:              BINARY ZSTD DO:0 FPO:4 SZ:66/57/0.86 VC:2 ENC:DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
-ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:70 SZ:162/112/0.69 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:70 SZ:162/112/0.69 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 contacts:
-.name:              BINARY UNCOMPRESSED DO:0 FPO:232 SZ:116/116/1.00 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
-.phoneNumber:       BINARY ZSTD DO:0 FPO:348 SZ:113/95/0.84 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+.name:              BINARY UNCOMPRESSED DO:0 FPO:232 SZ:116/116/1.00 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
+.phoneNumber:       BINARY ZSTD DO:0 FPO:348 SZ:113/95/0.84 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 
     owner TV=2 RL=0 DL=0
     ----------------------------------------------------------------------------
@@ -359,10 +366,10 @@ value 3: R:0 D:0 V:<null>
 		dump: `row group 0
 --------------------------------------------------------------------------------
 owner:              BINARY ZSTD DO:0 FPO:4 SZ:71/62/0.87 VC:2 ENC:DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
-ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:75 SZ:156/106/0.68 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+ownerPhoneNumbers:  BINARY GZIP DO:0 FPO:75 SZ:156/106/0.68 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 contacts:
-.name:              BINARY UNCOMPRESSED DO:0 FPO:231 SZ:110/110/1.00 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
-.phoneNumber:       BINARY ZSTD DO:0 FPO:341 SZ:108/90/0.83 VC:3 ENC:DELTA_LENGTH_BYTE_ARRAY,RLE ST:[no stats for this column]
+.name:              BINARY UNCOMPRESSED DO:0 FPO:231 SZ:110/110/1.00 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
+.phoneNumber:       BINARY ZSTD DO:0 FPO:341 SZ:108/90/0.83 VC:3 ENC:RLE,DELTA_LENGTH_BYTE_ARRAY ST:[no stats for this column]
 
     owner TV=2 RL=0 DL=0
     ----------------------------------------------------------------------------
@@ -409,6 +416,40 @@ BINARY contacts.phoneNumber
 value 1: R:0 D:2 V:555 987 6543
 value 2: R:1 D:1 V:<null>
 value 3: R:0 D:0 V:<null>
+`,
+	},
+
+	{
+		scenario: "omit `-` fields",
+		version:  v1,
+		rows: []interface{}{
+			&event{Name: "customer1", Type: "request", Value: 42.0},
+			&event{Name: "customer2", Type: "access", Value: 1.0},
+		},
+		dump: `row group 0
+--------------------------------------------------------------------------------
+name:   BINARY UNCOMPRESSED DO:4 FPO:49 SZ:73/73/1.00 VC:2 ENC:PLAIN,RLE_DICTIONARY ST:[no stats for this column]
+value:  DOUBLE UNCOMPRESSED DO:0 FPO:77 SZ:39/39/1.00 VC:2 ENC:PLAIN ST:[no stats for this column]
+
+    name TV=2 RL=0 DL=0 DS: 2 DE:PLAIN
+    ----------------------------------------------------------------------------
+    page 0:                  DLE:RLE RLE:RLE VLE:RLE_DICTIONARY ST:[no stats for this column] CRC:[verified] SZ:5 VC:2
+
+    value TV=2 RL=0 DL=0
+    ----------------------------------------------------------------------------
+    page 0:                  DLE:RLE RLE:RLE VLE:PLAIN ST:[no stats for this column] CRC:[verified] SZ:16 VC:2
+
+BINARY name
+--------------------------------------------------------------------------------
+*** row group 1 of 1, values 1 to 2 ***
+value 1: R:0 D:0 V:customer1
+value 2: R:0 D:0 V:customer2
+
+DOUBLE value
+--------------------------------------------------------------------------------
+*** row group 1 of 1, values 1 to 2 ***
+value 1: R:0 D:0 V:42.0
+value 2: R:0 D:0 V:1.0
 `,
 	},
 }