Repository: segmentio/parquet-go (go)

Issue:
Incorrect definition level in generic Writer
For the structure  below, if 'TB' is null, the definition level should be 1 (because T1 is defined), not zero. The non-generic writer does the right thing.

```go
package main

import (
	"fmt"
	//"github.com/davecgh/go-spew/spew"
	"github.com/vc42/parquet-go"
	"os"
)

type T2 struct {
	X string `parquet:",dict,optional"`
}

type T1 struct {
	TA *T2
	TB *T2
}

type T struct {
	T1 *T1
}

const nRows = 1

func main() {

	t := T{
		T1: &T1{
			TA: &T2{
				X: "abc",
			},
		},
	}

	f, err := os.OpenFile("/tmp/123.parquet", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		panic(err)
	}

	fmt.Printf("bad:\n")
	rows := make([]T, nRows)
	gw := parquet.NewGenericWriter[T](f)
	for i := 0; i < nRows; i++ {
		rows[i] = t
	}
	_, err = gw.Write(rows)
	if err != nil {
		panic(err)
	}
	gw.Close()

	fmt.Printf("good:\n")
	fw := parquet.NewWriter(f)
	for i := 0; i < nRows; i++ {
		err = fw.Write(&t)
		if err != nil {
			panic(err)
		}
	}
	fw.Close()

	f.Close()

	//rows2, err := parquet.ReadFile[T]("/tmp/123.parquet")
	//spew.Dump(rows2)
}
```

To get the output below, you need to instrument the 'write.go' with a print statement:
```go
func (c *writerColumn) writeBufferedPage(page BufferedPage) (int64, error) {
	fmt.Printf("col: %v max_def %v def %v\n", c.columnPath, c.maxDefinitionLevel, page.DefinitionLevels())
```

bad:
```
col: T1.TA.X max_def 3 def [3]
col: T1.TB.X max_def 3 def [0]
```

good:
```
col: T1.TA.X max_def 3 def [3]
col: T1.TB.X max_def 3 def [1]
```

The bug causes the google big query loader crash.


Pull request:
fix definition level when optional columns have no values
Fixes #272 


Patch:
diff --git a/column_buffer.go b/column_buffer.go
index f35932a..af07e5c 100644
--- a/column_buffer.go
+++ b/column_buffer.go
@@ -337,7 +337,7 @@ func (col *optionalColumnBuffer) writeValues(rows sparse.Array, levels columnLev
 	// we still need to output a row to the buffer to record the definition
 	// level.
 	if rows.Len() == 0 {
-		col.definitionLevels = append(col.definitionLevels, 0)
+		col.definitionLevels = append(col.definitionLevels, levels.definitionLevel)
 		col.rows = append(col.rows, -1)
 		return
 	}


Test patch:
diff --git a/writer_go18_test.go b/writer_go18_test.go
index 851d6fd..f18e998 100644
--- a/writer_go18_test.go
+++ b/writer_go18_test.go
@@ -3,6 +3,7 @@
 package parquet_test
 
 import (
+	"bytes"
 	"io"
 	"math/rand"
 	"reflect"
@@ -82,3 +83,63 @@ func benchmarkGenericWriter[Row generator[Row]](b *testing.B) {
 		})
 	})
 }
+
+func TestIssue272(t *testing.T) {
+	type T2 struct {
+		X string `parquet:",dict,optional"`
+	}
+
+	type T1 struct {
+		TA *T2
+		TB *T2
+	}
+
+	type T struct {
+		T1 *T1
+	}
+
+	const nRows = 1
+
+	row := T{
+		T1: &T1{
+			TA: &T2{
+				X: "abc",
+			},
+		},
+	}
+
+	rows := make([]T, nRows)
+	for i := range rows {
+		rows[i] = row
+	}
+
+	b := new(bytes.Buffer)
+	w := parquet.NewGenericWriter[T](b)
+
+	if _, err := w.Write(rows); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	f := bytes.NewReader(b.Bytes())
+	r := parquet.NewGenericReader[T](f)
+
+	parquetRows := make([]parquet.Row, nRows)
+	n, err := r.ReadRows(parquetRows)
+	if err != nil && err != io.EOF {
+		t.Fatal(err)
+	}
+	if n != nRows {
+		t.Fatalf("wrong number of rows read: want=%d got=%d", nRows, n)
+	}
+	for _, r := range parquetRows {
+		if d := r[0].DefinitionLevel(); d != 3 {
+			t.Errorf("wrong definition level for column 0: %d", d)
+		}
+		if d := r[1].DefinitionLevel(); d != 1 {
+			t.Errorf("wrong definition level for column 1: %d", d)
+		}
+	}
+}