Repository: segmentio/parquet-go (go)

Issue:
definitionLevel redux
If we want to support a slice of pointers, we need to fix *excessive* in this case def level due to the fact that it's incremented twice, as a slice member and as a pointer(optional):

test case:
```go
//
package main

import (
	"bytes"
	"fmt"
	//"github.com/davecgh/go-spew/spew"
	//"io"

	"github.com/segmentio/parquet-go"
)

type T2 struct {
	Id        int    `parquet:",plain,optional"`
	Name      string `parquet:",plain,optional"`
	//Timestamp string `json:"timestamp,omitempty" parquet:"timestamp,plain,timestamp,optional"`
	//Date      string `json:"timestamp,omitempty" parquet:"date,plain,date,optional"`
}

type T1 struct {
	TA []*T2
}

type T struct {
	T1 *T1
}

const nRows = 1

func main() {

	t := T{
		T1: &T1{
			TA: []*T2{&T2{
				Id: 43,
				Name:      "john",
				//Timestamp: "2022-04-12 19:51:52",
				//Date:      "2022-04-12",
			},
			},
		},
	}

	buffer := new(bytes.Buffer)

	rows := make([]T, nRows)
	for i := 0; i < nRows; i++ {
		t.T1.TA[0].Id = i+40
		rows[i] = t
	}

	fmt.Printf("bad:\n")
	gw := parquet.NewGenericWriter[T](buffer)
	_, err := gw.Write(rows)
	if err != nil {
		panic(err)
	}
	gw.Close()

	fmt.Printf("good:\n")
	w := parquet.NewWriter(buffer)
	for i := 0; i < nRows; i++ {
		err := w.Write(&t)
		if err != nil {
			panic(err)
		}
	}
	w.Close()

	//r := bytes.NewReader(buffer.Bytes())
	//rows2, err := parquet.Read[T](r, int64(buffer.Len()))
	//if err != io.EOF {
	//	panic(err)
	//}
	//spew.Dump(rows2)
}
```

Output:
bad:
```
col: T1.TA.Id max_def 3 def [4]
col: T1.TA.Name max_def 3 def [4]
```
good:
```
col: T1.TA.Id max_def 3 def [3]
col: T1.TA.Name max_def 3 def [3]
```

Possible fix (not to increment if the element is a pointer, not sure if there's a better way):

```diff
diff --git a/column_buffer_go18.go b/column_buffer_go18.go
index 5f5d7f7..f0cb8d6 100644
--- a/column_buffer_go18.go
+++ b/column_buffer_go18.go
@@ -250,7 +250,9 @@ func writeRowsFuncOfSlice(t reflect.Type, schema *Schema, path columnPath) write
                        elemLevels := levels
                        if a.Len() > 0 {
                                b = a.Slice(0, 1)
-                               elemLevels.definitionLevel++
+                               if elemType.Kind() != reflect.Pointer {
+                                       elemLevels.definitionLevel++
+                               }
                        }
 
                        if err := writeRows(columns, b, elemLevels); err != nil {
```

Pull request:
fix issue 279
Fixes #279 

Patch:
diff --git a/column_buffer_go18.go b/column_buffer_go18.go
index 5f5d7f7..918b0d3 100644
--- a/column_buffer_go18.go
+++ b/column_buffer_go18.go
@@ -235,6 +235,15 @@ func writeRowsFuncOfSlice(t reflect.Type, schema *Schema, path columnPath) write
 	elemType := t.Elem()
 	elemSize := uintptr(elemType.Size())
 	writeRows := writeRowsFuncOf(elemType, schema, path)
+
+	// When the element is a pointer type, the writeRows function will be an
+	// instance returned by writeRowsFuncOfPointer, which handles incrementing
+	// the definition level if the pointer value is not nil.
+	definitionLevelIncrement := byte(0)
+	if elemType.Kind() != reflect.Ptr {
+		definitionLevelIncrement = 1
+	}
+
 	return func(columns []ColumnBuffer, rows sparse.Array, levels columnLevels) error {
 		if rows.Len() == 0 {
 			return writeRows(columns, rows, levels)
@@ -250,7 +259,7 @@ func writeRowsFuncOfSlice(t reflect.Type, schema *Schema, path columnPath) write
 			elemLevels := levels
 			if a.Len() > 0 {
 				b = a.Slice(0, 1)
-				elemLevels.definitionLevel++
+				elemLevels.definitionLevel += definitionLevelIncrement
 			}
 
 			if err := writeRows(columns, b, elemLevels); err != nil {


Test patch:
diff --git a/writer_go18_test.go b/writer_go18_test.go
index f18e998..69224fc 100644
--- a/writer_go18_test.go
+++ b/writer_go18_test.go
@@ -143,3 +143,66 @@ func TestIssue272(t *testing.T) {
 		}
 	}
 }
+
+func TestIssue279(t *testing.T) {
+	type T2 struct {
+		Id   int    `parquet:",plain,optional"`
+		Name string `parquet:",plain,optional"`
+	}
+
+	type T1 struct {
+		TA []*T2
+	}
+
+	type T struct {
+		T1 *T1
+	}
+
+	const nRows = 1
+
+	row := T{
+		T1: &T1{
+			TA: []*T2{
+				&T2{
+					Id:   43,
+					Name: "john",
+				},
+			},
+		},
+	}
+
+	rows := make([]T, nRows)
+	for i := range rows {
+		rows[i] = row
+	}
+
+	b := new(bytes.Buffer)
+	w := parquet.NewGenericWriter[T](b)
+
+	if _, err := w.Write(rows); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	f := bytes.NewReader(b.Bytes())
+	r := parquet.NewGenericReader[T](f)
+
+	parquetRows := make([]parquet.Row, nRows)
+	n, err := r.ReadRows(parquetRows)
+	if err != nil && err != io.EOF {
+		t.Fatal(err)
+	}
+	if n != nRows {
+		t.Fatalf("wrong number of rows read: want=%d got=%d", nRows, n)
+	}
+	for _, r := range parquetRows {
+		if d := r[0].DefinitionLevel(); d != 3 {
+			t.Errorf("wrong definition level for column 0: %d", d)
+		}
+		if d := r[1].DefinitionLevel(); d != 3 {
+			t.Errorf("wrong definition level for column 1: %d", d)
+		}
+	}
+}