Repository: parquet-go/parquet-go (go)

Issue:
Writer silently skip row (or just column) for `FixedLenByteArray(N)` schema if value `len(value)` != N`
I was trying to write a schema:

```
	schema := parquet.NewSchema("testRow", parquet.Group{
		"fixedField": parquet.Leaf(parquet.FixedLenByteArrayType(32)),
	})
```

And was providing a `[]byte` that hadn't exactly 32 bytes. This was leading my resulting file to be empty, it had no rows while it should had one.

Here a test case the show the issue:

```
func TestParquetFixedByte(t *testing.T) {
	schema := parquet.NewSchema("testRow", parquet.Group{
		"fixedField": parquet.Leaf(parquet.FixedLenByteArrayType(32)),
	})

	file, err := os.Create("/tmp/file.parquet")
	require.NoError(t, err)
	defer file.Close()

	writer := parquet.NewGenericWriter[any](file, schema)
	n, err := writer.WriteRows([]parquet.Row{
		{parquet.FixedLenByteArrayValue(make([]byte, 16))},
	})
	require.NoError(t, err)
	require.Equal(t, 1, n)

	require.NoError(t, writer.Close())
	require.NoError(t, file.Close())

	type testRow struct {
		FixedField []byte `parquet:"fixedField"`
	}

	rows, err := parquet.ReadFile[testRow]("/tmp/file.parquet", schema)
	require.NoError(t, err)

	require.Len(t, rows, 1)
	require.Equal(t, hex.EncodeToString(make([]byte, 32)), hex.EncodeToString(rows[0].FixedField[:]))
}
```

If you change `{parquet.FixedLenByteArrayValue(make([]byte, 16))}` to be `{parquet.FixedLenByteArrayValue(make([]byte, 32))}`, it passes.


Pull request:
validate size of fixed-length values written
Fixes #178 

Patch:
diff --git a/column_buffer.go b/column_buffer.go
index 4e0616d..d27236a 100644
--- a/column_buffer.go
+++ b/column_buffer.go
@@ -1629,8 +1629,11 @@ func (col *fixedLenByteArrayColumnBuffer) Write(b []byte) (int, error) {
 }
 
 func (col *fixedLenByteArrayColumnBuffer) WriteFixedLenByteArrays(values []byte) (int, error) {
+	if len(values) == 0 {
+		return 0, nil
+	}
 	d, m := len(values)/col.size, len(values)%col.size
-	if m != 0 {
+	if d == 0 || m != 0 {
 		return 0, fmt.Errorf("cannot write FIXED_LEN_BYTE_ARRAY values of size %d from input of size %d", col.size, len(values))
 	}
 	col.data = append(col.data, values...)
@@ -1638,7 +1641,10 @@ func (col *fixedLenByteArrayColumnBuffer) WriteFixedLenByteArrays(values []byte)
 }
 
 func (col *fixedLenByteArrayColumnBuffer) WriteValues(values []Value) (int, error) {
-	for _, v := range values {
+	for i, v := range values {
+		if n := len(v.byteArray()); n != col.size {
+			return i, fmt.Errorf("cannot write FIXED_LEN_BYTE_ARRAY values of size %d from input of size %d", col.size, n)
+		}
 		col.data = append(col.data, v.byteArray()...)
 	}
 	return len(values), nil
@@ -2024,7 +2030,7 @@ func writeRowsFuncOf(t reflect.Type, schema *Schema, path columnPath) writeRowsF
 
 	case reflect.Array:
 		if t.Elem().Kind() == reflect.Uint8 {
-			return writeRowsFuncOfRequired(t, schema, path)
+			return writeRowsFuncOfArray(t, schema, path)
 		}
 
 	case reflect.Pointer:
@@ -2157,6 +2163,16 @@ func writeRowsFuncOfOptional(t reflect.Type, schema *Schema, path columnPath, wr
 	}
 }
 
+func writeRowsFuncOfArray(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc {
+	column := schema.mapping.lookup(path)
+	arrayLen := t.Len()
+	columnLen := column.node.Type().Length()
+	if arrayLen != columnLen {
+		panic(fmt.Sprintf("cannot convert Go values of type "+typeNameOf(t)+" to FIXED_LEN_BYTE_ARRAY(%d)", columnLen))
+	}
+	return writeRowsFuncOfRequired(t, schema, path)
+}
+
 func writeRowsFuncOfPointer(t reflect.Type, schema *Schema, path columnPath) writeRowsFunc {
 	elemType := t.Elem()
 	elemSize := uintptr(elemType.Size())


Test patch:
diff --git a/parquet_test.go b/parquet_test.go
index 59bde83..71ef16f 100644
--- a/parquet_test.go
+++ b/parquet_test.go
@@ -419,6 +419,47 @@ func TestIssue423(t *testing.T) {
 	assertRowsEqual(t, writeRows, readRows)
 }
 
+func TestIssue178(t *testing.T) {
+	schema := parquet.NewSchema("testRow", parquet.Group{
+		"fixedField": parquet.Leaf(parquet.FixedLenByteArrayType(32)),
+	})
+
+	tests := []struct {
+		name string
+		test func(*testing.T) error
+	}{
+		{
+			name: "WriteRows",
+			test: func(t *testing.T) error {
+				buffer := new(bytes.Buffer)
+				writer := parquet.NewGenericWriter[any](buffer, schema)
+				_, err := writer.WriteRows([]parquet.Row{
+					{parquet.FixedLenByteArrayValue(make([]byte, 16))},
+				})
+				return err
+			},
+		},
+
+		{
+			name: "WriteFixedLenByteArray",
+			test: func(t *testing.T) error {
+				buffer := parquet.NewGenericBuffer[any](schema)
+				column := buffer.ColumnBuffers()[0]
+				_, err := column.(parquet.FixedLenByteArrayWriter).WriteFixedLenByteArrays(make([]byte, 16))
+				return err
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := test.test(t); err == nil {
+				t.Fatal("expected error when writing a byte slice of length 16 to a fixed length byte array column of length 32")
+			}
+		})
+	}
+}
+
 func TestReadFileGenericMultipleRowGroupsMultiplePages(t *testing.T) {
 	type MyRow struct {
 		ID    [16]byte `parquet:"id,delta,uuid"`