Repository: segmentio/parquet-go
Pull request #110 (2022-03-21 17:15:02)

Document compatibility guarantees


Patch:
diff --git a/.words b/.words
new file mode 100644
index 0000000..abc9140
--- /dev/null
+++ b/.words
@@ -0,0 +1,10 @@
+
+RowType
+Twilio
+bottlenecked
+decompressors
+int96
+reindexing
+repositions
+schemas
+ColumnPages
diff --git a/README.md b/README.md
index 36e449a..82a1d19 100644
--- a/README.md
+++ b/README.md
@@ -32,11 +32,26 @@ The package is distributed as a standard Go module that programs can take a
 dependency on and install with the following command:
 
 ```
-$ go get github.com/segmentio/parquet-go
+go get github.com/segmentio/parquet-go
 ```
 
 Go 1.17 or later is required to use the package.
 
+### Compatibility Guarantees
+
+The package is currently released as a pre-v1 version, which gives maintainers
+the freedom to break backward compatibility to help improve the APIs as we learn
+which initial design decisions would need to be revisited to better support the
+use cases that the library solves for. These occurrences are expected to be rare
+in frequency and documentation will be produce to guide users on how to adapt
+their programs to breaking changes.
+
+One planned breaking change will be to support generics, which were released in
+Go 1.18 and will simplify the code and provide better type safety.
+The targetted timeline for this change is end of 2022; programs depending on
+parquet-go will be expected to compile with Go 1.18 by then, at which point
+we will drop compatibility with Go 1.17.
+
 ## Usage
 
 The following sections describe how to use APIs exposed by the library,
diff --git a/dictionary.go b/dictionary.go
index 08c5faf..83de7d6 100644
--- a/dictionary.go
+++ b/dictionary.go
@@ -181,7 +181,7 @@ func newFixedLenByteArrayDictionary(typ Type, bufferSize int) *fixedLenByteArray
 	return &fixedLenByteArrayDictionary{
 		typ:    typ,
 		size:   size,
-		values: make([]byte, 0, dictCap(bufferSize, size)),
+		values: make([]byte, 0, dictCap(bufferSize, size)*size),
 	}
 }
 
@@ -349,7 +349,9 @@ func (page *indexedPage) RepetitionLevels() []int8 { return nil }
 
 func (page *indexedPage) DefinitionLevels() []int8 { return nil }
 
-func (page *indexedPage) WriteTo(e encoding.Encoder) error { return e.EncodeInt32(page.values) }
+func (page *indexedPage) WriteTo(e encoding.Encoder) error {
+	return e.EncodeInt32(page.values)
+}
 
 func (page *indexedPage) Values() ValueReader { return &indexedPageReader{page: page} }
 
diff --git a/encoding/encoding.go b/encoding/encoding.go
index 202b998..ee27659 100644
--- a/encoding/encoding.go
+++ b/encoding/encoding.go
@@ -70,19 +70,19 @@ type Encoder interface {
 	// knows that all values can fit in 16 bits.
 	EncodeInt16(data []int16) error
 
-	// Encodes an array of 32 bits integer values using this encoder.
+	// Encodes an array of 32 bit integer values using this encoder.
 	EncodeInt32(data []int32) error
 
-	// Encodes an array of 64 bits integer values using this encoder.
+	// Encodes an array of 64 bit integer values using this encoder.
 	EncodeInt64(data []int64) error
 
-	// Encodes an array of 96 bits integer values using this encoder.
+	// Encodes an array of 96 bit integer values using this encoder.
 	EncodeInt96(data []deprecated.Int96) error
 
-	// Encodes an array of 32 bits floating point values using this encoder.
+	// Encodes an array of 32 bit floating point values using this encoder.
 	EncodeFloat(data []float32) error
 
-	// Encodes an array of 64 bits floating point values using this encoder.
+	// Encodes an array of 64 bit floating point values using this encoder.
 	EncodeDouble(data []float64) error
 
 	// Encodes an array of variable length byte array values using this encoder.
diff --git a/encoding/notsupported.go b/encoding/notsupported.go
index cef14ef..e7b7a85 100644
--- a/encoding/notsupported.go
+++ b/encoding/notsupported.go
@@ -21,9 +21,9 @@ var (
 	// ErrInvalidArguments is an error returned when arguments passed to the
 	// encoding functions are incorrect and will lead to an expected failure.
 	//
-	// As with ErrNotSupported, this error may be wrapped with specific information
-	// about the problem and application are expected to use errors.Is for
-	// comparisons
+	// As with ErrNotSupported, this error may be wrapped with specific
+	// information about the problem and applications are expected to use
+	// errors.Is for comparisons.
 	ErrInvalidArguments = errors.New("invalid encoding arguments")
 )
 
diff --git a/reader.go b/reader.go
index dd2151e..b354cef 100644
--- a/reader.go
+++ b/reader.go
@@ -184,7 +184,7 @@ func (r *Reader) updateReadSchema(rowType reflect.Type) error {
 // ReadRow reads the next row from r and appends in to the given Row buffer.
 //
 // The returned values are laid out in the order expected by the
-// parquet.(*Schema).Reconstrct method.
+// parquet.(*Schema).Reconstruct method.
 //
 // The method returns io.EOF when no more rows can be read from r.
 func (r *Reader) ReadRow(row Row) (Row, error) {
diff --git a/value.go b/value.go
index c600d8d..b28c624 100644
--- a/value.go
+++ b/value.go
@@ -489,7 +489,7 @@ func (v Value) Format(w fmt.State, r rune) {
 		case w.Flag('+'):
 			fmt.Fprintf(w, "%+[1]c %+[1]d %+[1]r %+[1]s", v)
 		case w.Flag('#'):
-			fmt.Fprintf(w, "parquet.Value{%+[1]c,%+[1]d,%+[1]r,%+[1]s}", v)
+			fmt.Fprintf(w, "parquet.Value{%+[1]c, %+[1]d, %+[1]r, %+[1]s}", v)
 		default:
 			v.Format(w, 's')
 		}


Test patch:
diff --git a/row_group_test.go b/row_group_test.go
index 615f914..fd15929 100644
--- a/row_group_test.go
+++ b/row_group_test.go
@@ -106,210 +106,216 @@ func testSeekToRow(t *testing.T, newRowGroup func([]Person) parquet.RowGroup) {
 	}
 }
 
-var mergeRowGroupsTests = [...]struct {
+type mergeRowGroupTestCase struct {
 	scenario string
 	options  []parquet.RowGroupOption
 	input    []parquet.RowGroup
 	output   parquet.RowGroup
-}{
-	{
-		scenario: "no row groups",
-		options: []parquet.RowGroupOption{
-			parquet.SchemaOf(Person{}),
-		},
-		output: sortedRowGroup(
-			[]parquet.RowGroupOption{
+}
+
+// Use a function instead of an array because otherwise the sortedRowGroup()
+// calls will exercise the code even if you're not running these test cases.
+func mergeRowGroupsTests() []mergeRowGroupTestCase {
+	return []mergeRowGroupTestCase{
+		{
+			scenario: "no row groups",
+			options: []parquet.RowGroupOption{
 				parquet.SchemaOf(Person{}),
 			},
-		),
-	},
+			output: sortedRowGroup(
+				[]parquet.RowGroupOption{
+					parquet.SchemaOf(Person{}),
+				},
+			),
+		},
 
-	{
-		scenario: "a single row group",
-		input: []parquet.RowGroup{
-			sortedRowGroup(nil,
+		{
+			scenario: "a single row group",
+			input: []parquet.RowGroup{
+				sortedRowGroup(nil,
+					Person{FirstName: "some", LastName: "one", Age: 30},
+					Person{FirstName: "some", LastName: "one else", Age: 31},
+					Person{FirstName: "and", LastName: "you", Age: 32},
+				),
+			},
+			output: sortedRowGroup(nil,
 				Person{FirstName: "some", LastName: "one", Age: 30},
 				Person{FirstName: "some", LastName: "one else", Age: 31},
 				Person{FirstName: "and", LastName: "you", Age: 32},
 			),
 		},
-		output: sortedRowGroup(nil,
-			Person{FirstName: "some", LastName: "one", Age: 30},
-			Person{FirstName: "some", LastName: "one else", Age: 31},
-			Person{FirstName: "and", LastName: "you", Age: 32},
-		),
-	},
-
-	{
-		scenario: "two row groups without ordering",
-		input: []parquet.RowGroup{
-			sortedRowGroup(nil, Person{FirstName: "some", LastName: "one", Age: 30}),
-			sortedRowGroup(nil, Person{FirstName: "some", LastName: "one else", Age: 31}),
-		},
-		output: sortedRowGroup(nil,
-			Person{FirstName: "some", LastName: "one", Age: 30},
-			Person{FirstName: "some", LastName: "one else", Age: 31},
-		),
-	},
-
-	{
-		scenario: "three row groups without ordering",
-		input: []parquet.RowGroup{
-			sortedRowGroup(nil, Person{FirstName: "some", LastName: "one", Age: 30}),
-			sortedRowGroup(nil, Person{FirstName: "some", LastName: "one else", Age: 31}),
-			sortedRowGroup(nil, Person{FirstName: "question", LastName: "answer", Age: 42}),
-		},
-		output: sortedRowGroup(nil,
-			Person{FirstName: "some", LastName: "one", Age: 30},
-			Person{FirstName: "some", LastName: "one else", Age: 31},
-			Person{FirstName: "question", LastName: "answer", Age: 42},
-		),
-	},
-
-	{
-		scenario: "row groups sorted by ascending last name",
-		options: []parquet.RowGroupOption{
-			parquet.SortingColumns(
-				parquet.Ascending("LastName"),
+
+		{
+			scenario: "two row groups without ordering",
+			input: []parquet.RowGroup{
+				sortedRowGroup(nil, Person{FirstName: "some", LastName: "one", Age: 30}),
+				sortedRowGroup(nil, Person{FirstName: "some", LastName: "one else", Age: 31}),
+			},
+			output: sortedRowGroup(nil,
+				Person{FirstName: "some", LastName: "one", Age: 30},
+				Person{FirstName: "some", LastName: "one else", Age: 31},
 			),
 		},
-		input: []parquet.RowGroup{
-			sortedRowGroup(
-				[]parquet.RowGroupOption{
-					parquet.SortingColumns(
-						parquet.Ascending("LastName"),
-					),
-				},
-				Person{FirstName: "Han", LastName: "Solo"},
-				Person{FirstName: "Luke", LastName: "Skywalker"},
-			),
-			sortedRowGroup(
-				[]parquet.RowGroupOption{
-					parquet.SortingColumns(
-						parquet.Ascending("LastName"),
-					),
-				},
-				Person{FirstName: "Obiwan", LastName: "Kenobi"},
+
+		{
+			scenario: "three row groups without ordering",
+			input: []parquet.RowGroup{
+				sortedRowGroup(nil, Person{FirstName: "some", LastName: "one", Age: 30}),
+				sortedRowGroup(nil, Person{FirstName: "some", LastName: "one else", Age: 31}),
+				sortedRowGroup(nil, Person{FirstName: "question", LastName: "answer", Age: 42}),
+			},
+			output: sortedRowGroup(nil,
+				Person{FirstName: "some", LastName: "one", Age: 30},
+				Person{FirstName: "some", LastName: "one else", Age: 31},
+				Person{FirstName: "question", LastName: "answer", Age: 42},
 			),
 		},
-		output: sortedRowGroup(nil,
-			Person{FirstName: "Obiwan", LastName: "Kenobi"},
-			Person{FirstName: "Luke", LastName: "Skywalker"},
-			Person{FirstName: "Han", LastName: "Solo"},
-		),
-	},
-
-	{
-		scenario: "row groups sorted by descending last name",
-		options: []parquet.RowGroupOption{
-			parquet.SortingColumns(
-				parquet.Descending("LastName"),
+
+		{
+			scenario: "row groups sorted by ascending last name",
+			options: []parquet.RowGroupOption{
+				parquet.SortingColumns(
+					parquet.Ascending("LastName"),
+				),
+			},
+			input: []parquet.RowGroup{
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Ascending("LastName"),
+						),
+					},
+					Person{FirstName: "Han", LastName: "Solo"},
+					Person{FirstName: "Luke", LastName: "Skywalker"},
+				),
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Ascending("LastName"),
+						),
+					},
+					Person{FirstName: "Obiwan", LastName: "Kenobi"},
+				),
+			},
+			output: sortedRowGroup(nil,
+				Person{FirstName: "Obiwan", LastName: "Kenobi"},
+				Person{FirstName: "Luke", LastName: "Skywalker"},
+				Person{FirstName: "Han", LastName: "Solo"},
 			),
 		},
-		input: []parquet.RowGroup{
-			sortedRowGroup(
-				[]parquet.RowGroupOption{
-					parquet.SortingColumns(
-						parquet.Descending("LastName"),
-					),
-				},
+
+		{
+			scenario: "row groups sorted by descending last name",
+			options: []parquet.RowGroupOption{
+				parquet.SortingColumns(
+					parquet.Descending("LastName"),
+				),
+			},
+			input: []parquet.RowGroup{
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Descending("LastName"),
+						),
+					},
+					Person{FirstName: "Han", LastName: "Solo"},
+					Person{FirstName: "Luke", LastName: "Skywalker"},
+				),
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Descending("LastName"),
+						),
+					},
+					Person{FirstName: "Obiwan", LastName: "Kenobi"},
+				),
+			},
+			output: sortedRowGroup(nil,
 				Person{FirstName: "Han", LastName: "Solo"},
 				Person{FirstName: "Luke", LastName: "Skywalker"},
-			),
-			sortedRowGroup(
-				[]parquet.RowGroupOption{
-					parquet.SortingColumns(
-						parquet.Descending("LastName"),
-					),
-				},
 				Person{FirstName: "Obiwan", LastName: "Kenobi"},
 			),
 		},
-		output: sortedRowGroup(nil,
-			Person{FirstName: "Han", LastName: "Solo"},
-			Person{FirstName: "Luke", LastName: "Skywalker"},
-			Person{FirstName: "Obiwan", LastName: "Kenobi"},
-		),
-	},
-
-	{
-		scenario: "row groups sorted by ascending last and first name",
-		options: []parquet.RowGroupOption{
-			parquet.SortingColumns(
-				parquet.Ascending("LastName"),
-				parquet.Ascending("FirstName"),
-			),
-		},
-		input: []parquet.RowGroup{
-			sortedRowGroup(
-				[]parquet.RowGroupOption{
-					parquet.SortingColumns(
-						parquet.Ascending("LastName"),
-						parquet.Ascending("FirstName"),
-					),
-				},
-				Person{FirstName: "Luke", LastName: "Skywalker"},
-				Person{FirstName: "Han", LastName: "Solo"},
-			),
-			sortedRowGroup(
-				[]parquet.RowGroupOption{
-					parquet.SortingColumns(
-						parquet.Ascending("LastName"),
-						parquet.Ascending("FirstName"),
-					),
-				},
+
+		{
+			scenario: "row groups sorted by ascending last and first name",
+			options: []parquet.RowGroupOption{
+				parquet.SortingColumns(
+					parquet.Ascending("LastName"),
+					parquet.Ascending("FirstName"),
+				),
+			},
+			input: []parquet.RowGroup{
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Ascending("LastName"),
+							parquet.Ascending("FirstName"),
+						),
+					},
+					Person{FirstName: "Luke", LastName: "Skywalker"},
+					Person{FirstName: "Han", LastName: "Solo"},
+				),
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Ascending("LastName"),
+							parquet.Ascending("FirstName"),
+						),
+					},
+					Person{FirstName: "Obiwan", LastName: "Kenobi"},
+					Person{FirstName: "Anakin", LastName: "Skywalker"},
+				),
+			},
+			output: sortedRowGroup(nil,
 				Person{FirstName: "Obiwan", LastName: "Kenobi"},
 				Person{FirstName: "Anakin", LastName: "Skywalker"},
-			),
-		},
-		output: sortedRowGroup(nil,
-			Person{FirstName: "Obiwan", LastName: "Kenobi"},
-			Person{FirstName: "Anakin", LastName: "Skywalker"},
-			Person{FirstName: "Luke", LastName: "Skywalker"},
-			Person{FirstName: "Han", LastName: "Solo"},
-		),
-	},
-
-	{
-		scenario: "row groups with conversion to a different schema",
-		options: []parquet.RowGroupOption{
-			parquet.SchemaOf(LastNameOnly{}),
-			parquet.SortingColumns(
-				parquet.Ascending("LastName"),
-			),
-		},
-		input: []parquet.RowGroup{
-			sortedRowGroup(
-				[]parquet.RowGroupOption{
-					parquet.SortingColumns(
-						parquet.Ascending("LastName"),
-					),
-				},
-				Person{FirstName: "Han", LastName: "Solo"},
 				Person{FirstName: "Luke", LastName: "Skywalker"},
+				Person{FirstName: "Han", LastName: "Solo"},
 			),
-			sortedRowGroup(
+		},
+
+		{
+			scenario: "row groups with conversion to a different schema",
+			options: []parquet.RowGroupOption{
+				parquet.SchemaOf(LastNameOnly{}),
+				parquet.SortingColumns(
+					parquet.Ascending("LastName"),
+				),
+			},
+			input: []parquet.RowGroup{
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Ascending("LastName"),
+						),
+					},
+					Person{FirstName: "Han", LastName: "Solo"},
+					Person{FirstName: "Luke", LastName: "Skywalker"},
+				),
+				sortedRowGroup(
+					[]parquet.RowGroupOption{
+						parquet.SortingColumns(
+							parquet.Ascending("LastName"),
+						),
+					},
+					Person{FirstName: "Obiwan", LastName: "Kenobi"},
+					Person{FirstName: "Anakin", LastName: "Skywalker"},
+				),
+			},
+			output: sortedRowGroup(
 				[]parquet.RowGroupOption{
 					parquet.SortingColumns(
 						parquet.Ascending("LastName"),
 					),
 				},
-				Person{FirstName: "Obiwan", LastName: "Kenobi"},
-				Person{FirstName: "Anakin", LastName: "Skywalker"},
+				LastNameOnly{LastName: "Solo"},
+				LastNameOnly{LastName: "Skywalker"},
+				LastNameOnly{LastName: "Skywalker"},
+				LastNameOnly{LastName: "Kenobi"},
 			),
 		},
-		output: sortedRowGroup(
-			[]parquet.RowGroupOption{
-				parquet.SortingColumns(
-					parquet.Ascending("LastName"),
-				),
-			},
-			LastNameOnly{LastName: "Solo"},
-			LastNameOnly{LastName: "Skywalker"},
-			LastNameOnly{LastName: "Skywalker"},
-			LastNameOnly{LastName: "Kenobi"},
-		),
-	},
+	}
 }
 
 func selfRowGroup(rowGroup parquet.RowGroup) parquet.RowGroup {
@@ -342,7 +348,7 @@ func TestMergeRowGroups(t *testing.T) {
 		{scenario: "file", function: fileRowGroup},
 	} {
 		t.Run(adapter.scenario, func(t *testing.T) {
-			for _, test := range mergeRowGroupsTests {
+			for _, test := range mergeRowGroupsTests() {
 				t.Run(test.scenario, func(t *testing.T) {
 					input := make([]parquet.RowGroup, len(test.input))
 					for i := range test.input {
diff --git a/writer_test.go b/writer_test.go
index 9b64e46..5562db2 100644
--- a/writer_test.go
+++ b/writer_test.go
@@ -9,6 +9,7 @@ import (
 	"testing"
 	"testing/quick"
 
+	"github.com/google/uuid"
 	"github.com/hexops/gotextdiff"
 	"github.com/hexops/gotextdiff/myers"
 	"github.com/hexops/gotextdiff/span"
@@ -565,3 +566,40 @@ func TestBloomFilterForDict(t *testing.T) {
 		t.Error("bloom filter should have contained 'test'")
 	}
 }
+
+func TestWriterRepeatedUUIDDict(t *testing.T) {
+	inputID := uuid.MustParse("123456ab-0000-0000-0000-000000000000")
+	records := []struct {
+		List []uuid.UUID `parquet:"list,dict"`
+	}{{
+		[]uuid.UUID{inputID},
+	}}
+	schema := parquet.SchemaOf(&records[0])
+	b := bytes.NewBuffer(nil)
+	w := parquet.NewWriter(b, schema)
+	err := w.Write(records[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := parquet.OpenFile(bytes.NewReader(b.Bytes()), int64(b.Len()))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rows := f.RowGroup(0).Rows()
+	row, err := rows.ReadRow(nil)
+	if err != nil {
+		t.Fatalf("reading row from parquet file: %v", err)
+	}
+	if len(row) != 1 {
+		t.Errorf("expected 1 value in row, got %d", len(row))
+	}
+	if !bytes.Equal(inputID[:], row[0].Bytes()) {
+		t.Errorf("expected to get UUID %q back out, got %q", inputID, row[0].Bytes())
+	}
+}