Repository: polarsignals/frostdb
Pull request #620 (2023-12-08 18:09:44)

Support struct pointers on (*Table).Write
I faced a challenge while writing a struct pointer to the table. I ended up building arrow record by hand and use `(*Table)InsertRecord`.

There is no technical reason why we can't support pointers to struct. We already use `reflect` to process the values.

This commit dereferences pointer values before processing them. Basically passing `Sample` or `(*Sample)` to `(*Table).Write` should behave the same.


Patch:
diff --git a/dynparquet/schema.go b/dynparquet/schema.go
index 0bdda9e..fa6bf37 100644
--- a/dynparquet/schema.go
+++ b/dynparquet/schema.go
@@ -1003,7 +1003,7 @@ func ValuesToBuffer(schema *Schema, vals ...any) (*Buffer, error) {
 
 	// Collect dynamic columns
 	for _, v := range vals {
-		val := reflect.ValueOf(v)
+		val := deref(reflect.ValueOf(v))
 		for _, col := range schema.Columns() {
 			cv := findColumn(val, col.Name, v)
 			if cv == nil {
@@ -1011,16 +1011,16 @@ func ValuesToBuffer(schema *Schema, vals ...any) (*Buffer, error) {
 			}
 			switch col.Dynamic {
 			case true:
-				switch reflect.TypeOf(cv).Kind() {
+				switch derefType(reflect.TypeOf(cv)).Kind() {
 				case reflect.Struct:
-					dynVals := reflect.ValueOf(cv)
+					dynVals := deref(reflect.ValueOf(cv))
 					for j := 0; j < dynVals.NumField(); j++ {
 						dynamicColumns[col.Name] = append(dynamicColumns[col.Name], ToSnakeCase(dynVals.Type().Field(j).Name))
 					}
 				case reflect.Slice:
 					dynVals := reflect.ValueOf(cv)
 					for j := 0; j < dynVals.Len(); j++ {
-						pair := reflect.ValueOf(dynVals.Index(j).Interface())
+						pair := deref(reflect.ValueOf(dynVals.Index(j).Interface()))
 						dynamicColumns[col.Name] = append(dynamicColumns[col.Name], ToSnakeCase(fmt.Sprintf("%v", pair.Field(0))))
 					}
 				case reflect.Map:
@@ -1045,7 +1045,7 @@ func ValuesToBuffer(schema *Schema, vals ...any) (*Buffer, error) {
 	// Create all rows
 	for _, v := range vals {
 		row := []parquet.Value{}
-		val := reflect.ValueOf(v)
+		val := deref(reflect.ValueOf(v))
 
 		colIdx := 0
 		for _, col := range schema.Columns() {
@@ -1055,9 +1055,9 @@ func ValuesToBuffer(schema *Schema, vals ...any) (*Buffer, error) {
 			}
 			switch col.Dynamic {
 			case true:
-				switch reflect.TypeOf(cv).Kind() {
+				switch derefType(reflect.TypeOf(cv)).Kind() {
 				case reflect.Struct:
-					dynVals := reflect.ValueOf(cv)
+					dynVals := deref(reflect.ValueOf(cv))
 					for _, dyncol := range dynamicColumns[col.Name] {
 						found := false
 						for j := 0; j < dynVals.NumField(); j++ {
@@ -1146,6 +1146,20 @@ func ValuesToBuffer(schema *Schema, vals ...any) (*Buffer, error) {
 	return pb, nil
 }
 
+func deref(v reflect.Value) reflect.Value {
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	return v
+}
+
+func derefType(v reflect.Type) reflect.Type {
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	return v
+}
+
 // NewBuffer returns a new buffer with a concrete parquet schema generated
 // using the given concrete dynamic column names.
 func (s *Schema) NewBuffer(dynamicColumns map[string][]string) (*Buffer, error) {


Test patch:
diff --git a/table_test.go b/table_test.go
index 2e43ca0..6ebc0d9 100644
--- a/table_test.go
+++ b/table_test.go
@@ -1209,3 +1209,69 @@ func TestTableUniquePrimaryIndex(t *testing.T) {
 		}))
 	require.Equal(t, 1, rowsRead)
 }
+
+func TestTable_write_ptr_struct(t *testing.T) {
+	// This is taken from simple example and modified to use pointers
+	columnstore, err := New()
+	require.Nil(t, err)
+	defer columnstore.Close()
+
+	database, err := columnstore.DB(context.Background(), "simple_db")
+	require.Nil(t, err)
+
+	schema := &schemapb.Schema{
+		Name: "simple_schema",
+		Columns: []*schemapb.Column{{
+			Name: "names",
+			StorageLayout: &schemapb.StorageLayout{
+				Type:     schemapb.StorageLayout_TYPE_STRING,
+				Encoding: schemapb.StorageLayout_ENCODING_RLE_DICTIONARY,
+				Nullable: true,
+			},
+			Dynamic: true,
+		}, {
+			Name: "value",
+			StorageLayout: &schemapb.StorageLayout{
+				Type: schemapb.StorageLayout_TYPE_INT64,
+			},
+			Dynamic: false,
+		}},
+		SortingColumns: []*schemapb.SortingColumn{{
+			Name:      "names",
+			Direction: schemapb.SortingColumn_DIRECTION_ASCENDING,
+		}},
+	}
+
+	table, err := database.Table(
+		"simple_table",
+		NewTableConfig(schema),
+	)
+	require.Nil(t, err)
+
+	type FirstLast struct {
+		FirstName string
+		Surname   string
+	}
+
+	type Simple struct {
+		Names *FirstLast
+		Value int64
+	}
+	frederic := &Simple{
+		Names: &FirstLast{
+			FirstName: "Frederic",
+			Surname:   "Brancz",
+		},
+		Value: 100,
+	}
+
+	thor := &Simple{
+		Names: &FirstLast{
+			FirstName: "Thor",
+			Surname:   "Hansen",
+		},
+		Value: 99,
+	}
+	_, err = table.Write(context.Background(), frederic, thor)
+	require.Nil(t, err)
+}