Repository: polarsignals/frostdb
Pull request #624 (2023-12-12 19:22:21)

Support repeated columns for Generic Record Builder
Support repeated columns for all base types `[]int64` `[]float64`  `[]bool` `[]string`.   Use all supported tags on this column like other base columns( set encoding ,compression etc)

Since we deal with slices I took the liberty to make repeated fields nullable by default. 

Example

```go
		type Example struct {
			StringDict []string `frostdb:",rle_dict"`
		}
```


Patch:
diff --git a/dynparquet/record_builder.go b/dynparquet/record_builder.go
index bbdb9ba..1fe437e 100644
--- a/dynparquet/record_builder.go
+++ b/dynparquet/record_builder.go
@@ -29,8 +29,6 @@ const (
 //
 //   - Dynamic columns are supported but only for  maps and struct slices
 //
-//   - Repeated Columns are not supported
-//
 //   - Nested Columns are not supported
 //
 // # Tags
@@ -87,6 +85,15 @@ const (
 //		// Use supported tags to customize the column value
 //		Labels []Label `frostdb:"labels,dyn"`
 //	}
+//
+// # Repeated columns
+//
+// Fields of type []int64, []float64, []bool, and []string are supported. These
+// are represented as arrow.LIST.
+//
+// Generated schema for the repeated columns applies all supported tags. By
+// default repeated fields are nullable. You can safely pass nil slices for
+// repeated columns.
 type Build[T any] struct {
 	fields []*fieldRecord
 	buffer []arrow.Array
@@ -208,7 +215,13 @@ func NewBuild[T any](mem memory.Allocator) *Build[T] {
 				fr.nullable = true
 				fr.build = newMapFieldBuilder(newFieldFunc(typ, mem, name))
 			default:
-				panic("frostdb/dynschema: repeated columns not supported ")
+				typ, styp = baseType(fty.Elem(), dictionary)
+				fr.typ = styp
+				fr.repeated = true
+				// Repeated columns are always nullable
+				fr.nullable = true
+				typ = arrow.ListOf(typ)
+				fr.build = newFieldBuild(typ, mem, name, true)
 			}
 		case reflect.Int64, reflect.Float64, reflect.Bool, reflect.String:
 			typ, styp = baseType(fty, dictionary)
@@ -270,6 +283,7 @@ func (b Build[T]) Schema(name string) (s *schemapb.Schema) {
 				Encoding:    f.encoding,
 				Compression: f.compression,
 				Nullable:    f.nullable,
+				Repeated:    f.repeated,
 			},
 		})
 		if f.sort {
@@ -530,12 +544,133 @@ func newFieldBuild(dt arrow.DataType, mem memory.Allocator, name string, nullabl
 		f.buildFunc = func(v reflect.Value) error {
 			return e.AppendString(v.Interface().(string))
 		}
+	case *array.ListBuilder:
+		switch build := e.ValueBuilder().(type) {
+		case *array.Int64Builder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyInt(v, func(i int64) error {
+					build.Append(i)
+					return nil
+				})
+			}
+		case *array.Int64DictionaryBuilder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyInt(v, build.Append)
+			}
+
+		case *array.Float64Builder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyFloat64(v, func(i float64) error {
+					build.Append(i)
+					return nil
+				})
+			}
+		case *array.Float64DictionaryBuilder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyFloat64(v, build.Append)
+			}
+
+		case *array.StringBuilder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyString(v, func(i string) error {
+					build.Append(i)
+					return nil
+				})
+			}
+		case *array.BinaryDictionaryBuilder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.Len() == 0 {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyString(v, build.AppendString)
+			}
+		case *array.BooleanBuilder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyBool(v, func(i bool) error {
+					build.Append(i)
+					return nil
+				})
+			}
+		}
 	default:
 		panic("frostdb:dynschema: unsupported array builder " + b.Type().String())
 	}
 	return
 }
 
+func applyString(v reflect.Value, apply func(string) error) error {
+	return listApply[string](v, func(v reflect.Value) string {
+		return v.Interface().(string)
+	}, apply)
+}
+
+func applyFloat64(v reflect.Value, apply func(float64) error) error {
+	return listApply[float64](v, func(v reflect.Value) float64 {
+		return v.Float()
+	}, apply)
+}
+
+func applyBool(v reflect.Value, apply func(bool) error) error {
+	return listApply[bool](v, func(v reflect.Value) bool {
+		return v.Bool()
+	}, apply)
+}
+
+func applyInt(v reflect.Value, apply func(int64) error) error {
+	return listApply[int64](v, func(v reflect.Value) int64 {
+		return v.Int()
+	}, apply)
+}
+
+func listApply[T any](v reflect.Value, fn func(reflect.Value) T, apply func(T) error) error {
+	for i := 0; i < v.Len(); i++ {
+		err := apply(fn(v.Index(i)))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func newUUIDSliceField(mem memory.Allocator, name string) (f *fieldBuilderFunc) {
 	dt := &arrow.DictionaryType{
 		IndexType: &arrow.Int32Type{},
@@ -584,6 +719,7 @@ type fieldRecord struct {
 	dynamic     bool
 	preHash     bool
 	nullable    bool
+	repeated    bool
 	sort        bool
 	nullFirst   bool
 	sortOrder   int


Test patch:
diff --git a/dynparquet/record_builder_test.go b/dynparquet/record_builder_test.go
index e047a0d..100b9fc 100644
--- a/dynparquet/record_builder_test.go
+++ b/dynparquet/record_builder_test.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/apache/arrow/go/v14/arrow/memory"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 )
 
@@ -41,6 +42,94 @@ func TestBuild(t *testing.T) {
 		require.Nil(t, err)
 		require.JSONEq(t, want, string(got))
 	})
+
+	t.Run("Repeated", func(t *testing.T) {
+		type Repeated struct {
+			Int        []int64
+			Float      []float64
+			Bool       []bool
+			String     []string
+			StringDict []string `frostdb:",rle_dict"`
+		}
+		b := NewBuild[Repeated](memory.DefaultAllocator)
+		defer b.Release()
+
+		wantSchema := `{
+  "name": "repeated",
+  "columns": [
+    {
+      "name": "int",
+      "storageLayout": {
+        "type": "TYPE_INT64",
+        "nullable": true,
+        "repeated": true
+      }
+    },
+    {
+      "name": "float",
+      "storageLayout": {
+        "type": "TYPE_DOUBLE",
+        "nullable": true,
+        "repeated": true
+      }
+    },
+    {
+      "name": "bool",
+      "storageLayout": {
+        "type": "TYPE_BOOL",
+        "nullable": true,
+        "repeated": true
+      }
+    },
+    {
+      "name": "string",
+      "storageLayout": {
+        "type": "TYPE_STRING",
+        "nullable": true,
+        "repeated": true
+      }
+    },
+    {
+      "name": "string_dict",
+      "storageLayout": {
+        "type": "TYPE_STRING",
+        "encoding": "ENCODING_RLE_DICTIONARY",
+        "nullable": true,
+        "repeated": true
+      }
+    }
+  ]
+}`
+		m := protojson.MarshalOptions{Multiline: true}
+		d, _ := m.Marshal(b.Schema("repeated"))
+		require.JSONEq(t, wantSchema, string(d))
+
+		err := b.Append(
+			Repeated{}, // nulls
+			Repeated{
+				Int:        []int64{1, 2},
+				Float:      []float64{1, 2},
+				Bool:       []bool{true, true},
+				String:     []string{"a", "b"},
+				StringDict: []string{"a", "b"},
+			},
+			Repeated{
+				Int:        []int64{1, 2},
+				Float:      []float64{1, 2},
+				Bool:       []bool{true, true},
+				String:     []string{"a", "b"},
+				StringDict: []string{"c", "d"},
+			},
+		)
+		require.Nil(t, err)
+		want := `[{"bool":null,"float":null,"int":null,"string":null,"string_dict":null}
+,{"bool":[true,true],"float":[1,2],"int":[1,2],"string":["a","b"],"string_dict":["a","b"]}
+,{"bool":[true,true],"float":[1,2],"int":[1,2],"string":["a","b"],"string_dict":["c","d"]}
+]`
+		r := b.NewRecord()
+		data, _ := r.MarshalJSON()
+		require.JSONEq(t, want, string(data))
+	})
 }
 
 func BenchmarkBuild_Append_Then_NewRecord(b *testing.B) {