Repository: polarsignals/frostdb
Pull request #749 (2024-02-20 15:04:56)

help add the logic of creating arrow builder for the type uint64 in Go
Hi, long time no see. Wish you all the best. 
Recently I learned the base knowledge of Apache Arrow and read the code for creating Arrow Builder in your project. I noticed that you are only supporting some primitive types in Go, and these types are int64, float64, string, and bool. 
So I would like to help implement builders for other primitive types in Go, such as uint64, uint8, etc.
I also noticed that we should define the values of enums in protobuf, and the type uint64 has already been defined there. So this PR is just a case for what I want to do, and it implements the arrow builder for the type uint64. 
Would like to wait for your opinions or concerns, thanks!

elliotchenzichang: Thanks a lot. @gernest 
elliotchenzichang: Hi @gernest could we merge this PR first if no concerns？ Thanks！

Patch:
diff --git a/internal/records/record_builder.go b/internal/records/record_builder.go
index 44745d1..6005aca 100644
--- a/internal/records/record_builder.go
+++ b/internal/records/record_builder.go
@@ -202,7 +202,7 @@ func NewBuild[T any](mem memory.Allocator) *Build[T] {
 				typ = arrow.ListOf(typ)
 				fr.build = newFieldBuild(typ, mem, name, true)
 			}
-		case reflect.Int64, reflect.Float64, reflect.Bool, reflect.String:
+		case reflect.Int64, reflect.Float64, reflect.Bool, reflect.String, reflect.Uint64:
 			typ, styp = baseType(fty, dictionary)
 			fr.typ = styp
 			fr.nullable = nullable
@@ -510,6 +510,9 @@ func baseType(fty reflect.Type, dictionary bool) (typ arrow.DataType, sty schema
 	case reflect.String:
 		typ = arrow.BinaryTypes.String
 		sty = schemapb.StorageLayout_TYPE_STRING
+	case reflect.Uint64:
+		typ = arrow.PrimitiveTypes.Uint64
+		sty = schemapb.StorageLayout_TYPE_UINT64
 	default:
 		panic("frostdb/dynschema: " + fty.String() + " is npt supported")
 	}
@@ -569,6 +572,29 @@ func newFieldBuild(dt arrow.DataType, mem memory.Allocator, name string, nullabl
 			}
 			return e.Append(v.Int())
 		}
+	case *array.Uint64Builder:
+		f.buildFunc = func(v reflect.Value) error {
+			if nullable {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				v = v.Elem()
+			}
+			e.Append(v.Uint())
+			return nil
+		}
+	case *array.Uint64DictionaryBuilder:
+		f.buildFunc = func(v reflect.Value) error {
+			if nullable {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				v = v.Elem()
+			}
+			return e.Append(v.Uint())
+		}
 	case *array.Float64Builder:
 		f.buildFunc = func(v reflect.Value) error {
 			if nullable {
@@ -652,7 +678,19 @@ func newFieldBuild(dt arrow.DataType, mem memory.Allocator, name string, nullabl
 				build.Reserve(v.Len())
 				return applyInt(v, build.Append)
 			}
-
+		case *array.Uint64Builder:
+			f.buildFunc = func(v reflect.Value) error {
+				if v.IsNil() {
+					e.AppendNull()
+					return nil
+				}
+				e.Append(true)
+				build.Reserve(v.Len())
+				return applyUInt(v, func(i uint64) error {
+					build.Append(i)
+					return nil
+				})
+			}
 		case *array.Float64Builder:
 			f.buildFunc = func(v reflect.Value) error {
 				if v.IsNil() {
@@ -744,6 +782,12 @@ func applyInt(v reflect.Value, apply func(int64) error) error {
 	}, apply)
 }
 
+func applyUInt(v reflect.Value, apply func(uint64) error) error {
+	return listApply[uint64](v, func(v reflect.Value) uint64 {
+		return v.Uint()
+	}, apply)
+}
+
 func listApply[T any](v reflect.Value, fn func(reflect.Value) T, apply func(T) error) error {
 	for i := 0; i < v.Len(); i++ {
 		err := apply(fn(v.Index(i)))


Test patch:
diff --git a/internal/records/record_builder_test.go b/internal/records/record_builder_test.go
index 3d9413b..f17c234 100644
--- a/internal/records/record_builder_test.go
+++ b/internal/records/record_builder_test.go
@@ -53,6 +53,7 @@ func TestBuild(t *testing.T) {
 			Bool       []bool
 			String     []string
 			StringDict []string `frostdb:",rle_dict"`
+			Uint64     []uint64
 		}
 		b := records.NewBuild[Repeated](memory.DefaultAllocator)
 		defer b.Release()
@@ -100,6 +101,14 @@ func TestBuild(t *testing.T) {
         "nullable": true,
         "repeated": true
       }
+    },
+ 	{
+      "name": "uint64",
+      "storageLayout": {
+        "type": "TYPE_UINT64",
+        "nullable": true,
+        "repeated": true
+      }
     }
   ]
 }`
@@ -115,6 +124,7 @@ func TestBuild(t *testing.T) {
 				Bool:       []bool{true, true},
 				String:     []string{"a", "b"},
 				StringDict: []string{"a", "b"},
+				Uint64:     []uint64{1, 2},
 			},
 			Repeated{
 				Int:        []int64{1, 2},
@@ -122,12 +132,13 @@ func TestBuild(t *testing.T) {
 				Bool:       []bool{true, true},
 				String:     []string{"a", "b"},
 				StringDict: []string{"c", "d"},
+				Uint64:     []uint64{1, 2},
 			},
 		)
 		require.Nil(t, err)
-		want := `[{"bool":null,"float":null,"int":null,"string":null,"string_dict":null}
-,{"bool":[true,true],"float":[1,2],"int":[1,2],"string":["a","b"],"string_dict":["a","b"]}
-,{"bool":[true,true],"float":[1,2],"int":[1,2],"string":["a","b"],"string_dict":["c","d"]}
+		want := `[{"bool":null,"float":null,"int":null,"string":null,"string_dict":null, "uint64":null}
+,{"bool":[true,true],"float":[1,2],"int":[1,2],"string":["a","b"],"string_dict":["a","b"],"uint64":[1, 2]}
+,{"bool":[true,true],"float":[1,2],"int":[1,2],"string":["a","b"],"string_dict":["c","d"],"uint64":[1, 2]}
 ]`
 		r := b.NewRecord()
 		data, _ := r.MarshalJSON()