Repository: apache/iceberg-go
Pull request #316 (2025-03-01 19:08:08)

feat(partitions): Add PartitionToPath method to Specs
This is necessary for snapshot creation and new datafiles/manifests for writing.

zeroshade: Yup! That's what it's based on

Patch:
diff --git a/partitions.go b/partitions.go
index ebf97ff..5bb0660 100644
--- a/partitions.go
+++ b/partitions.go
@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"iter"
+	"net/url"
+	"path"
 	"slices"
 	"strings"
 )
@@ -236,6 +238,27 @@ func (ps *PartitionSpec) PartitionType(schema *Schema) *StructType {
 	return &StructType{FieldList: nestedFields}
 }
 
+// PartitionToPath produces a proper partition path from the data and schema by
+// converting the values to human readable strings and properly escaping.
+//
+// The path will be in the form of `name1=value1/name2=value2/...`.
+//
+// This does not apply the transforms to the data, it is assumed the provided data
+// has already been transformed appropriately.
+func (ps *PartitionSpec) PartitionToPath(data structLike, sc *Schema) string {
+	partType := ps.PartitionType(sc)
+
+	segments := make([]string, 0, len(partType.FieldList))
+	for i := range partType.Fields() {
+		valueStr := ps.fields[i].Transform.ToHumanStr(data.Get(i))
+
+		segments = append(segments, fmt.Sprintf("%s=%s",
+			url.QueryEscape(ps.fields[i].Name), url.QueryEscape(valueStr)))
+	}
+
+	return path.Join(segments...)
+}
+
 // AssignFreshPartitionSpecIDs creates a new PartitionSpec by reassigning the field IDs
 // from the old schema to the corresponding fields in the fresh schema, while re-assigning
 // the actual Spec IDs to 1000 + the position of the field in the partition spec.
diff --git a/transforms.go b/transforms.go
index 477ef18..71614a0 100644
--- a/transforms.go
+++ b/transforms.go
@@ -19,6 +19,7 @@ package iceberg
 
 import (
 	"encoding"
+	"encoding/base64"
 	"encoding/binary"
 	"fmt"
 	"math"
@@ -86,6 +87,8 @@ type Transform interface {
 	Equals(Transform) bool
 	Apply(Optional[Literal]) Optional[Literal]
 	Project(name string, pred BoundPredicate) (UnboundPredicate, error)
+
+	ToHumanStr(any) string
 }
 
 // IdentityTransform uses the identity function, performing no transformation
@@ -109,6 +112,25 @@ func (IdentityTransform) Apply(value Optional[Literal]) Optional[Literal] {
 	return value
 }
 
+func (IdentityTransform) ToHumanStr(val any) string {
+	switch v := val.(type) {
+	case nil:
+		return "null"
+	case []byte:
+		return base64.StdEncoding.EncodeToString(v)
+	case bool:
+		return strconv.FormatBool(v)
+	case Date:
+		return v.ToTime().Format("2006-01-02")
+	case Time:
+		return v.ToTime().Format("15:04:05.999999")
+	case Timestamp:
+		return v.ToTime().Format("2006-01-02T15:04:05.999999")
+	default:
+		return fmt.Sprintf("%v", v)
+	}
+}
+
 func (t IdentityTransform) Project(name string, pred BoundPredicate) (UnboundPredicate, error) {
 	if _, ok := pred.Term().(*BoundTransform); ok {
 		return projectTransformPredicate(t, name, pred)
@@ -146,6 +168,8 @@ func (VoidTransform) Apply(value Optional[Literal]) Optional[Literal] {
 	return Optional[Literal]{}
 }
 
+func (VoidTransform) ToHumanStr(any) string { return "null" }
+
 func (VoidTransform) Project(string, BoundPredicate) (UnboundPredicate, error) {
 	return nil, nil
 }
@@ -273,6 +297,14 @@ func (t BucketTransform) Transformer(src Type) func(any) Optional[int32] {
 	}
 }
 
+func (BucketTransform) ToHumanStr(val any) string {
+	if val == nil {
+		return "null"
+	}
+
+	return fmt.Sprintf("%v", val)
+}
+
 func (t BucketTransform) Project(name string, pred BoundPredicate) (UnboundPredicate, error) {
 	if _, ok := pred.Term().(*BoundTransform); ok {
 		return projectTransformPredicate(t, name, pred)
@@ -399,6 +431,17 @@ func (t TruncateTransform) Apply(value Optional[Literal]) (out Optional[Literal]
 	return
 }
 
+func (TruncateTransform) ToHumanStr(val any) string {
+	switch v := val.(type) {
+	case nil:
+		return "null"
+	case []byte:
+		return base64.StdEncoding.EncodeToString(v)
+	default:
+		return fmt.Sprintf("%v", v)
+	}
+}
+
 func (t TruncateTransform) Project(name string, pred BoundPredicate) (UnboundPredicate, error) {
 	if _, ok := pred.Term().(*BoundTransform); ok {
 		return projectTransformPredicate(t, name, pred)
@@ -545,6 +588,15 @@ func (YearTransform) Apply(value Optional[Literal]) (out Optional[Literal]) {
 	return
 }
 
+func (YearTransform) ToHumanStr(val any) string {
+	switch v := val.(type) {
+	case int32:
+		return strconv.Itoa(int(v) + epochTM.Year())
+	default:
+		return "null"
+	}
+}
+
 func (t YearTransform) Project(name string, pred BoundPredicate) (UnboundPredicate, error) {
 	return projectTimeTransform(t, name, pred)
 }
@@ -620,6 +672,16 @@ func (MonthTransform) Apply(value Optional[Literal]) (out Optional[Literal]) {
 	return
 }
 
+func (t MonthTransform) ToHumanStr(val any) string {
+	switch v := val.(type) {
+	case int32:
+		tm := epochTM.AddDate(0, int(v), 0)
+		return tm.Format("2006-01")
+	default:
+		return "null"
+	}
+}
+
 func (t MonthTransform) Project(name string, pred BoundPredicate) (UnboundPredicate, error) {
 	return projectTimeTransform(t, name, pred)
 }
@@ -684,6 +746,16 @@ func (DayTransform) Apply(value Optional[Literal]) (out Optional[Literal]) {
 	return
 }
 
+func (DayTransform) ToHumanStr(val any) string {
+	switch v := val.(type) {
+	case int32:
+		tm := epochTM.AddDate(0, 0, int(v))
+		return tm.Format("2006-01-02")
+	default:
+		return "null"
+	}
+}
+
 func (t DayTransform) Project(name string, pred BoundPredicate) (UnboundPredicate, error) {
 	return projectTimeTransform(t, name, pred)
 }
@@ -738,6 +810,16 @@ func (HourTransform) Apply(value Optional[Literal]) (out Optional[Literal]) {
 	return
 }
 
+func (HourTransform) ToHumanStr(val any) string {
+	switch v := val.(type) {
+	case int32:
+		tm := epochTM.Add(time.Duration(v) * time.Hour)
+		return tm.Format("2006-01-02-15")
+	default:
+		return "null"
+	}
+}
+
 func (t HourTransform) Project(name string, pred BoundPredicate) (UnboundPredicate, error) {
 	return projectTimeTransform(t, name, pred)
 }
diff --git a/types.go b/types.go
index cc9f1b0..7330e95 100644
--- a/types.go
+++ b/types.go
@@ -55,6 +55,17 @@ func (p Properties) GetBool(key string, defVal bool) bool {
 	return defVal
 }
 
+func (p Properties) GetInt(key string, defVal int) int {
+	if v, ok := p[key]; ok {
+		i, err := strconv.ParseInt(v, 10, 64)
+		if err != nil {
+			return defVal
+		}
+		return int(i)
+	}
+	return defVal
+}
+
 // Type is an interface representing any of the available iceberg types,
 // such as primitives (int32/int64/etc.) or nested types (list/struct/map).
 type Type interface {
@@ -461,6 +472,10 @@ type Decimal struct {
 	Scale int
 }
 
+func (d Decimal) String() string {
+	return d.Val.ToString(int32(d.Scale))
+}
+
 type PrimitiveType interface {
 	Type
 	primitive()
@@ -547,6 +562,10 @@ func (DateType) String() string { return "date" }
 
 type Time int64
 
+func (t Time) ToTime() time.Time {
+	return time.UnixMicro(int64(t)).UTC()
+}
+
 // TimeType represents a number of microseconds since midnight.
 type TimeType struct{}
 


Test patch:
diff --git a/partitions_test.go b/partitions_test.go
index fd29190..fda7b86 100644
--- a/partitions_test.go
+++ b/partitions_test.go
@@ -141,3 +141,31 @@ func TestPartitionType(t *testing.T) {
 	actual := spec.PartitionType(tableSchemaSimple)
 	assert.Truef(t, expected.Equals(actual), "expected: %s, got: %s", expected, actual)
 }
+
+type partitionRecord []any
+
+func (p partitionRecord) Size() int            { return len(p) }
+func (p partitionRecord) Get(pos int) any      { return p[pos] }
+func (p partitionRecord) Set(pos int, val any) { p[pos] = val }
+
+func TestPartitionSpecToPath(t *testing.T) {
+	schema := iceberg.NewSchema(0,
+		iceberg.NestedField{ID: 1, Name: "str", Type: iceberg.PrimitiveTypes.String},
+		iceberg.NestedField{ID: 2, Name: "other_str", Type: iceberg.PrimitiveTypes.String},
+		iceberg.NestedField{ID: 3, Name: "int", Type: iceberg.PrimitiveTypes.Int32, Required: true})
+
+	spec := iceberg.NewPartitionSpecID(3,
+		iceberg.PartitionField{SourceID: 1, FieldID: 1000,
+			Transform: iceberg.TruncateTransform{Width: 19}, Name: "my#str%bucket"},
+		iceberg.PartitionField{SourceID: 2, FieldID: 1001,
+			Transform: iceberg.IdentityTransform{}, Name: "other str+bucket"},
+		iceberg.PartitionField{SourceID: 3, FieldID: 1002,
+			Transform: iceberg.BucketTransform{NumBuckets: 25}, Name: "my!int:bucket"})
+
+	record := partitionRecord{"my+str", "( )", int32(10)}
+	// both partition field names and values should be URL encoded, with spaces
+	// mapping to plus signs, to match Java behavior:
+	// https://github.com/apache/iceberg/blob/ca3db931b0f024f0412084751ac85dd4ef2da7e7/api/src/main/java/org/apache/iceberg/PartitionSpec.java#L198-L204
+	assert.Equal(t, "my%23str%25bucket=my%2Bstr/other+str%2Bbucket=%28+%29/my%21int%3Abucket=10",
+		spec.PartitionToPath(record, schema))
+}
diff --git a/transforms_test.go b/transforms_test.go
index a455ede..837cf26 100644
--- a/transforms_test.go
+++ b/transforms_test.go
@@ -21,6 +21,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/apache/arrow-go/v18/arrow/decimal"
 	"github.com/apache/iceberg-go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -87,3 +88,53 @@ func TestParseTransform(t *testing.T) {
 		})
 	}
 }
+
+func TestToHumanString(t *testing.T) {
+	decVal, _ := decimal.Decimal128FromString("14.21", 4, 2)
+	negDecVal, _ := decimal.Decimal128FromString("-1.50", 9, 2)
+
+	tests := []struct {
+		transform iceberg.Transform
+		input     any
+		expected  string
+	}{
+		{iceberg.YearTransform{}, int32(47), "2017"},
+		{iceberg.MonthTransform{}, int32(575), "2017-12"},
+		{iceberg.DayTransform{}, int32(17501), "2017-12-01"},
+		{iceberg.YearTransform{}, nil, "null"},
+		{iceberg.MonthTransform{}, nil, "null"},
+		{iceberg.DayTransform{}, nil, "null"},
+		{iceberg.HourTransform{}, nil, "null"},
+		{iceberg.HourTransform{}, int32(420042), "2017-12-01-18"},
+		{iceberg.YearTransform{}, int32(-1), "1969"},
+		{iceberg.MonthTransform{}, int32(-1), "1969-12"},
+		{iceberg.DayTransform{}, int32(-1), "1969-12-31"},
+		{iceberg.HourTransform{}, int32(-1), "1969-12-31-23"},
+		{iceberg.YearTransform{}, int32(0), "1970"},
+		{iceberg.MonthTransform{}, int32(0), "1970-01"},
+		{iceberg.DayTransform{}, int32(0), "1970-01-01"},
+		{iceberg.HourTransform{}, int32(0), "1970-01-01-00"},
+		{iceberg.VoidTransform{}, nil, "null"},
+		{iceberg.IdentityTransform{}, nil, "null"},
+		{iceberg.TruncateTransform{Width: 1}, []byte{0x00, 0x01, 0x02, 0x03}, "AAECAw=="},
+		{iceberg.TruncateTransform{Width: 1}, iceberg.Decimal{
+			Val: decVal, Scale: 2}, "14.21"},
+		{iceberg.TruncateTransform{Width: 1}, int32(123), "123"},
+		{iceberg.TruncateTransform{Width: 1}, int64(123), "123"},
+		{iceberg.TruncateTransform{Width: 1}, "foo", "foo"},
+		{iceberg.IdentityTransform{}, nil, "null"},
+		{iceberg.IdentityTransform{}, iceberg.Date(17501), "2017-12-01"},
+		{iceberg.IdentityTransform{}, iceberg.Time(36775038194), "10:12:55.038194"},
+		{iceberg.IdentityTransform{}, iceberg.Timestamp(1512151975038194), "2017-12-01T18:12:55.038194"},
+		{iceberg.IdentityTransform{}, int64(-1234567890000), "-1234567890000"},
+		{iceberg.IdentityTransform{}, "a/b/c=d", "a/b/c=d"},
+		{iceberg.IdentityTransform{}, []byte("foo"), "Zm9v"},
+		{iceberg.IdentityTransform{}, iceberg.Decimal{Val: negDecVal, Scale: 2}, "-1.50"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.expected, func(t *testing.T) {
+			assert.Equal(t, tt.expected, tt.transform.ToHumanStr(tt.input))
+		})
+	}
+}