Repository: apache/iceberg-go
Pull request #319 (2025-03-05 17:02:20)

feat(table): Simple location provider
Add the location provider interface and simple implementation


Patch:
diff --git a/catalog/catalog.go b/catalog/catalog.go
index a241110..6b2118f 100644
--- a/catalog/catalog.go
+++ b/catalog/catalog.go
@@ -37,6 +37,7 @@ import (
 
 	"github.com/apache/iceberg-go"
 	"github.com/apache/iceberg-go/catalog/internal"
+	iceinternal "github.com/apache/iceberg-go/internal"
 	"github.com/apache/iceberg-go/table"
 )
 
@@ -196,7 +197,7 @@ func getUpdatedPropsAndUpdateSummary(currentProps iceberg.Properties, removals [
 	summary := PropertiesUpdateSummary{
 		Removed: removed,
 		Updated: updated,
-		Missing: iceberg.Difference(removals, removed),
+		Missing: iceinternal.Difference(removals, removed),
 	}
 
 	return updatedProps, summary, nil
diff --git a/catalog/glue/glue.go b/catalog/glue/glue.go
index ae6abba..b6e2849 100644
--- a/catalog/glue/glue.go
+++ b/catalog/glue/glue.go
@@ -216,7 +216,7 @@ func (c *Catalog) LoadTable(ctx context.Context, identifier table.Identifier, pr
 		return nil, fmt.Errorf("failed to load table %s.%s: %w", database, tableName, err)
 	}
 
-	icebergTable, err := table.NewFromLocation([]string{tableName}, location, iofs)
+	icebergTable, err := table.NewFromLocation([]string{tableName}, location, iofs, c)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create table from location %s.%s: %w", database, tableName, err)
 	}
diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go
index 1950dfb..e7d1acb 100644
--- a/catalog/rest/rest.go
+++ b/catalog/rest/rest.go
@@ -653,7 +653,7 @@ func (r *Catalog) tableFromResponse(ctx context.Context, identifier []string, me
 		return nil, err
 	}
 
-	return table.New(id, metadata, loc, iofs), nil
+	return table.New(id, metadata, loc, iofs, r), nil
 }
 
 func (r *Catalog) ListTables(ctx context.Context, namespace table.Identifier) iter.Seq2[table.Identifier, error] {
diff --git a/catalog/sql/sql.go b/catalog/sql/sql.go
index 54c4ca6..972116b 100644
--- a/catalog/sql/sql.go
+++ b/catalog/sql/sql.go
@@ -389,7 +389,7 @@ func (c *Catalog) LoadTable(ctx context.Context, identifier table.Identifier, pr
 		return nil, err
 	}
 
-	return table.NewFromLocation(identifier, result.MetadataLocation.String, iofs)
+	return table.NewFromLocation(identifier, result.MetadataLocation.String, iofs, c)
 }
 
 func (c *Catalog) DropTable(ctx context.Context, identifier table.Identifier) error {
diff --git a/internal/utils.go b/internal/utils.go
new file mode 100644
index 0000000..ced65cb
--- /dev/null
+++ b/internal/utils.go
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package internal
+
+// Helper function to find the difference between two slices (a - b).
+func Difference(a, b []string) []string {
+	m := make(map[string]bool)
+	for _, item := range b {
+		m[item] = true
+	}
+
+	diff := make([]string, 0)
+	for _, item := range a {
+		if !m[item] {
+			diff = append(diff, item)
+		}
+	}
+
+	return diff
+}
diff --git a/table/locations.go b/table/locations.go
new file mode 100644
index 0000000..a95ef78
--- /dev/null
+++ b/table/locations.go
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/apache/iceberg-go"
+	"github.com/google/uuid"
+)
+
+type LocationProvider interface {
+	NewTableMetadataFileLocation(newVersion int) (string, error)
+	NewMetadataLocation(metadataFileName string) string
+}
+
+type simpleLocationProvider struct {
+	tableLoc     *url.URL
+	tableProps   iceberg.Properties
+	dataPath     *url.URL
+	metadataPath *url.URL
+}
+
+func (slp *simpleLocationProvider) NewTableMetadataFileLocation(newVersion int) (string, error) {
+	if newVersion < 0 {
+		return "", fmt.Errorf("%w: table metadata version %d must be a non-negative integer",
+			iceberg.ErrInvalidArgument, newVersion)
+	}
+
+	newUUID, err := uuid.NewRandom()
+	if err != nil {
+		return "", err
+	}
+
+	fname := fmt.Sprintf("%05d-%s.metadata.json", newVersion, newUUID)
+
+	return slp.NewMetadataLocation(fname), nil
+}
+
+func (slp *simpleLocationProvider) NewMetadataLocation(metadataFileName string) string {
+	return slp.metadataPath.JoinPath(metadataFileName).String()
+}
+
+func newSimpleLocationProvider(tableLoc *url.URL, tableProps iceberg.Properties) (*simpleLocationProvider, error) {
+	out := &simpleLocationProvider{
+		tableLoc:   tableLoc,
+		tableProps: tableProps,
+	}
+
+	var err error
+	if propPath, ok := tableProps[WriteDataPathKey]; ok {
+		out.dataPath, err = url.Parse(propPath)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		out.dataPath = out.tableLoc.JoinPath("data")
+	}
+
+	if propPath, ok := tableProps[WriteMetadataPathKey]; ok {
+		out.metadataPath, err = url.Parse(propPath)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		out.metadataPath = out.tableLoc.JoinPath("metadata")
+	}
+
+	return out, nil
+}
+
+type objectStoreLocationProvider struct {
+	*simpleLocationProvider
+
+	includePartitionPaths bool
+}
+
+func newObjectStoreLocationProvider(tableLoc *url.URL, tableProps iceberg.Properties) (*objectStoreLocationProvider, error) {
+	slp, err := newSimpleLocationProvider(tableLoc, tableProps)
+	if err != nil {
+		return nil, err
+	}
+
+	return &objectStoreLocationProvider{
+		simpleLocationProvider: slp,
+		includePartitionPaths: tableProps.GetBool(WriteObjectStorePartitionedPathsKey,
+			WriteObjectStorePartitionedPathsDefault),
+	}, nil
+}
+
+func LoadLocationProvider(tableLocation string, tableProps iceberg.Properties) (LocationProvider, error) {
+	u, err := url.Parse(tableLocation)
+	if err != nil {
+		return nil, err
+	}
+
+	if tableProps.GetBool(ObjectStoreEnabledKey, ObjectStoreEnabledDefault) {
+		return newObjectStoreLocationProvider(u, tableProps)
+	}
+
+	return newSimpleLocationProvider(u, tableProps)
+}
diff --git a/table/properties.go b/table/properties.go
new file mode 100644
index 0000000..6c1d688
--- /dev/null
+++ b/table/properties.go
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+const (
+	WriteDataPathKey                        = "write.data.path"
+	WriteMetadataPathKey                    = "write.metadata.path"
+	WriteObjectStorePartitionedPathsKey     = "write.object-storage.partitioned-paths"
+	WriteObjectStorePartitionedPathsDefault = true
+	ObjectStoreEnabledKey                   = "write.object-storage.enabled"
+	ObjectStoreEnabledDefault               = false
+)
diff --git a/table/requirements.go b/table/requirements.go
index 57cac63..3ea3255 100644
--- a/table/requirements.go
+++ b/table/requirements.go
@@ -41,6 +41,7 @@ const (
 type Requirement interface {
 	// Validate checks that the current table metadata satisfies the requirement.
 	Validate(Metadata) error
+	GetType() string
 }
 
 // baseRequirement is a common struct that all requirements embed. It is used to
@@ -49,6 +50,10 @@ type baseRequirement struct {
 	Type string `json:"type"`
 }
 
+func (b baseRequirement) GetType() string {
+	return b.Type
+}
+
 type assertCreate struct {
 	baseRequirement
 }
@@ -123,16 +128,17 @@ func (a *assertRefSnapshotID) Validate(meta Metadata) error {
 			break
 		}
 	}
-	if r == nil {
-		return fmt.Errorf("requirement failed: branch or tag %s is missing, expected %d", a.Ref, a.SnapshotID)
-	}
 
-	if a.SnapshotID == nil {
-		return fmt.Errorf("requirement failed: %s %s was created concurrently", r.SnapshotRefType, a.Ref)
-	}
+	if r != nil {
+		if a.SnapshotID == nil {
+			return fmt.Errorf("requirement failed: %s %s was created concurrently", r.SnapshotRefType, a.Ref)
+		}
 
-	if r.SnapshotID != *a.SnapshotID {
-		return fmt.Errorf("requirement failed: %s %s has changed: expected id %d, found %d", r.SnapshotRefType, a.Ref, a.SnapshotID, r.SnapshotID)
+		if r.SnapshotID != *a.SnapshotID {
+			return fmt.Errorf("requirement failed: %s %s has changed: expected id %d, found %d", r.SnapshotRefType, a.Ref, a.SnapshotID, r.SnapshotID)
+		}
+	} else if a.SnapshotID != nil {
+		return fmt.Errorf("requirement failed: branch or tag %s is missing, expected %d", a.Ref, a.SnapshotID)
 	}
 
 	return nil
diff --git a/table/table.go b/table/table.go
index 753e33a..b6b7786 100644
--- a/table/table.go
+++ b/table/table.go
@@ -18,6 +18,7 @@
 package table
 
 import (
+	"context"
 	"runtime"
 	"slices"
 
@@ -27,11 +28,17 @@ import (
 
 type Identifier = []string
 
+type CatalogIO interface {
+	LoadTable(context.Context, Identifier, iceberg.Properties) (*Table, error)
+	CommitTable(context.Context, *Table, []Requirement, []Update) (Metadata, string, error)
+}
+
 type Table struct {
 	identifier       Identifier
 	metadata         Metadata
 	metadataLocation string
 	fs               io.IO
+	cat              CatalogIO
 }
 
 func (t Table) Equals(other Table) bool {
@@ -40,11 +47,10 @@ func (t Table) Equals(other Table) bool {
 		t.metadata.Equals(other.metadata)
 }
 
-func (t Table) Identifier() Identifier   { return t.identifier }
-func (t Table) Metadata() Metadata       { return t.metadata }
-func (t Table) MetadataLocation() string { return t.metadataLocation }
-func (t Table) FS() io.IO                { return t.fs }
-
+func (t Table) Identifier() Identifier               { return t.identifier }
+func (t Table) Metadata() Metadata                   { return t.metadata }
+func (t Table) MetadataLocation() string             { return t.metadataLocation }
+func (t Table) FS() io.IO                            { return t.fs }
 func (t Table) Schema() *iceberg.Schema              { return t.metadata.CurrentSchema() }
 func (t Table) Spec() iceberg.PartitionSpec          { return t.metadata.PartitionSpec() }
 func (t Table) SortOrder() SortOrder                 { return t.metadata.SortOrder() }
@@ -62,6 +68,10 @@ func (t Table) Schemas() map[int]*iceberg.Schema {
 	return m
 }
 
+func (t Table) LocationProvider() (LocationProvider, error) {
+	return LoadLocationProvider(t.metadataLocation, t.metadata.Properties())
+}
+
 type ScanOption func(*Scan)
 
 func noopOption(*Scan) {}
@@ -154,16 +164,17 @@ func (t Table) Scan(opts ...ScanOption) *Scan {
 	return s
 }
 
-func New(ident Identifier, meta Metadata, location string, fs io.IO) *Table {
+func New(ident Identifier, meta Metadata, location string, fs io.IO, cat CatalogIO) *Table {
 	return &Table{
 		identifier:       ident,
 		metadata:         meta,
 		metadataLocation: location,
 		fs:               fs,
+		cat:              cat,
 	}
 }
 
-func NewFromLocation(ident Identifier, metalocation string, fsys io.IO) (*Table, error) {
+func NewFromLocation(ident Identifier, metalocation string, fsys io.IO, cat CatalogIO) (*Table, error) {
 	var meta Metadata
 
 	if rf, ok := fsys.(io.ReadFileIO); ok {
@@ -187,5 +198,5 @@ func NewFromLocation(ident Identifier, metalocation string, fsys io.IO) (*Table,
 		}
 	}
 
-	return New(ident, meta, metalocation, fsys), nil
+	return New(ident, meta, metalocation, fsys, cat), nil
 }
diff --git a/utils.go b/utils.go
index b35b768..4fae972 100644
--- a/utils.go
+++ b/utils.go
@@ -206,20 +206,3 @@ func (l literalSet) All(fn func(Literal) bool) bool {
 
 	return true
 }
-
-// Helper function to find the difference between two slices (a - b).
-func Difference(a, b []string) []string {
-	m := make(map[string]bool)
-	for _, item := range b {
-		m[item] = true
-	}
-
-	diff := make([]string, 0)
-	for _, item := range a {
-		if !m[item] {
-			diff = append(diff, item)
-		}
-	}
-
-	return diff
-}


Test patch:
diff --git a/cmd/iceberg/output_test.go b/cmd/iceberg/output_test.go
index 8a79d0c..e32cb5a 100644
--- a/cmd/iceberg/output_test.go
+++ b/cmd/iceberg/output_test.go
@@ -188,7 +188,7 @@ func TestDescribeTable(t *testing.T) {
 
 	for _, tt := range testArgs {
 		meta, _ := table.ParseMetadataBytes([]byte(tt.meta))
-		table := table.New([]string{"t"}, meta, "", nil)
+		table := table.New([]string{"t"}, meta, "", nil, nil)
 		buf.Reset()
 
 		text{}.DescribeTable(table)
diff --git a/utils_test.go b/internal/utils_test.go
similarity index 95%
rename from utils_test.go
rename to internal/utils_test.go
index 9f4f888..a90b3eb 100644
--- a/utils_test.go
+++ b/internal/utils_test.go
@@ -15,11 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package iceberg
+package internal_test
 
 import (
 	"testing"
 
+	"github.com/apache/iceberg-go/internal"
 	"github.com/stretchr/testify/require"
 )
 
@@ -71,7 +72,7 @@ func TestDifference(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			assert := require.New(t)
-			result := Difference(tt.a, tt.b)
+			result := internal.Difference(tt.a, tt.b)
 			assert.ElementsMatch(tt.expected, result)
 		})
 	}
diff --git a/table/locations_test.go b/table/locations_test.go
new file mode 100644
index 0000000..2ef2e53
--- /dev/null
+++ b/table/locations_test.go
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/apache/iceberg-go"
+	"github.com/apache/iceberg-go/table"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLocationProviderMetadataDefaultLocation(t *testing.T) {
+	provider, err := table.LoadLocationProvider("table_location", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "table_location/metadata/manifest.avro", provider.NewMetadataLocation("manifest.avro"))
+}
+
+func TestLocationProviderMetadataLocationCustomPath(t *testing.T) {
+	provider, err := table.LoadLocationProvider("table_location",
+		iceberg.Properties{table.WriteMetadataPathKey: "s3://table-location/custom/path"})
+	require.NoError(t, err)
+
+	assert.Equal(t, "s3://table-location/custom/path/metadata.json", provider.NewMetadataLocation("metadata.json"))
+}
+
+func TestLocationProviderMetadataLocationTrailingSlash(t *testing.T) {
+	provider, err := table.LoadLocationProvider("table_location",
+		iceberg.Properties{table.WriteMetadataPathKey: "s3://table-location/custom/path/"})
+	require.NoError(t, err)
+
+	assert.Equal(t, "s3://table-location/custom/path/metadata.json", provider.NewMetadataLocation("metadata.json"))
+}
+
+func TestLocationProviderMetadataFileLocation(t *testing.T) {
+	uuid.SetRand(strings.NewReader("0123456789abcdefghijkl"))
+	defer uuid.SetRand(nil)
+
+	provider, err := table.LoadLocationProvider("table_location", nil)
+	require.NoError(t, err)
+
+	loc, err := provider.NewTableMetadataFileLocation(1)
+	require.NoError(t, err)
+	assert.Equal(t, "table_location/metadata/00001-30313233-3435-4637-b839-616263646566.metadata.json", loc)
+}
+
+func TestLocationProviderMetadataFileLocationCustomPath(t *testing.T) {
+	uuid.SetRand(strings.NewReader("0123456789abcdefghijkl"))
+	defer uuid.SetRand(nil)
+
+	provider, err := table.LoadLocationProvider("table_location",
+		iceberg.Properties{table.WriteMetadataPathKey: "s3://table-location/custom/path/"})
+	require.NoError(t, err)
+
+	loc, err := provider.NewTableMetadataFileLocation(1)
+	require.NoError(t, err)
+	assert.Equal(t, "s3://table-location/custom/path/00001-30313233-3435-4637-b839-616263646566.metadata.json", loc)
+}
diff --git a/table/table_test.go b/table/table_test.go
index f09054c..f139ae2 100644
--- a/table/table_test.go
+++ b/table/table_test.go
@@ -44,7 +44,7 @@ func (t *TableTestSuite) SetupSuite() {
 		Return(&internal.MockFile{Contents: bytes.NewReader([]byte(table.ExampleTableMetadataV2))}, nil)
 	defer mockfs.AssertExpectations(t.T())
 
-	tbl, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfs)
+	tbl, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfs, nil)
 	t.Require().NoError(err)
 	t.Require().NotNil(tbl)
 
@@ -62,7 +62,7 @@ func (t *TableTestSuite) TestNewTableFromReadFile() {
 		Return([]byte(table.ExampleTableMetadataV2), nil)
 	defer mockfsReadFile.AssertExpectations(t.T())
 
-	tbl2, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfsReadFile)
+	tbl2, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfsReadFile, nil)
 	t.Require().NoError(err)
 	t.Require().NotNil(tbl2)