Repository: apache/iceberg-go
Pull request #169 (2024-10-10 01:56:56)

feat(catalog/glue): add support for list namespaces
Add support for `ListNamespaces` to the Glue catalog.

### Implementation Details

- Similar to tables, used the same `ICEBERG` parameter to differentiate between Iceberg catalog databases and other databases in the Glue catalog.
- Throw an error when `parent` parameter is given to `ListNamespaces` to align with [Python](https://github.com/apache/iceberg-python/blob/main/pyiceberg/catalog/glue.py#L709C11-L709C50) implementation.

### Testing Done

- Tests are added similar to others in the test file.
- Set up AWS Glue Catalog in my personal account and created a database and verified that it can be fetched.


Patch:
diff --git a/catalog/glue.go b/catalog/glue.go
index b398185..91b21ff 100644
--- a/catalog/glue.go
+++ b/catalog/glue.go
@@ -30,7 +30,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/glue/types"
 )
 
-const glueTableTypeIceberg = "ICEBERG"
+const glueTypeIceberg = "ICEBERG"
 
 var (
 	_ Catalog = (*GlueCatalog)(nil)
@@ -39,6 +39,7 @@ var (
 type glueAPI interface {
 	GetTable(ctx context.Context, params *glue.GetTableInput, optFns ...func(*glue.Options)) (*glue.GetTableOutput, error)
 	GetTables(ctx context.Context, params *glue.GetTablesInput, optFns ...func(*glue.Options)) (*glue.GetTablesOutput, error)
+	GetDatabases(ctx context.Context, params *glue.GetDatabasesInput, optFns ...func(*glue.Options)) (*glue.GetDatabasesOutput, error)
 }
 
 type GlueCatalog struct {
@@ -77,7 +78,7 @@ func (c *GlueCatalog) ListTables(ctx context.Context, namespace table.Identifier
 		}
 
 		icebergTables = append(icebergTables,
-			filterTableListByType(database, tblsRes.TableList, glueTableTypeIceberg)...)
+			filterTableListByType(database, tblsRes.TableList, glueTypeIceberg)...)
 
 		if tblsRes.NextToken == nil {
 			break
@@ -150,8 +151,33 @@ func (c *GlueCatalog) UpdateNamespaceProperties(ctx context.Context, namespace t
 	return PropertiesUpdateSummary{}, fmt.Errorf("%w: [Glue Catalog] update namespace properties", iceberg.ErrNotImplemented)
 }
 
+// ListNamespaces returns a list of Iceberg namespaces from the given Glue catalog.
 func (c *GlueCatalog) ListNamespaces(ctx context.Context, parent table.Identifier) ([]table.Identifier, error) {
-	return nil, fmt.Errorf("%w: [Glue Catalog] list namespaces", iceberg.ErrNotImplemented)
+	params := &glue.GetDatabasesInput{}
+
+	if parent != nil {
+		return nil, fmt.Errorf("hierarchical namespace is not supported")
+	}
+
+	var icebergNamespaces []table.Identifier
+
+	for {
+		databasesResp, err := c.glueSvc.GetDatabases(ctx, params)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list databases: %w", err)
+		}
+
+		icebergNamespaces = append(icebergNamespaces,
+			filterDatabaseListByType(databasesResp.DatabaseList, glueTypeIceberg)...)
+
+		if databasesResp.NextToken == nil {
+			break
+		}
+
+		params.NextToken = databasesResp.NextToken
+	}
+
+	return icebergNamespaces, nil
 }
 
 // GetTable loads a table from the Glue Catalog using the given database and table name.
@@ -214,3 +240,16 @@ func filterTableListByType(database string, tableList []types.Table, tableType s
 
 	return filtered
 }
+
+func filterDatabaseListByType(databases []types.Database, databaseType string) []table.Identifier {
+	var filtered []table.Identifier
+
+	for _, database := range databases {
+		if database.Parameters["database_type"] != databaseType {
+			continue
+		}
+		filtered = append(filtered, GlueDatabaseIdentifier(aws.ToString(database.Name)))
+	}
+
+	return filtered
+}


Test patch:
diff --git a/catalog/glue_test.go b/catalog/glue_test.go
index 1d3c42f..5889537 100644
--- a/catalog/glue_test.go
+++ b/catalog/glue_test.go
@@ -44,6 +44,11 @@ func (m *mockGlueClient) GetTables(ctx context.Context, params *glue.GetTablesIn
 	return args.Get(0).(*glue.GetTablesOutput), args.Error(1)
 }
 
+func (m *mockGlueClient) GetDatabases(ctx context.Context, params *glue.GetDatabasesInput, optFns ...func(*glue.Options)) (*glue.GetDatabasesOutput, error) {
+	args := m.Called(ctx, params, optFns)
+	return args.Get(0).(*glue.GetDatabasesOutput), args.Error(1)
+}
+
 func TestGlueGetTable(t *testing.T) {
 	assert := require.New(t)
 
@@ -105,7 +110,37 @@ func TestGlueListTables(t *testing.T) {
 	assert.Equal([]string{"test_database", "test_table"}, tables[0])
 }
 
-func TestGlueListTableIntegration(t *testing.T) {
+func TestGlueListNamespaces(t *testing.T) {
+	assert := require.New(t)
+
+	mockGlueSvc := &mockGlueClient{}
+
+	mockGlueSvc.On("GetDatabases", mock.Anything, &glue.GetDatabasesInput{}, mock.Anything).Return(&glue.GetDatabasesOutput{
+		DatabaseList: []types.Database{
+			{
+				Name: aws.String("test_database"),
+				Parameters: map[string]string{
+					"database_type": "ICEBERG",
+				},
+			},
+			{
+				Name:       aws.String("other_database"),
+				Parameters: map[string]string{},
+			},
+		},
+	}, nil).Once()
+
+	glueCatalog := &GlueCatalog{
+		glueSvc: mockGlueSvc,
+	}
+
+	databases, err := glueCatalog.ListNamespaces(context.TODO(), nil)
+	assert.NoError(err)
+	assert.Len(databases, 1)
+	assert.Equal([]string{"test_database"}, databases[0])
+}
+
+func TestGlueListTablesIntegration(t *testing.T) {
 	if os.Getenv("TEST_DATABASE_NAME") == "" {
 		t.Skip()
 	}
@@ -146,3 +181,19 @@ func TestGlueLoadTableIntegration(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal([]string{os.Getenv("TEST_TABLE_NAME")}, table.Identifier())
 }
+
+func TestGlueListNamespacesIntegration(t *testing.T) {
+	if os.Getenv("TEST_DATABASE_NAME") == "" {
+		t.Skip()
+	}
+	assert := require.New(t)
+
+	awscfg, err := config.LoadDefaultConfig(context.TODO(), config.WithClientLogMode(aws.LogRequest|aws.LogResponse))
+	assert.NoError(err)
+
+	catalog := NewGlueCatalog(WithAwsConfig(awscfg))
+
+	namespaces, err := catalog.ListNamespaces(context.TODO(), nil)
+	assert.NoError(err)
+	assert.Contains(namespaces, []string{os.Getenv("TEST_DATABASE_NAME")})
+}