Repository: cloudspannerecosystem/harbourbridge
Pull request #224 (2021-11-12 12:24:04)

Dynamodb Support for pg spanner
Added changes to support dynamodb to pg spanner migration.

Manually tested for now.

bharadwaj-aditya: You will have some conflicts based on the other DynamoDb PR merged by Charvi. If there are significant changes, I will take another pass at this

Patch:
diff --git a/sources/dynamodb/data.go b/sources/dynamodb/data.go
index d2740bc..2a6fcb6 100644
--- a/sources/dynamodb/data.go
+++ b/sources/dynamodb/data.go
@@ -49,7 +49,14 @@ func cvtRow(attrsMap map[string]*dynamodb.AttributeValue, srcSchema schema.Table
 			srcStrVal = "null"
 		} else {
 			// Convert data to the target type.
-			spVal, err = cvtColValue(attrsMap[srcCol], srcSchema.ColDefs[srcCol].Type.Name, spSchema.ColDefs[spCols[i]].T.Name)
+			spCol := spCols[i]
+			spColDef := spSchema.ColDefs[spCol]
+			srcColDef := srcSchema.ColDefs[srcCol]
+			if spColDef.T.IsArray {
+				spVal, err = convArray(attrsMap[srcCol], srcColDef.Type.Name, spColDef.T.Name)
+			} else {
+				spVal, err = convScalar(attrsMap[srcCol], srcColDef.Type.Name, spColDef.T.Name)
+			}
 			if err != nil {
 				badCols = append(badCols, srcCol)
 			}
@@ -61,7 +68,46 @@ func cvtRow(attrsMap map[string]*dynamodb.AttributeValue, srcSchema schema.Table
 	return spVals, badCols, srcStrVals
 }
 
-func cvtColValue(attrVal *dynamodb.AttributeValue, srcType string, spType string) (interface{}, error) {
+func convArray(attrVal *dynamodb.AttributeValue, srcType string, spType string) (interface{}, error) {
+	switch spType {
+	case ddl.Bytes:
+		switch srcType {
+		case typeBinarySet:
+			return attrVal.BS, nil
+		}
+	case ddl.String:
+		switch srcType {
+		case typeStringSet:
+			var strArr []string
+			for _, s := range attrVal.SS {
+				strArr = append(strArr, *s)
+			}
+			return strArr, nil
+		case typeNumberStringSet:
+			var strArr []string
+			for _, s := range attrVal.NS {
+				strArr = append(strArr, *s)
+			}
+			return strArr, nil
+		}
+	case ddl.Numeric:
+		switch srcType {
+		case typeNumberSet:
+			var numArr []big.Rat
+			for _, s := range attrVal.NS {
+				val, ok := (&big.Rat{}).SetString(*s)
+				if !ok {
+					return nil, fmt.Errorf("failed to convert '%v' to an NUMERIC array", attrVal.NS)
+				}
+				numArr = append(numArr, *val)
+			}
+			return numArr, nil
+		}
+	}
+	return nil, fmt.Errorf("can't convert value of type %s to Spanner type %s", attrVal.GoString(), spType)
+}
+
+func convScalar(attrVal *dynamodb.AttributeValue, srcType string, spType string) (interface{}, error) {
 	switch spType {
 	case ddl.Bool:
 		switch srcType {
@@ -72,12 +118,14 @@ func cvtColValue(attrVal *dynamodb.AttributeValue, srcType string, spType string
 		switch srcType {
 		case typeBinary:
 			return attrVal.B, nil
-		case typeBinarySet:
-			return attrVal.BS, nil
 		}
 	case ddl.String:
 		switch srcType {
-		case typeMap, typeList:
+		case typeString:
+			return *attrVal.S, nil
+		case typeNumberString:
+			return *attrVal.N, nil
+		case typeMap, typeList, typeStringSet, typeNumberStringSet, typeNumberSet, typeBinarySet:
 			// For typeMap and typeList, attrVal is a very verbose data
 			// structure that contains null entries for unused type cases. We
 			// strip these out using stripNull. If it is important that the
@@ -95,22 +143,6 @@ func cvtColValue(attrVal *dynamodb.AttributeValue, srcType string, spType string
 				return nil, fmt.Errorf("failed to convert %v to a json string", attrVal.GoString())
 			}
 			return string(b), nil
-		case typeString:
-			return *attrVal.S, nil
-		case typeStringSet:
-			var strArr []string
-			for _, s := range attrVal.SS {
-				strArr = append(strArr, *s)
-			}
-			return strArr, nil
-		case typeNumberString:
-			return *attrVal.N, nil
-		case typeNumberStringSet:
-			var strArr []string
-			for _, s := range attrVal.NS {
-				strArr = append(strArr, *s)
-			}
-			return strArr, nil
 		}
 	case ddl.Numeric:
 		switch srcType {
@@ -121,16 +153,6 @@ func cvtColValue(attrVal *dynamodb.AttributeValue, srcType string, spType string
 				return nil, fmt.Errorf("failed to convert '%v' to an NUMERIC type", s)
 			}
 			return *val, nil
-		case typeNumberSet:
-			var numArr []big.Rat
-			for _, s := range attrVal.NS {
-				val, ok := (&big.Rat{}).SetString(*s)
-				if !ok {
-					return nil, fmt.Errorf("failed to convert '%v' to an NUMERIC array", attrVal.NS)
-				}
-				numArr = append(numArr, *val)
-			}
-			return numArr, nil
 		}
 	}
 	return nil, fmt.Errorf("can't convert value of type %s to Spanner type %s", attrVal.GoString(), spType)
diff --git a/sources/dynamodb/toddl.go b/sources/dynamodb/toddl.go
index d1f47ec..571f732 100644
--- a/sources/dynamodb/toddl.go
+++ b/sources/dynamodb/toddl.go
@@ -16,6 +16,7 @@
 package dynamodb
 
 import (
+	"github.com/cloudspannerecosystem/harbourbridge/common/constants"
 	"github.com/cloudspannerecosystem/harbourbridge/internal"
 	"github.com/cloudspannerecosystem/harbourbridge/schema"
 	"github.com/cloudspannerecosystem/harbourbridge/spanner/ddl"
@@ -25,12 +26,21 @@ import (
 type ToDdlImpl struct {
 }
 
+// Functions below implement the common.ToDdl interface
 // toSpannerType maps a scalar source schema type (defined by id and
 // mods) into a Spanner type. This is the core source-to-Spanner type
 // mapping.  toSpannerType returns the Spanner type and a list of type
 // conversion issues encountered.
 func (tdi ToDdlImpl) ToSpannerType(conv *internal.Conv, columnType schema.Type) (ddl.Type, []internal.SchemaIssue) {
-	switch columnType.Name {
+	ty, issues := toSpannerTypeInternal(conv, columnType.Name)
+	if conv.TargetDb == constants.TargetExperimentalPostgres {
+		ty = overrideExperimentalType(ty)
+	}
+	return ty, issues
+}
+
+func toSpannerTypeInternal(conv *internal.Conv, id string) (ddl.Type, []internal.SchemaIssue) {
+	switch id {
 	case typeNumber:
 		return ddl.Type{Name: ddl.Numeric}, nil
 	case typeNumberString, typeString, typeList, typeMap:
@@ -49,3 +59,11 @@ func (tdi ToDdlImpl) ToSpannerType(conv *internal.Conv, columnType schema.Type)
 		return ddl.Type{Name: ddl.String, Len: ddl.MaxLength}, []internal.SchemaIssue{internal.NoGoodType}
 	}
 }
+
+// Override the types to map to experimental postgres types.
+func overrideExperimentalType(originalType ddl.Type) ddl.Type {
+	if originalType.IsArray {
+		return ddl.Type{Name: ddl.String, Len: ddl.MaxLength}
+	}
+	return originalType
+}


Test patch:
diff --git a/sources/dynamodb/data_test.go b/sources/dynamodb/data_test.go
index 7347a82..fffa5c4 100644
--- a/sources/dynamodb/data_test.go
+++ b/sources/dynamodb/data_test.go
@@ -125,7 +125,34 @@ func TestCvtRowWithError(t *testing.T) {
 	assert.Equal(t, []string{attrs["a"].GoString()}, srcStrVals)
 }
 
-func TestCvtColValue(t *testing.T) {
+func TestConvArray(t *testing.T) {
+	str := "str-1"
+	stringSetVal := []*string{&str}
+	binarySetVal := [][]byte{[]byte("ABC")}
+	numStr := "1234.56789"
+	numVal := big.NewRat(123456789, 100000)
+
+	testcases := []struct {
+		name    string
+		srcType string                   // Source DB type.
+		spType  string                   // Spanner DB type.
+		in      *dynamodb.AttributeValue // Input value for conversion.
+		want    interface{}              // Expected result.
+	}{
+		{"binary set", typeBinarySet, ddl.Bytes, &dynamodb.AttributeValue{BS: binarySetVal}, binarySetVal},
+		{"string set", typeStringSet, ddl.String, &dynamodb.AttributeValue{SS: stringSetVal}, []string{str}},
+		{"number string set", typeNumberStringSet, ddl.String, &dynamodb.AttributeValue{NS: []*string{&numStr}}, []string{numStr}},
+		{"number set", typeNumberSet, ddl.Numeric, &dynamodb.AttributeValue{NS: []*string{&numStr}}, []big.Rat{*numVal}},
+	}
+
+	for _, tc := range testcases {
+		cvtVal, err := convArray(tc.in, tc.srcType, tc.spType)
+		assert.Nil(t, err, fmt.Sprintf("Failed to convert %v from %s to %s", tc.in, typeString, ddl.String))
+		assert.Equal(t, tc.want, cvtVal, tc.name)
+	}
+}
+
+func TestConvScalar(t *testing.T) {
 	str := "str-1"
 	numStr := "1234.56789"
 	boolVal := true
@@ -150,19 +177,19 @@ func TestCvtColValue(t *testing.T) {
 	}{
 		{"bool", typeBool, ddl.Bool, &dynamodb.AttributeValue{BOOL: &boolVal}, true},
 		{"binary", typeBinary, ddl.Bytes, &dynamodb.AttributeValue{B: binaryVal}, binaryVal},
-		{"binary set", typeBinarySet, ddl.Bytes, &dynamodb.AttributeValue{BS: binarySetVal}, binarySetVal},
+		{"binary set", typeBinarySet, ddl.String, &dynamodb.AttributeValue{BS: binarySetVal}, "[\"ABC\"]"},
 		{"map", typeMap, ddl.String, &dynamodb.AttributeValue{M: mapVal}, "{\"list\":[\"str-1\",\"1234.56789\"]}"},
 		{"list", typeList, ddl.String, &dynamodb.AttributeValue{L: listVal}, "[\"str-1\",\"1234.56789\"]"},
 		{"string", typeString, ddl.String, &dynamodb.AttributeValue{S: &str}, str},
-		{"string set", typeStringSet, ddl.String, &dynamodb.AttributeValue{SS: stringSetVal}, []string{str}},
+		{"string set", typeStringSet, ddl.String, &dynamodb.AttributeValue{SS: stringSetVal}, "[\"str-1\"]"},
 		{"number string", typeNumberString, ddl.String, &dynamodb.AttributeValue{N: &numStr}, numStr},
-		{"number string set", typeNumberStringSet, ddl.String, &dynamodb.AttributeValue{NS: []*string{&numStr}}, []string{numStr}},
+		{"number string set", typeNumberStringSet, ddl.String, &dynamodb.AttributeValue{NS: []*string{&numStr}}, "[\"1234.56789\"]"},
 		{"number", typeNumber, ddl.Numeric, &dynamodb.AttributeValue{N: &numStr}, *numVal},
-		{"number set", typeNumberSet, ddl.Numeric, &dynamodb.AttributeValue{NS: []*string{&numStr}}, []big.Rat{*numVal}},
+		{"number set", typeNumberSet, ddl.String, &dynamodb.AttributeValue{NS: []*string{&numStr}}, "[\"1234.56789\"]"},
 	}
 
 	for _, tc := range testcases {
-		cvtVal, err := cvtColValue(tc.in, tc.srcType, tc.spType)
+		cvtVal, err := convScalar(tc.in, tc.srcType, tc.spType)
 		assert.Nil(t, err, fmt.Sprintf("Failed to convert %v from %s to %s", tc.in, typeString, ddl.String))
 		assert.Equal(t, tc.want, cvtVal, tc.name)
 	}