Repository: cloudspannerecosystem/harbourbridge
Pull request #348 (2022-07-19 13:32:38)

UI backlogs 
synth_id column display in UI
Update Issues and Suggestions count bug fix
Update object detail on changing tab between source db and spanner db
alignment of the edit columns under index
changed source db from pg_dump to Postgres during pg_dump load


Patch:
diff --git a/cmd/data.go b/cmd/data.go
index 167d28c..1fb1917 100644
--- a/cmd/data.go
+++ b/cmd/data.go
@@ -116,10 +116,6 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface
 	conv.Audit.MigrationType = migration.MigrationData_DATA_ONLY.Enum()
 	dataCoversionStartTime := time.Now()
 
-	// If filePrefix not explicitly set, use dbName as prefix.
-	if cmd.filePrefix == "" {
-		cmd.filePrefix = dbName + "."
-	}
 	if !sourceProfile.UseTargetSchema() {
 		err = conversion.ReadSessionFile(conv, cmd.sessionJSON)
 		if err != nil {
@@ -138,7 +134,7 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface
 	)
 	if !cmd.dryRun {
 		now := time.Now()
-		adminClient, client, dbURI, err = CreateDatabaseClient(ctx, targetProfile, sourceProfile.Driver, ioHelper)
+		adminClient, client, dbURI, err = CreateDatabaseClient(ctx, targetProfile, sourceProfile.Driver, dbName, ioHelper)
 		if err != nil {
 			err = fmt.Errorf("can't create database client: %v", err)
 			return subcommands.ExitFailure
@@ -178,6 +174,10 @@ func (cmd *DataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface
 	dataCoversionDuration := dataCoversionEndTime.Sub(dataCoversionStartTime)
 	conv.Audit.DataConversionDuration = dataCoversionDuration
 
+	// If filePrefix not explicitly set, use dbName as prefix.
+	if cmd.filePrefix == "" {
+		cmd.filePrefix = targetProfile.Conn.Sp.Dbname + "."
+	}
 	conversion.Report(sourceProfile.Driver, bw.DroppedRowsByTable(), ioHelper.BytesRead, banner, conv, cmd.filePrefix+reportFile, ioHelper.Out)
 	conversion.WriteBadData(bw, conv, banner, cmd.filePrefix+badDataFile, ioHelper.Out)
 	// Cleanup hb tmp data directory.
diff --git a/cmd/schema.go b/cmd/schema.go
index a62a79d..0a85965 100644
--- a/cmd/schema.go
+++ b/cmd/schema.go
@@ -124,7 +124,7 @@ func (cmd *SchemaCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interfa
 	)
 
 	if !cmd.dryRun {
-		adminClient, _, dbURI, err = CreateDatabaseClient(ctx, targetProfile, sourceProfile.Driver, ioHelper)
+		adminClient, _, dbURI, err = CreateDatabaseClient(ctx, targetProfile, sourceProfile.Driver, dbName, ioHelper)
 		if err != nil {
 			err = fmt.Errorf("can't create database client: %v", err)
 			return subcommands.ExitFailure
diff --git a/cmd/schema_and_data.go b/cmd/schema_and_data.go
index c53a1f9..e5517e6 100644
--- a/cmd/schema_and_data.go
+++ b/cmd/schema_and_data.go
@@ -135,7 +135,7 @@ func (cmd *SchemaAndDataCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...
 
 	if !cmd.dryRun {
 		conversion.Report(sourceProfile.Driver, nil, ioHelper.BytesRead, "", conv, cmd.filePrefix+reportFile, ioHelper.Out)
-		adminClient, client, dbURI, err = CreateDatabaseClient(ctx, targetProfile, sourceProfile.Driver, ioHelper)
+		adminClient, client, dbURI, err = CreateDatabaseClient(ctx, targetProfile, sourceProfile.Driver, dbName, ioHelper)
 		if err != nil {
 			err = fmt.Errorf("can't create database client: %v", err)
 			return subcommands.ExitFailure
diff --git a/cmd/utils.go b/cmd/utils.go
index deaff58..143de93 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -26,7 +26,10 @@ import (
 )
 
 // CreateDatabaseClient creates new database client and admin client.
-func CreateDatabaseClient(ctx context.Context, targetProfile profiles.TargetProfile, driver string, ioHelper utils.IOStreams) (*database.DatabaseAdminClient, *sp.Client, string, error) {
+func CreateDatabaseClient(ctx context.Context, targetProfile profiles.TargetProfile, driver, dbName string, ioHelper utils.IOStreams) (*database.DatabaseAdminClient, *sp.Client, string, error) {
+	if targetProfile.Conn.Sp.Dbname == "" {
+		targetProfile.Conn.Sp.Dbname = dbName
+	}
 	project, instance, dbName, err := targetProfile.GetResourceIds(ctx, time.Now(), driver, ioHelper.Out)
 	if err != nil {
 		return nil, nil, "", err
diff --git a/internal/report.go b/internal/report.go
index 215445a..55e6b96 100644
--- a/internal/report.go
+++ b/internal/report.go
@@ -461,13 +461,13 @@ var IssueDB = map[SchemaIssue]struct {
 	MultiDimensionalArray: {Brief: "Spanner doesn't support multi-dimensional arrays", severity: warning},
 	NoGoodType:            {Brief: "No appropriate Spanner type", severity: warning},
 	Numeric:               {Brief: "Spanner does not support numeric. This type mapping could lose precision and is not recommended for production use", severity: warning},
-	NumericThatFits:       {Brief: "Spanner does not support numeric, but this type mapping preserves the numeric's specified precision", severity: note},
+	NumericThatFits:       {Brief: "Spanner does not support numeric, but this type mapping preserves the numeric's specified precision", severity: suggestion},
 	Decimal:               {Brief: "Spanner does not support decimal. This type mapping could lose precision and is not recommended for production use", severity: warning},
-	DecimalThatFits:       {Brief: "Spanner does not support decimal, but this type mapping preserves the decimal's specified precision", severity: note},
+	DecimalThatFits:       {Brief: "Spanner does not support decimal, but this type mapping preserves the decimal's specified precision", severity: suggestion},
 	Serial:                {Brief: "Spanner does not support autoincrementing types", severity: warning},
 	AutoIncrement:         {Brief: "Spanner does not support auto_increment attribute", severity: warning},
-	Timestamp:             {Brief: "Spanner timestamp is closer to PostgreSQL timestamptz", severity: note, batch: true},
-	Datetime:              {Brief: "Spanner timestamp is closer to MySQL timestamp", severity: note, batch: true},
+	Timestamp:             {Brief: "Spanner timestamp is closer to PostgreSQL timestamptz", severity: suggestion, batch: true},
+	Datetime:              {Brief: "Spanner timestamp is closer to MySQL timestamp", severity: suggestion, batch: true},
 	Time:                  {Brief: "Spanner does not support time/year types", severity: note, batch: true},
 	Widened:               {Brief: "Some columns will consume more storage in Spanner", severity: note, batch: true},
 	StringOverflow:        {Brief: "String overflow issue might occur as maximum supported length in Spanner is 2621440", severity: warning},
diff --git a/sources/dynamodb/streaming.go b/sources/dynamodb/streaming.go
index a22aa45..db1e24a 100644
--- a/sources/dynamodb/streaming.go
+++ b/sources/dynamodb/streaming.go
@@ -153,44 +153,64 @@ func ProcessStream(wgStream *sync.WaitGroup, streamClient dynamodbstreamsiface.D
 	defer wgStream.Done()
 	wgShard := &sync.WaitGroup{}
 
-	var lastProcessedShardId *string = nil
+	processingStarted := make(map[string]bool)
+
 	passAfterUserExit := false
 	for {
-		result, err := fetchShards(streamClient, lastProcessedShardId, streamArn)
+		shards, err := scanShards(streamClient, streamArn)
 		if err != nil {
-			streamInfo.Unexpected(fmt.Sprintf("Couldn't fetch shards for table %s: %s", srcTable, err))
+			streamInfo.Unexpected(fmt.Sprintf("Couldn't scan shards for table %s: %s", srcTable, err))
 			break
 		}
-		for _, shard := range result.Shards {
-			lastProcessedShardId = shard.ShardId
+		for _, shard := range shards {
+			shardId := *shard.ShardId
+			if _, ok := processingStarted[shardId]; !ok {
+				processingStarted[shardId] = false
+				streamInfo.SetShardStatus(shardId, false)
+			}
+		}
+		for _, shard := range shards {
+			shardId := *shard.ShardId
+			if !processingStarted[shardId] {
+				processingStarted[shardId] = true
 
-			wgShard.Add(1)
-			go ProcessShard(wgShard, streamInfo, conv, streamClient, shard, streamArn, srcTable)
+				wgShard.Add(1)
+				go ProcessShard(wgShard, streamInfo, conv, streamClient, shard, streamArn, srcTable)
+			}
 		}
 
-		if result.LastEvaluatedShardId == nil && passAfterUserExit {
+		if passAfterUserExit {
 			break
-		}
-		if streamInfo.UserExit {
+		} else if streamInfo.UserExit {
 			passAfterUserExit = true
-		} else if len(result.Shards) == 0 {
-			time.Sleep(10 * time.Second)
+		} else {
+			time.Sleep(20 * time.Second)
 		}
 	}
 	wgShard.Wait()
 }
 
-// fetchShards fetches latest unprocessed shards from a given DynamoDB Stream after the lastProcessedShardId.
-func fetchShards(streamClient dynamodbstreamsiface.DynamoDBStreamsAPI, lastProcessedShardId *string, streamArn string) (*dynamodbstreams.StreamDescription, error) {
+// scanShards fetches all the shards from a given DynamoDB Stream.
+func scanShards(streamClient dynamodbstreamsiface.DynamoDBStreamsAPI, streamArn string) ([]*dynamodbstreams.Shard, error) {
 	describeStreamInput := &dynamodbstreams.DescribeStreamInput{
-		ExclusiveStartShardId: lastProcessedShardId,
+		ExclusiveStartShardId: nil,
 		StreamArn:             &streamArn,
 	}
-	result, err := streamClient.DescribeStream(describeStreamInput)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected call to DescribeStream: %v", err)
+	var scanResult []*dynamodbstreams.Shard
+	for {
+		result, err := streamClient.DescribeStream(describeStreamInput)
+		if err != nil {
+			return nil, fmt.Errorf("unexpected call to DescribeStream: %v", err)
+		}
+		scanResult = append(scanResult, result.StreamDescription.Shards...)
+
+		if result.StreamDescription.LastEvaluatedShardId == nil {
+			break
+		} else {
+			describeStreamInput.ExclusiveStartShardId = result.StreamDescription.LastEvaluatedShardId
+		}
 	}
-	return result.StreamDescription, nil
+	return scanResult, nil
 }
 
 // checkTrimmedDataError checks if the error is an TrimmedDataAccessException.
@@ -208,7 +228,6 @@ func ProcessShard(wgShard *sync.WaitGroup, streamInfo *StreamingInfo, conv *inte
 	waitForParentShard(streamInfo, shard.ParentShardId)
 
 	shardId := *shard.ShardId
-	streamInfo.SetShardStatus(shardId, false)
 
 	var lastEvaluatedSequenceNumber *string = nil
 	passAfterUserExit := false
@@ -267,9 +286,9 @@ func waitForParentShard(streamInfo *StreamingInfo, parentShard *string) {
 	if parentShard != nil {
 		for {
 			streamInfo.lock.Lock()
-			done := streamInfo.ShardProcessed[*parentShard]
+			done, ok := streamInfo.ShardProcessed[*parentShard]
 			streamInfo.lock.Unlock()
-			if done {
+			if !ok || done {
 				return
 			} else {
 				time.Sleep(6 * time.Second)
diff --git a/ui/src/app/components/object-detail/object-detail.component.html b/ui/src/app/components/object-detail/object-detail.component.html
index 8d098cf..f545077 100644
--- a/ui/src/app/components/object-detail/object-detail.component.html
+++ b/ui/src/app/components/object-detail/object-detail.component.html
@@ -21,9 +21,12 @@
       </svg>
     </div>
     <div class="inner-text">
-      {{
-        'To view and modify an object details, click on the object name on the Spanner draft panel.'
-      }}
+      <span *ngIf="currentDatabase == 'spanner'">
+        To view and modify an object details, click on the object name on the Spanner draft panel.
+      </span>
+      <span *ngIf="currentDatabase == 'source'">
+        To view an object details, click on the object name on the {{ srcDbName }} panel.
+      </span>
     </div>
   </div>
 </div>
diff --git a/ui/src/app/components/object-detail/object-detail.component.scss b/ui/src/app/components/object-detail/object-detail.component.scss
index cc07001..690b928 100644
--- a/ui/src/app/components/object-detail/object-detail.component.scss
+++ b/ui/src/app/components/object-detail/object-detail.component.scss
@@ -55,7 +55,7 @@
 }
 
 .add-index-column-container {
-  margin-left: 40%;
+  margin-left: 45%;
   padding: 10px;
   .index-column-list {
     margin-right: 1rem;
@@ -206,6 +206,14 @@
   align-items: center;
 }
 
+.interleave-tab-container {
+  height: 200px;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+}
+
 .save-button {
   width: 100%;
   button {
diff --git a/ui/src/app/components/object-detail/object-detail.component.ts b/ui/src/app/components/object-detail/object-detail.component.ts
index ad5733a..5cc35aa 100644
--- a/ui/src/app/components/object-detail/object-detail.component.ts
+++ b/ui/src/app/components/object-detail/object-detail.component.ts
@@ -33,6 +33,7 @@ export class ObjectDetailComponent implements OnInit {
   @Input() ddlStmts: any = {}
   @Input() fkData: IFkTabData[] = []
   @Input() tableData: IColumnTabData[] = []
+  @Input() currentDatabase: string = 'spanner'
   @Input() indexData: IIndexData[] = []
   @Output() updateSidebar = new EventEmitter<boolean>()
   ObjectExplorerNodeType = ObjectExplorerNodeType
@@ -129,6 +130,7 @@ export class ObjectDetailComponent implements OnInit {
     this.currentObject = changes['currentObject']?.currentValue || this.currentObject
     this.tableData = changes['tableData']?.currentValue || this.tableData
     this.indexData = changes['indexData']?.currentValue || this.indexData
+    this.currentDatabase = changes['currentDatabase']?.currentValue || this.currentDatabase
     this.currentTabIndex = this.currentObject?.type === ObjectExplorerNodeType.Table ? 0 : -1
     this.isObjectSelected = this.currentObject ? true : false
     this.isEditMode = false
@@ -277,7 +279,7 @@ export class ObjectDetailComponent implements OnInit {
       const srDataType = item.srcDataType
       const spDataType = item.spDataType
       let brief: string = ''
-      this.typeMap[srDataType].forEach((type: any) => {
+      this.typeMap[srDataType]?.forEach((type: any) => {
         if (spDataType == type.T) brief = type.Brief
       })
       this.isSpTableSuggesstionDisplay.push(brief !== '')
diff --git a/ui/src/app/components/object-explorer/object-explorer.component.html b/ui/src/app/components/object-explorer/object-explorer.component.html
index 418b810..9443894 100644
--- a/ui/src/app/components/object-explorer/object-explorer.component.html
+++ b/ui/src/app/components/object-explorer/object-explorer.component.html
@@ -1,5 +1,9 @@
 <div class="container">
-  <mat-tab-group [ngClass]="[isLeftColumnCollapse ? 'hidden' : 'display']" selectedIndex="1">
+  <mat-tab-group
+    [ngClass]="[isLeftColumnCollapse ? 'hidden' : 'display']"
+    selectedIndex="1"
+    (selectedTabChange)="onTabChanged()"
+  >
     <mat-tab [label]="srcDbName.toUpperCase()">
       <div class="filter-wrapper">
         <div class="left">
diff --git a/ui/src/app/components/object-explorer/object-explorer.component.ts b/ui/src/app/components/object-explorer/object-explorer.component.ts
index e94bb11..a7f7111 100644
--- a/ui/src/app/components/object-explorer/object-explorer.component.ts
+++ b/ui/src/app/components/object-explorer/object-explorer.component.ts
@@ -20,6 +20,8 @@ export class ObjectExplorerComponent implements OnInit {
   spannerSortOrder: string = ''
   srcSearchText: string = ''
   spannerSearchText: string = ''
+  selectedTab: string = 'spanner'
+  @Output() selectedDatabase = new EventEmitter<string>()
   @Output() selectObject = new EventEmitter<FlatNode>()
   @Output() updateSpannerTable = new EventEmitter<IUpdateTableArgument>()
   @Output() updateSrcTable = new EventEmitter<IUpdateTableArgument>()
@@ -146,4 +148,14 @@ export class ObjectExplorerComponent implements OnInit {
       return false
     }
   }
+  onTabChanged() {
+    if (this.selectedTab == 'spanner') {
+      this.selectedTab = 'source'
+    } else {
+      this.selectedTab = 'spanner'
+    }
+    this.selectedDatabase.emit(this.selectedTab)
+    this.currentSelectedObject = null
+    this.selectObject.emit(undefined)
+  }
 }
diff --git a/ui/src/app/components/session-listing/session-listing.component.html b/ui/src/app/components/session-listing/session-listing.component.html
index fb1c9b0..3b416aa 100644
--- a/ui/src/app/components/session-listing/session-listing.component.html
+++ b/ui/src/app/components/session-listing/session-listing.component.html
@@ -11,7 +11,8 @@
           <div class="table-header-container">
             <span>Session Name</span>
             <button mat-icon-button (click)="toggleFilterDisplay('sessionName')">
-              <mat-icon>filter_list</mat-icon>
+              <mat-icon *ngIf="displayFilter['sessionName']">keyboard_arrow_up</mat-icon>
+              <mat-icon *ngIf="!displayFilter['sessionName']">filter_list</mat-icon>
             </button>
           </div>
           <mat-form-field
@@ -36,7 +37,8 @@
           <div class="table-header-container">
             <span>Editor</span>
             <button mat-icon-button (click)="toggleFilterDisplay('editorName')">
-              <mat-icon>filter_list</mat-icon>
+              <mat-icon *ngIf="displayFilter['editorName']">keyboard_arrow_up</mat-icon>
+              <mat-icon *ngIf="!displayFilter['editorName']">filter_list</mat-icon>
             </button>
           </div>
           <mat-form-field *ngIf="displayFilter.editorName" appearance="standard">
@@ -57,7 +59,8 @@
           <div class="table-header-container">
             <span>Database Type</span>
             <button mat-icon-button (click)="toggleFilterDisplay('databaseType')">
-              <mat-icon>filter_list</mat-icon>
+              <mat-icon *ngIf="displayFilter['databaseType']">keyboard_arrow_up</mat-icon>
+              <mat-icon *ngIf="!displayFilter['databaseType']">filter_list</mat-icon>
             </button>
           </div>
           <mat-form-field *ngIf="displayFilter.databaseType" appearance="standard">
@@ -78,7 +81,8 @@
           <div class="table-header-container">
             <span>Database Name</span>
             <button mat-icon-button (click)="toggleFilterDisplay('databaseName')">
-              <mat-icon>filter_list</mat-icon>
+              <mat-icon *ngIf="displayFilter['databaseName']">keyboard_arrow_up</mat-icon>
+              <mat-icon *ngIf="!displayFilter['databaseName']">filter_list</mat-icon>
             </button>
           </div>
           <mat-form-field *ngIf="displayFilter.databaseName" appearance="standard">
diff --git a/ui/src/app/components/session-listing/session-listing.component.scss b/ui/src/app/components/session-listing/session-listing.component.scss
index b9a9337..f0af4cc 100644
--- a/ui/src/app/components/session-listing/session-listing.component.scss
+++ b/ui/src/app/components/session-listing/session-listing.component.scss
@@ -55,3 +55,22 @@ table {
 .notes-wrapper {
   margin-top: 10px;
 }
+
+.mat-column-SessionName,
+.mat-column-EditorName,
+.mat-column-DatabaseType,
+.mat-column-DatabaseName {
+  width: 13vw;
+  min-width: 150px;
+}
+
+.mat-column-Action,
+.mat-column-Notes,
+.mat-column-CreateTimestamp {
+  width: 15vw;
+  min-width: 150px;
+}
+
+.mat-form-field {
+  width: 80%;
+}
diff --git a/ui/src/app/components/session-listing/session-listing.component.ts b/ui/src/app/components/session-listing/session-listing.component.ts
index 5a60a21..102316b 100644
--- a/ui/src/app/components/session-listing/session-listing.component.ts
+++ b/ui/src/app/components/session-listing/session-listing.component.ts
@@ -127,6 +127,7 @@ export class SessionListingComponent implements OnInit {
       this.router.navigate(['/workspace'])
     })
   }
+
   openSpannerConfigDialog() {
     this.clickEvent.openSpannerConfig()
   }
diff --git a/ui/src/app/components/summary/summary.component.ts b/ui/src/app/components/summary/summary.component.ts
index 4b4d966..29b0d3f 100644
--- a/ui/src/app/components/summary/summary.component.ts
+++ b/ui/src/app/components/summary/summary.component.ts
@@ -35,9 +35,14 @@ export class SummaryComponent implements OnInit {
           let s = summary.get(this.currentObject.name)
           if (s) {
             this.initiateSummaryCollection(s)
-            this.summaryCount = s.NotesCount + s.WarningsCount
-            this.changeIssuesLabel.emit(s.NotesCount + s.WarningsCount)
+            this.summaryCount = s.NotesCount + s.WarningsCount + s.ErrorsCount + s.SuggestionsCount
+            this.changeIssuesLabel.emit(
+              s.NotesCount + s.WarningsCount + s.ErrorsCount + s.SuggestionsCount
+            )
           }
+        } else {
+          this.summaryCount = 0
+          this.changeIssuesLabel.emit(0)
         }
       },
     })
@@ -52,9 +57,14 @@ export class SummaryComponent implements OnInit {
       let s = this.summary.get(this.currentObject.name)
       if (s) {
         this.initiateSummaryCollection(s)
-        this.summaryCount = s.NotesCount + s.WarningsCount
-        this.changeIssuesLabel.emit(s.NotesCount + s.WarningsCount)
+        this.summaryCount = s.NotesCount + s.WarningsCount + s.ErrorsCount + s.SuggestionsCount
+        this.changeIssuesLabel.emit(
+          s.NotesCount + s.WarningsCount + s.ErrorsCount + s.SuggestionsCount
+        )
       }
+    } else {
+      this.summaryCount = 0
+      this.changeIssuesLabel.emit(0)
     }
   }
 
diff --git a/ui/src/app/components/workspace/workspace.component.html b/ui/src/app/components/workspace/workspace.component.html
index dbbfd37..c27b243 100644
--- a/ui/src/app/components/workspace/workspace.component.html
+++ b/ui/src/app/components/workspace/workspace.component.html
@@ -28,6 +28,7 @@
       [ngClass]="[isLeftColumnCollapse ? 'left-column-collapse' : 'left-column-expand']"
     >
       <app-object-explorer
+        (selectedDatabase)="changeCurrentDatabase($event)"
         (selectObject)="changeCurrentObject($event)"
         (updateSpannerTable)="updateSpannerTable($event)"
         (updateSrcTable)="updateSrcTable($event)"
@@ -53,6 +54,7 @@
         [typeMap]="typeMap"
         [ddlStmts]="ddlStmts"
         [fkData]="fkData"
+        [currentDatabase]="currentDatabase"
       >
       </app-object-detail>
     </div>
diff --git a/ui/src/app/components/workspace/workspace.component.ts b/ui/src/app/components/workspace/workspace.component.ts
index adae54f..bbc130f 100644
--- a/ui/src/app/components/workspace/workspace.component.ts
+++ b/ui/src/app/components/workspace/workspace.component.ts
@@ -40,6 +40,7 @@ export class WorkspaceComponent implements OnInit, OnDestroy {
   objectExplorerInitiallyRender: boolean = false
   srcDbName: string = localStorage.getItem(StorageKeys.SourceDbName) as string
   conversionRatePercentages: ConversionRate = { good: 0, ok: 0, bad: 0 }
+  currentDatabase: string = 'spanner'
   constructor(
     private data: DataService,
     private conversion: ConversionService,
@@ -151,7 +152,7 @@ export class WorkspaceComponent implements OnInit, OnDestroy {
   }
 
   changeCurrentObject(object: FlatNode) {
-    if (object.type === ObjectExplorerNodeType.Table) {
+    if (object?.type === ObjectExplorerNodeType.Table) {
       this.currentObject = object
       this.tableData = this.currentObject
         ? this.conversion.getColumnMapping(this.currentObject.name, this.conv)
@@ -161,12 +162,18 @@ export class WorkspaceComponent implements OnInit, OnDestroy {
       this.fkData = this.currentObject
         ? this.conversion.getFkMapping(this.currentObject.name, this.conv)
         : []
-    } else {
+    } else if (object?.type === ObjectExplorerNodeType.Index) {
       this.currentObject = object
       this.indexData = this.conversion.getIndexMapping(object.parent, this.conv, object.name)
+    } else {
+      this.currentObject = null
     }
   }
 
+  changeCurrentDatabase(database: string) {
+    this.currentDatabase = database
+  }
+
   updateIssuesLabel(count: number) {
     setTimeout(() => {
       this.issuesAndSuggestionsLabel = `ISSUES AND SUGGESTIONS (${count})`
diff --git a/ui/src/app/model/edit-table.ts b/ui/src/app/model/edit-table.ts
index 0882b94..1e7a526 100644
--- a/ui/src/app/model/edit-table.ts
+++ b/ui/src/app/model/edit-table.ts
@@ -1,6 +1,6 @@
 export default interface IColumnTabData {
   spOrder: number | string
-  srcOrder: number
+  srcOrder: number | string
   srcColName: string
   srcDataType: string
   spColName: string
diff --git a/ui/src/app/services/conversion/conversion.service.ts b/ui/src/app/services/conversion/conversion.service.ts
index e61dfcd..e9f59dd 100644
--- a/ui/src/app/services/conversion/conversion.service.ts
+++ b/ui/src/app/services/conversion/conversion.service.ts
@@ -147,25 +147,47 @@ export class ConversionService {
 
   getColumnMapping(tableName: string, data: IConv): IColumnTabData[] {
     let srcTableName = data.ToSource[tableName].Name
-    return data.SrcSchema[srcTableName].ColNames.map((name: string, i: number) => {
-      let spColName = data.ToSpanner[srcTableName].Cols[name]
-      let srcPks = data.SrcSchema[srcTableName].PrimaryKeys
-      let spannerColDef = data.SpSchema[tableName].ColDefs[spColName]
-      return {
-        spOrder: spannerColDef ? i + 1 : '',
-        srcOrder: i + 1,
-        spColName: spannerColDef ? spColName : '',
-        spDataType: spannerColDef ? spannerColDef.T.Name : '',
-        srcColName: name,
-        srcDataType: data.SrcSchema[srcTableName].ColDefs[name].Type.Name,
-        spIsPk: spannerColDef
-          ? data.SpSchema[tableName].Pks.map((p) => p.Col).indexOf(spColName) !== -1
-          : false,
-        srcIsPk: srcPks ? srcPks.map((p) => p.Column).indexOf(name) !== -1 : false,
-        spIsNotNull: spannerColDef ? data.SpSchema[tableName].ColDefs[spColName].NotNull : false,
-        srcIsNotNull: data.SrcSchema[srcTableName].ColDefs[name].NotNull,
+    const res: IColumnTabData[] = data.SrcSchema[srcTableName].ColNames.map(
+      (name: string, i: number) => {
+        let spColName = data.ToSpanner[srcTableName].Cols[name]
+        let srcPks = data.SrcSchema[srcTableName].PrimaryKeys
+        let spannerColDef = data.SpSchema[tableName].ColDefs[spColName]
+        return {
+          spOrder: spannerColDef ? i + 1 : '',
+          srcOrder: i + 1,
+          spColName: spannerColDef ? spColName : '',
+          spDataType: spannerColDef ? spannerColDef.T.Name : '',
+          srcColName: name,
+          srcDataType: data.SrcSchema[srcTableName].ColDefs[name].Type.Name,
+          spIsPk: spannerColDef
+            ? data.SpSchema[tableName].Pks.map((p) => p.Col).indexOf(spColName) !== -1
+            : false,
+          srcIsPk: srcPks ? srcPks.map((p) => p.Column).indexOf(name) !== -1 : false,
+          spIsNotNull: spannerColDef ? data.SpSchema[tableName].ColDefs[spColName].NotNull : false,
+          srcIsNotNull: data.SrcSchema[srcTableName].ColDefs[name].NotNull,
+        }
+      }
+    )
+    data.SpSchema[srcTableName].ColNames.forEach((name: string, i: number) => {
+      if (data.SrcSchema[srcTableName].ColNames.indexOf(name) < 0) {
+        let spannerColDef = data.SpSchema[tableName].ColDefs[name]
+        res.push({
+          spOrder: i + 1,
+          srcOrder: '',
+          spColName: name,
+          spDataType: spannerColDef ? spannerColDef.T.Name : '',
+          srcColName: '',
+          srcDataType: '',
+          spIsPk: spannerColDef
+            ? data.SpSchema[tableName].Pks.map((p) => p.Col).indexOf(name) !== -1
+            : false,
+          srcIsPk: false,
+          spIsNotNull: spannerColDef ? data.SpSchema[tableName].ColDefs[name].NotNull : false,
+          srcIsNotNull: false,
+        })
       }
     })
+    return res
   }
 
   getPkMapping(tableData: IColumnTabData[]): IColumnTabData[] {
diff --git a/ui/src/app/utils/utils.ts b/ui/src/app/utils/utils.ts
index 3fd6a78..ef8a238 100644
--- a/ui/src/app/utils/utils.ts
+++ b/ui/src/app/utils/utils.ts
@@ -4,7 +4,7 @@ export function extractSourceDbName(srcDbName: string) {
   if (srcDbName == 'mysql' || srcDbName == 'mysqldump') {
     return SourceDbNames.MySQL
   }
-  if (srcDbName === 'postgres' || srcDbName === 'pgdump') {
+  if (srcDbName === 'postgres' || srcDbName === 'pgdump' || srcDbName === 'pg_dump') {
     return SourceDbNames.Postgres
   }
   if (srcDbName === 'oracle') {


Test patch:
diff --git a/sources/dynamodb/streaming_test.go b/sources/dynamodb/streaming_test.go
index 6418690..38f4f28 100644
--- a/sources/dynamodb/streaming_test.go
+++ b/sources/dynamodb/streaming_test.go
@@ -130,31 +130,17 @@ func TestProcessStream(t *testing.T) {
 	assert.Equal(t, int64(1), streamInfo.TotalUnexpecteds())
 }
 
-func Test_fetchShards(t *testing.T) {
+func Test_scanShards(t *testing.T) {
 	streamArn := "testStreamArn"
 	tableName := "testTable"
 	type args struct {
-		streamClient         dynamodbstreamsiface.DynamoDBStreamsAPI
-		lastEvaluatedShardId *string
-		streamArn            string
-	}
-	streamDescription := dynamodbstreams.StreamDescription{
-		LastEvaluatedShardId: aws.String("shard2"),
-		Shards: []*dynamodbstreams.Shard{
-			{
-				ShardId: aws.String("shard1"),
-			},
-			{
-				ShardId: aws.String("shard2"),
-			},
-		},
-		StreamArn: &streamArn,
-		TableName: &tableName,
+		streamClient dynamodbstreamsiface.DynamoDBStreamsAPI
+		streamArn    string
 	}
 	tests := []struct {
 		name    string
 		args    args
-		want    *dynamodbstreams.StreamDescription
+		want    []*dynamodbstreams.Shard
 		wantErr bool
 	}{
 		{
@@ -163,13 +149,53 @@ func Test_fetchShards(t *testing.T) {
 				streamClient: &mockDynamoStreamsClient{
 					describeStreamOutputs: []dynamodbstreams.DescribeStreamOutput{
 						{
-							StreamDescription: &streamDescription,
+							StreamDescription: &dynamodbstreams.StreamDescription{
+								LastEvaluatedShardId: aws.String("shard2"),
+								Shards: []*dynamodbstreams.Shard{
+									{
+										ShardId: aws.String("shard1"),
+									},
+									{
+										ShardId: aws.String("shard2"),
+									},
+								},
+								StreamArn: &streamArn,
+								TableName: &tableName,
+							},
+						},
+						{
+							StreamDescription: &dynamodbstreams.StreamDescription{
+								LastEvaluatedShardId: nil,
+								Shards: []*dynamodbstreams.Shard{
+									{
+										ShardId: aws.String("shard3"),
+									},
+									{
+										ShardId: aws.String("shard4"),
+									},
+								},
+								StreamArn: &streamArn,
+								TableName: &tableName,
+							},
 						},
 					},
 				},
 				streamArn: streamArn,
 			},
-			want:    &streamDescription,
+			want: []*dynamodbstreams.Shard{
+				{
+					ShardId: aws.String("shard1"),
+				},
+				{
+					ShardId: aws.String("shard2"),
+				},
+				{
+					ShardId: aws.String("shard3"),
+				},
+				{
+					ShardId: aws.String("shard4"),
+				},
+			},
 			wantErr: false,
 		},
 		{
@@ -184,13 +210,13 @@ func Test_fetchShards(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := fetchShards(tt.args.streamClient, tt.args.lastEvaluatedShardId, tt.args.streamArn)
+			got, err := scanShards(tt.args.streamClient, tt.args.streamArn)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("fetchShards() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("scanShards() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("fetchShards() = %v, want %v", got, tt.want)
+				t.Errorf("scanShards() = %v, want %v", got, tt.want)
 			}
 		})
 	}