Repository: pingcap/dumpling
Pull request #60 (2020-04-22 12:43:29)

fix dumpling dumping databases without tables
1. update tidb-tools to fix match nil pointer problem https://github.com/pingcap/tidb-tools/pull/334.
2. When dumping databases without tables, dumpling won't generate create database sqls. This PR solves this problem.


Patch:
diff --git a/go.mod b/go.mod
index 5702916..be45d12 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
 	github.com/pingcap/errors v0.11.4
 	github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
-	github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306103835-530c669f7112+incompatible
+	github.com/pingcap/tidb-tools v3.0.13+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/soheilhy/cmux v0.1.4
 	github.com/spf13/cobra v0.0.6
diff --git a/go.sum b/go.sum
index 1479df1..1b0457a 100644
--- a/go.sum
+++ b/go.sum
@@ -76,8 +76,8 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
 github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
-github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306103835-530c669f7112+incompatible h1:RYNZrH30AoPmXBJhpFNB6UYE1ivwApG3VfScyQm3opA=
-github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306103835-530c669f7112+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
+github.com/pingcap/tidb-tools v3.0.13+incompatible h1:XoifNFZsQ4QO9ogwIsZQYRv8KFF15BNG+Imnf+UiqZk=
+github.com/pingcap/tidb-tools v3.0.13+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
diff --git a/v4/export/black_white_list.go b/v4/export/black_white_list.go
index 2fad49c..65ef87c 100644
--- a/v4/export/black_white_list.go
+++ b/v4/export/black_white_list.go
@@ -91,6 +91,11 @@ func filterTables(conf *Config) error {
 				ignoredDBTable.AppendTable(dbName, table)
 			}
 		}
+		// 1. this dbName doesn't match black white list, don't add
+		// 2. this dbName matches black white list, but there is no table in this database, add
+		if _, ok := dbTables[dbName]; !ok && bwList.Apply(dbName, "") {
+			dbTables[dbName] = make([]*TableInfo, 0)
+		}
 	}
 
 	if len(ignoredDBTable) > 0 {
diff --git a/v4/export/dump.go b/v4/export/dump.go
index 0b84e5f..601397c 100644
--- a/v4/export/dump.go
+++ b/v4/export/dump.go
@@ -107,6 +107,9 @@ func dumpDatabases(ctx context.Context, conf *Config, db *sql.DB, writer Writer)
 			return err
 		}
 
+		if len(tables) == 0 {
+			continue
+		}
 		rateLimit := newRateLimit(conf.Threads)
 		var g errgroup.Group
 		for _, table := range tables {
diff --git a/v4/export/prepare.go b/v4/export/prepare.go
index ba78884..2e1e122 100644
--- a/v4/export/prepare.go
+++ b/v4/export/prepare.go
@@ -50,6 +50,7 @@ func listAllTables(db *sql.DB, databaseNames []string) (DatabaseTables, error) {
 		if err != nil {
 			return nil, err
 		}
+		dbTables[dbName] = make([]*TableInfo, 0, len(tables))
 		dbTables = dbTables.AppendTables(dbName, tables...)
 	}
 	return dbTables, nil


Test patch:
diff --git a/tests/empty_database/run.sh b/tests/empty_database/run.sh
new file mode 100644
index 0000000..8e38dab
--- /dev/null
+++ b/tests/empty_database/run.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -eu
+cur=$(cd `dirname $0`; pwd)
+
+DB_NAME="empty_test"
+
+# drop database on mysql
+run_sql "drop database if exists \`$DB_NAME\`;"
+
+# build data on mysql
+run_sql "create database \`$DB_NAME\`;"
+
+# dumping
+export DUMPLING_TEST_DATABASE=$DB_NAME
+run_dumpling
+
+sql="CREATE DATABASE \`$DB_NAME\`"
+cnt=$(sed "s/$sql/$sql\n/g" $DUMPLING_OUTPUT_DIR/$DB_NAME-schema-create.sql | grep -c "$sql") || true
+[ $cnt = 1 ]
+
diff --git a/v4/export/black_white_list_test.go b/v4/export/black_white_list_test.go
index 5758cca..21dc747 100644
--- a/v4/export/black_white_list_test.go
+++ b/v4/export/black_white_list_test.go
@@ -79,3 +79,41 @@ func (s *testBWListSuite) TestFilterTables(c *C) {
 	c.Assert(conf.Tables, HasLen, 1)
 	c.Assert(conf.Tables, DeepEquals, expectedDBTables)
 }
+
+func (s *testBWListSuite) TestFilterDatabaseWithNoTable(c *C) {
+	dbTables := DatabaseTables{}
+	expectedDBTables := DatabaseTables{}
+
+	dbTables["xxx"] = []*TableInfo{}
+	conf := &Config{
+		ServerInfo: ServerInfo{
+			ServerType: ServerTypeTiDB,
+		},
+		Tables: dbTables,
+		BlackWhiteList: BWListConf{
+			Mode: MySQLReplicationMode,
+			Rules: &MySQLReplicationConf{
+				Rules: &filter.Rules{
+					DoDBs: []string{"yyy"},
+				},
+			},
+		},
+	}
+	c.Assert(filterTables(conf), IsNil)
+	c.Assert(conf.Tables, HasLen, 0)
+
+	dbTables["xxx"] = []*TableInfo{}
+	expectedDBTables["xxx"] = []*TableInfo{}
+	conf.Tables = dbTables
+	conf.BlackWhiteList = BWListConf{
+		Mode: MySQLReplicationMode,
+		Rules: &MySQLReplicationConf{
+			Rules: &filter.Rules{
+				DoDBs: []string{"xxx"},
+			},
+		},
+	}
+	c.Assert(filterTables(conf), IsNil)
+	c.Assert(conf.Tables, HasLen, 1)
+	c.Assert(conf.Tables, DeepEquals, expectedDBTables)
+}