Repository: pingcap/tidb-lightning
Pull request #459 (2020-11-09 09:54:58)

config: filter out all system schemas by default
<!--
Thank you for contributing to TiDB-Lightning! Please read the [CONTRIBUTING](https://github.com/pingcap/tidb-lightning/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
-->

### What problem does this PR solve? <!--add issue link with summary if exists-->

If the data source contains system schemas, they may be accidentally get imported and corrupt the cluster.

### What is changed and how it works?

Explicitly filter out system schemas (`mysql.*`, etc.) like pingcap/dumpling#194

### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test

Side effects

Related changes

 - Need to update the documentation

# Release notes

* System schemas like `mysql.*` are excluded by default. Use `-f '*.*'` to explicitly include them.


Patch:
diff --git a/lightning/common/util.go b/lightning/common/util.go
index b483b0d..96f532d 100644
--- a/lightning/common/util.go
+++ b/lightning/common/util.go
@@ -317,3 +317,16 @@ type KvPair struct {
 func TableHasAutoRowID(info *model.TableInfo) bool {
 	return !info.PKIsHandle && !info.IsCommonHandle
 }
+
+// StringSliceEqual checks if two string slices are equal.
+func StringSliceEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
diff --git a/lightning/config/config.go b/lightning/config/config.go
index 098bea4..4bbb284 100644
--- a/lightning/config/config.go
+++ b/lightning/config/config.go
@@ -67,6 +67,16 @@ const (
 var (
 	defaultConfigPaths    = []string{"tidb-lightning.toml", "conf/tidb-lightning.toml"}
 	supportedStorageTypes = []string{"file", "local", "s3", "noop"}
+
+	DefaultFilter = []string{
+		"*.*",
+		"!mysql.*",
+		"!sys.*",
+		"!INFORMATION_SCHEMA.*",
+		"!PERFORMANCE_SCHEMA.*",
+		"!METRICS_SCHEMA.*",
+		"!INSPECTION_SCHEMA.*",
+	}
 )
 
 type DBStore struct {
@@ -344,7 +354,7 @@ func NewConfig() *Config {
 			},
 			StrictFormat:  false,
 			MaxRegionSize: MaxRegionSize,
-			Filter:        []string{"*.*"},
+			Filter:        DefaultFilter,
 		},
 		TikvImporter: TikvImporter{
 			Backend:         BackendImporter,
@@ -547,7 +557,7 @@ func (cfg *Config) Adjust() error {
 	// mydumper.filter and black-white-list cannot co-exist.
 	if cfg.HasLegacyBlackWhiteList() {
 		log.L().Warn("the config `black-white-list` has been deprecated, please replace with `mydumper.filter`")
-		if !(len(cfg.Mydumper.Filter) == 1 && cfg.Mydumper.Filter[0] == "*.*") {
+		if !common.StringSliceEqual(cfg.Mydumper.Filter, DefaultFilter) {
 			return errors.New("invalid config: `mydumper.filter` and `black-white-list` cannot be simultaneously defined")
 		}
 	}
diff --git a/lightning/config/global.go b/lightning/config/global.go
index 76ded7d..eb1075f 100644
--- a/lightning/config/global.go
+++ b/lightning/config/global.go
@@ -99,7 +99,7 @@ func NewGlobalConfig() *GlobalConfig {
 			LogLevel:   "error",
 		},
 		Mydumper: GlobalMydumper{
-			Filter: []string{"*.*"},
+			Filter: DefaultFilter,
 		},
 		TikvImporter: GlobalImporter{
 			Backend: "importer",
diff --git a/tidb-lightning.toml b/tidb-lightning.toml
index d7d5d1a..2150581 100644
--- a/tidb-lightning.toml
+++ b/tidb-lightning.toml
@@ -144,7 +144,7 @@ strict-format = false
 #default-file-rules = false
 
 # only import tables if the wildcard rules are matched. See documention for details.
-filter = ['*.*']
+filter = ['*.*', '!mysql.*', '!sys.*', '!INFORMATION_SCHEMA.*', '!PERFORMANCE_SCHEMA.*', '!METRICS_SCHEMA.*', '!INSPECTION_SCHEMA.*']
 
 # CSV files are imported according to MySQL's LOAD DATA INFILE rules.
 [mydumper.csv]


Test patch:
diff --git a/lightning/common/util_test.go b/lightning/common/util_test.go
index c43149d..9071540 100644
--- a/lightning/common/util_test.go
+++ b/lightning/common/util_test.go
@@ -190,3 +190,16 @@ func (s *utilSuite) TestSQLWithRetry(c *C) {
 
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
+
+func (s *utilSuite) TestStringSliceEqual(c *C) {
+	c.Assert(common.StringSliceEqual(nil, nil), IsTrue)
+	c.Assert(common.StringSliceEqual(nil, []string{}), IsTrue)
+	c.Assert(common.StringSliceEqual(nil, []string{"a"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a"}, nil), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"a"}), IsTrue)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"b"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "b", "c"}, []string{"a", "b", "c"}), IsTrue)
+	c.Assert(common.StringSliceEqual([]string{"a"}, []string{"a", "b", "c"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "b", "c"}, []string{"a", "b"}), IsFalse)
+	c.Assert(common.StringSliceEqual([]string{"a", "x", "y"}, []string{"a", "y", "x"}), IsFalse)
+}
diff --git a/lightning/config/config_test.go b/lightning/config/config_test.go
index 0110fcf..8cf67cd 100644
--- a/lightning/config/config_test.go
+++ b/lightning/config/config_test.go
@@ -612,3 +612,21 @@ func (s *configTestSuite) TestCronEncodeDecode(c *C) {
 	c.Assert(cfg2.LoadFromTOML([]byte(confStr)), IsNil)
 	c.Assert(cfg2.Cron, DeepEquals, cfg.Cron)
 }
+
+func (s *configTestSuite) TestAdjustWithLegacyBlackWhiteList(c *C) {
+	cfg := config.NewConfig()
+	assignMinimalLegalValue(cfg)
+	c.Assert(cfg.Mydumper.Filter, DeepEquals, config.DefaultFilter)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsFalse)
+
+	cfg.Mydumper.Filter = []string{"test.*"}
+	c.Assert(cfg.Adjust(), IsNil)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsFalse)
+
+	cfg.BWList.DoDBs = []string{"test"}
+	c.Assert(cfg.Adjust(), ErrorMatches, "invalid config: `mydumper\\.filter` and `black-white-list` cannot be simultaneously defined")
+
+	cfg.Mydumper.Filter = config.DefaultFilter
+	c.Assert(cfg.Adjust(), IsNil)
+	c.Assert(cfg.HasLegacyBlackWhiteList(), IsTrue)
+}