Repository: pingcap/dumpling
Pull request #146 (2020-08-28 05:52:44)

export: record location after connection pool established
<!--
Thank you for contributing to Dumpling! Please read the [CONTRIBUTING](https://github.com/pingcap/dumpling/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
-->

### What problem does this PR solve? <!--add issue link with summary if exists-->
support DM's scenario: when migrate from a DB that can't FTWRL, dumpling uses inconsistent dump. Record the location after connection pool established to indicate DM leave safe mode

### What is changed and how it works?
metadata now

```
Started dump at: 2020-08-26 17:54:52
SHOW MASTER STATUS:
	Log: mysql-bin.000002
	Pos: 194
	GTID:09bec856-ba95-11ea-850a-58f2b4af5188:1-103
SHOW MASTER STATUS: /* AFTER CONNECTION POOL ESTABLISHED */
	Log: mysql-bin.000002
	Pos: 199
	GTID:09bec856-ba95-11ea-850a-58f2b4af5188:1-103

Finished dump at: 2020-08-26 17:54:52
```

### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test

Side effects

Related changes
 
### Release note

- No release note


CLAassistant: [![CLA assistant check](https://cla-assistant.io/pull/badge/not_signed)](https://cla-assistant.io/pingcap/dumpling?pullRequest=146) <br/>Thank you for your submission! We really appreciate it. Like many open source projects, we ask that you sign our [Contributor License Agreement](https://cla-assistant.io/pingcap/dumpling?pullRequest=146) before we can accept your contribution.<br/><sub>You have signed the CLA already but the status is still pending? Let us [recheck](https://cla-assistant.io/check/pingcap/dumpling?pullRequest=146) it.</sub>
lance6716: PTAL @kennytm 

Patch:
diff --git a/v4/export/config.go b/v4/export/config.go
index 0d85e32..64b5d5c 100644
--- a/v4/export/config.go
+++ b/v4/export/config.go
@@ -60,6 +60,8 @@ type Config struct {
 	DumpEmptyDatabase  bool
 	OutputFileTemplate *template.Template `json:"-"`
 	SessionParams      map[string]interface{}
+
+	PosAfterConnect bool
 }
 
 func DefaultConfig() *Config {
@@ -94,6 +96,7 @@ func DefaultConfig() *Config {
 		DumpEmptyDatabase:  true,
 		SessionParams:      make(map[string]interface{}),
 		OutputFileTemplate: DefaultOutputFileTemplate,
+		PosAfterConnect:    false,
 	}
 }
 
diff --git a/v4/export/dump.go b/v4/export/dump.go
index e31647d..5a7d086 100755
--- a/v4/export/dump.go
+++ b/v4/export/dump.go
@@ -126,7 +126,7 @@ func Dump(pCtx context.Context, conf *Config) (err error) {
 			return withStack(err)
 		}
 		m.recordStartTime(time.Now())
-		err = m.recordGlobalMetaData(conn, conf.ServerInfo.ServerType)
+		err = m.recordGlobalMetaData(conn, conf.ServerInfo.ServerType, false)
 		if err != nil {
 			log.Info("get global metadata failed", zap.Error(err))
 		}
@@ -155,7 +155,7 @@ func Dump(pCtx context.Context, conf *Config) (err error) {
 			return withStack(err)
 		}
 		m.recordStartTime(time.Now())
-		err = m.recordGlobalMetaData(conn, conf.ServerInfo.ServerType)
+		err = m.recordGlobalMetaData(conn, conf.ServerInfo.ServerType, false)
 		if err != nil {
 			log.Info("get global metadata failed", zap.Error(err))
 		}
@@ -168,6 +168,16 @@ func Dump(pCtx context.Context, conf *Config) (err error) {
 	}
 	defer connectPool.Close()
 
+	if conf.PosAfterConnect {
+		conn := connectPool.getConn()
+		// record again, to provide a location to exit safe mode for DM
+		err = m.recordGlobalMetaData(conn, conf.ServerInfo.ServerType, true)
+		if err != nil {
+			log.Info("get global metadata (after connection pool established) failed", zap.Error(err))
+		}
+		connectPool.releaseConn(conn)
+	}
+
 	if conf.Consistency != "lock" {
 		conn := connectPool.getConn()
 		if err = prepareTableListToDump(conf, conn); err != nil {
diff --git a/v4/export/metadata.go b/v4/export/metadata.go
index 63d5d8c..c8e27cd 100644
--- a/v4/export/metadata.go
+++ b/v4/export/metadata.go
@@ -47,9 +47,13 @@ func (m *globalMetadata) recordFinishTime(t time.Time) {
 	m.buffer.WriteString("Finished dump at: " + t.Format(metadataTimeLayout) + "\n")
 }
 
-func (m *globalMetadata) recordGlobalMetaData(db *sql.Conn, serverType ServerType) error {
+func (m *globalMetadata) recordGlobalMetaData(db *sql.Conn, serverType ServerType, afterConn bool) error {
 	// get master status info
-	m.buffer.WriteString("SHOW MASTER STATUS:\n")
+	m.buffer.WriteString("SHOW MASTER STATUS:")
+	if afterConn {
+		m.buffer.WriteString(" /* AFTER CONNECTION POOL ESTABLISHED */")
+	}
+	m.buffer.WriteString("\n")
 	switch serverType {
 	// For MySQL:
 	// mysql> SHOW MASTER STATUS;
@@ -122,6 +126,11 @@ func (m *globalMetadata) recordGlobalMetaData(db *sql.Conn, serverType ServerTyp
 	if serverType == ServerTypeTiDB {
 		return nil
 	}
+
+	// omit follower status if called after connection pool established
+	if afterConn {
+		return nil
+	}
 	// get follower status info
 	var (
 		isms  bool


Test patch:
diff --git a/v4/export/metadata_test.go b/v4/export/metadata_test.go
index 0918f38..3bee987 100644
--- a/v4/export/metadata_test.go
+++ b/v4/export/metadata_test.go
@@ -32,7 +32,7 @@ func (s *testMetaDataSuite) TestMysqlMetaData(c *C) {
 
 	testFilePath := "/test"
 	m := newGlobalMetadata(testFilePath)
-	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL), IsNil)
+	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL, false), IsNil)
 	c.Assert(m.filePath, Equals, path.Join(testFilePath, metadataPath))
 
 	c.Assert(m.buffer.String(), Equals, "SHOW MASTER STATUS:\n"+
@@ -42,6 +42,44 @@ func (s *testMetaDataSuite) TestMysqlMetaData(c *C) {
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
 }
 
+func (s *testMetaDataSuite) TestMetaDataAfterConn(c *C) {
+	db, mock, err := sqlmock.New()
+	c.Assert(err, IsNil)
+	defer db.Close()
+	conn, err := db.Conn(context.Background())
+	c.Assert(err, IsNil)
+
+	logFile := "ON.000001"
+	pos := "7502"
+	gtidSet := "6ce40be3-e359-11e9-87e0-36933cb0ca5a:1-29"
+	rows := sqlmock.NewRows([]string{"File", "Position", "Binlog_Do_DB", "Binlog_Ignore_DB", "Executed_Gtid_Set"}).
+		AddRow(logFile, pos, "", "", gtidSet)
+	pos2 := "7510"
+	rows2 := sqlmock.NewRows([]string{"File", "Position", "Binlog_Do_DB", "Binlog_Ignore_DB", "Executed_Gtid_Set"}).
+		AddRow(logFile, pos2, "", "", gtidSet)
+	mock.ExpectQuery("SHOW MASTER STATUS").WillReturnRows(rows)
+	mock.ExpectQuery("SELECT @@default_master_connection").WillReturnError(fmt.Errorf("mock error"))
+	mock.ExpectQuery("SHOW SLAVE STATUS").WillReturnRows(
+		sqlmock.NewRows([]string{"exec_master_log_pos", "relay_master_log_file", "master_host", "Executed_Gtid_Set", "Seconds_Behind_Master"}))
+	mock.ExpectQuery("SHOW MASTER STATUS").WillReturnRows(rows2)
+
+	testFilePath := "/test"
+	m := newGlobalMetadata(testFilePath)
+	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL, false), IsNil)
+	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL, true), IsNil)
+	c.Assert(m.filePath, Equals, path.Join(testFilePath, metadataPath))
+
+	c.Assert(m.buffer.String(), Equals, "SHOW MASTER STATUS:\n"+
+		"\tLog: ON.000001\n"+
+		"\tPos: 7502\n"+
+		"\tGTID:6ce40be3-e359-11e9-87e0-36933cb0ca5a:1-29\n\n"+
+		"SHOW MASTER STATUS: /* AFTER CONNECTION POOL ESTABLISHED */\n"+
+		"\tLog: ON.000001\n"+
+		"\tPos: 7510\n"+
+		"\tGTID:6ce40be3-e359-11e9-87e0-36933cb0ca5a:1-29\n\n")
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
+}
+
 func (s *testMetaDataSuite) TestMysqlWithFollowersMetaData(c *C) {
 	db, mock, err := sqlmock.New()
 	c.Assert(err, IsNil)
@@ -62,7 +100,7 @@ func (s *testMetaDataSuite) TestMysqlWithFollowersMetaData(c *C) {
 
 	testFilePath := "/test"
 	m := newGlobalMetadata(testFilePath)
-	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL), IsNil)
+	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL, false), IsNil)
 	c.Assert(m.filePath, Equals, path.Join(testFilePath, metadataPath))
 
 	c.Assert(m.buffer.String(), Equals, "SHOW MASTER STATUS:\n"+
@@ -96,7 +134,7 @@ func (s *testMetaDataSuite) TestMariaDBMetaData(c *C) {
 	mock.ExpectQuery("SHOW SLAVE STATUS").WillReturnRows(rows)
 	testFilePath := "/test"
 	m := newGlobalMetadata(testFilePath)
-	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMariaDB), IsNil)
+	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMariaDB, false), IsNil)
 	c.Assert(m.filePath, Equals, path.Join(testFilePath, metadataPath))
 
 	c.Assert(mock.ExpectationsWereMet(), IsNil)
@@ -125,7 +163,7 @@ func (s *testMetaDataSuite) TestMariaDBWithFollowersMetaData(c *C) {
 
 	testFilePath := "/test"
 	m := newGlobalMetadata(testFilePath)
-	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL), IsNil)
+	c.Assert(m.recordGlobalMetaData(conn, ServerTypeMySQL, false), IsNil)
 	c.Assert(m.filePath, Equals, path.Join(testFilePath, metadataPath))
 
 	c.Assert(m.buffer.String(), Equals, "SHOW MASTER STATUS:\n"+