Repository: pingcap/dumpling
Pull request #45 (2020-03-10 14:25:48)

Output metadata after the dump operation
Add feature https://github.com/pingcap/dumpling/issues/43.

---
Support output a metadata into output dir after dump operation. DM need the position in metadata to get the start checkpoint for syncer in `all-mode`.
An example of Mydumper's metadata:
```
Started dump at: 2020-03-10 15:53:25
SHOW MASTER STATUS:
        Log: ON.000001
        Pos: 7502
        GTID:6ce40be3-e359-11e9-87e0-36933cb0ca5a:1-29

Finished dump at: 2020-03-10 15:53:25
```

kennytm: Please fix the test failure.

```
metadata_test.go:30:

    c.Assert(m.logFile, Equals, logFile)

... obtained string = ""

... expected string = "ON.000001"
```
lichunzhu: > Please fix the test failure.
> 
> ```
> metadata_test.go:30:
> 
>     c.Assert(m.logFile, Equals, logFile)
> 
> ... obtained string = ""
> 
> ... expected string = "ON.000001"
> ```

I'm checking that
GregoryIan: lgtm

Patch:
diff --git a/v4/export/dump.go b/v4/export/dump.go
index 0c64d25..0a1f6e7 100644
--- a/v4/export/dump.go
+++ b/v4/export/dump.go
@@ -3,8 +3,11 @@ package export
 import (
 	"context"
 	"database/sql"
+	"time"
 
 	_ "github.com/go-sql-driver/mysql"
+	"github.com/pingcap/log"
+	"go.uber.org/zap"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -51,6 +54,15 @@ func Dump(conf *Config) (err error) {
 		return err
 	}
 
+	m := newGlobalMetadata(conf.OutputDirPath)
+	// write metadata even if dump failed
+	defer m.writeGlobalMetaData()
+	m.recordStartTime(time.Now())
+	err = m.getGlobalMetaData(pool, conf.ServerInfo.ServerType)
+	if err != nil {
+		log.L().Info("get global metadata failed", zap.Error(err))
+	}
+
 	writer, err := NewSimpleWriter(conf)
 	if err != nil {
 		return err
@@ -59,6 +71,8 @@ func Dump(conf *Config) (err error) {
 		return err
 	}
 
+	m.recordFinishTime(time.Now())
+
 	return conCtrl.TearDown()
 }
 
diff --git a/v4/export/metadata.go b/v4/export/metadata.go
new file mode 100644
index 0000000..a4612f6
--- /dev/null
+++ b/v4/export/metadata.go
@@ -0,0 +1,136 @@
+package export
+
+import (
+	"database/sql"
+	"errors"
+	"path"
+	"time"
+)
+
+type globalMetadata struct {
+	logFile string
+	pos string
+	gtidSet string
+
+	filePath string
+	startTime time.Time
+	finishTime time.Time
+}
+
+const (
+	metadataPath       = "metadata"
+	metadataTimeLayout = "2006-01-02 15:04:05"
+
+	fileFieldIndex = 0
+	posFieldIndex = 1
+	gtidSetFieldIndex = 4
+
+	mariadbShowMasterStatusFieldNum = 4
+)
+
+func newGlobalMetadata(outputDir string) *globalMetadata {
+	return &globalMetadata{
+		filePath: path.Join(outputDir, metadataPath),
+	}
+}
+
+func (m globalMetadata) String() string {
+	str := ""
+	if m.startTime.IsZero() {
+		return str
+	}
+	str += "Started dump at: " + m.startTime.Format(metadataTimeLayout) + "\n"
+
+	str += "SHOW MASTER STATUS:"
+	if m.logFile != "" {
+		str += "\t\tLog: " + m.logFile + "\n"
+	}
+	if m.pos != "" {
+		str += "\t\tPos: " + m.pos + "\n"
+	}
+	if m.gtidSet != "" {
+		str += "\t\tGTID:" + m.gtidSet + "\n"
+	}
+
+	if m.finishTime.IsZero() {
+		return str
+	}
+	str += "Finished dump at: " + m.finishTime.Format(metadataTimeLayout) + "\n"
+	return str
+}
+
+func (m *globalMetadata) recordStartTime(t time.Time) {
+	m.startTime = t
+}
+
+func (m *globalMetadata) recordFinishTime(t time.Time)  {
+	m.finishTime = t
+}
+
+func (m *globalMetadata) getGlobalMetaData(db *sql.DB, serverType ServerType) error {
+	switch serverType {
+	// For MySQL:
+	// mysql> SHOW MASTER STATUS;
+	// +-----------+----------+--------------+------------------+-------------------------------------------+
+	// | File      | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set                         |
+	// +-----------+----------+--------------+------------------+-------------------------------------------+
+	// | ON.000001 |     7502 |              |                  | 6ce40be3-e359-11e9-87e0-36933cb0ca5a:1-29 |
+	// +-----------+----------+--------------+------------------+-------------------------------------------+
+	// 1 row in set (0.00 sec)
+	//
+	// For TiDB:
+	// mysql> SHOW MASTER STATUS;
+	// +-------------+--------------------+--------------+------------------+-------------------+
+	// | File        | Position           | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set |
+	// +-------------+--------------------+--------------+------------------+-------------------+
+	// | tidb-binlog | 415195906970746880 |              |                  |                   |
+	// +-------------+--------------------+--------------+------------------+-------------------+
+	// 1 row in set (0.00 sec)
+	case ServerTypeMySQL, ServerTypeTiDB:
+		str, err := ShowMasterStatus(db, showMasterStatusFieldNum)
+		if err != nil {
+			return err
+		}
+		m.logFile = str[fileFieldIndex]
+		m.pos = str[posFieldIndex]
+		m.gtidSet = str[gtidSetFieldIndex]
+	// For MariaDB:
+	// SHOW MASTER STATUS;
+	// +--------------------+----------+--------------+------------------+
+	// | File               | Position | Binlog_Do_DB | Binlog_Ignore_DB |
+	// +--------------------+----------+--------------+------------------+
+	// | mariadb-bin.000016 |      475 |              |                  |
+	// +--------------------+----------+--------------+------------------+
+	// SELECT @@global.gtid_binlog_pos;
+	// +--------------------------+
+	// | @@global.gtid_binlog_pos |
+	// +--------------------------+
+	// | 0-1-2                    |
+	// +--------------------------+
+	// 1 row in set (0.00 sec)
+	case ServerTypeMariaDB:
+		str, err := ShowMasterStatus(db, mariadbShowMasterStatusFieldNum)
+		if err != nil {
+			return err
+		}
+		m.logFile = str[fileFieldIndex]
+		m.pos = str[posFieldIndex]
+		err = db.QueryRow("SELECT @@global.gtid_binlog_pos").Scan(&m.gtidSet)
+		if err != nil {
+			return err
+		}
+	default:
+		return errors.New("unsupported serverType" +serverType.String() + "for getGlobalMetaData")
+	}
+	return nil
+}
+
+func (m globalMetadata) writeGlobalMetaData() error {
+	fileWriter, tearDown, err := buildFileWriter(m.filePath)
+	if err != nil {
+		return err
+	}
+	defer tearDown()
+
+	return write(fileWriter, m.String())
+}
\ No newline at end of file


Test patch:
diff --git a/v4/export/metadata_test.go b/v4/export/metadata_test.go
new file mode 100644
index 0000000..78d530d
--- /dev/null
+++ b/v4/export/metadata_test.go
@@ -0,0 +1,61 @@
+package export
+
+import (
+	"github.com/DATA-DOG/go-sqlmock"
+	. "github.com/pingcap/check"
+	"path"
+)
+
+var _ = Suite(&testMetaDataSuite{})
+
+type testMetaDataSuite struct{}
+
+func (s *testMetaDataSuite) TestMysqlMetaData(c *C) {
+	db, mock, err := sqlmock.New()
+	c.Assert(err, IsNil)
+	defer db.Close()
+
+	logFile := "ON.000001"
+	pos := "7502"
+	gtidSet := "6ce40be3-e359-11e9-87e0-36933cb0ca5a:1-29"
+	rows := sqlmock.NewRows([]string{"File", "Position", "Binlog_Do_DB", "Binlog_Ignore_DB", "Executed_Gtid_Set"}).
+		AddRow(logFile, pos, "", "", gtidSet)
+	mock.ExpectQuery("SHOW MASTER STATUS").WillReturnRows(rows)
+
+	testFilePath := "/test"
+	m := newGlobalMetadata(testFilePath)
+	c.Assert(m.getGlobalMetaData(db, ServerTypeMySQL), IsNil)
+	c.Assert(m.filePath, Equals, path.Join(testFilePath, metadataPath))
+
+	c.Assert(m.logFile, Equals, logFile)
+	c.Assert(m.pos, Equals, pos)
+	c.Assert(m.gtidSet, Equals, gtidSet)
+
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
+}
+
+func (s *testMetaDataSuite) TestMariaDBMetaData(c *C) {
+	db, mock, err := sqlmock.New()
+	c.Assert(err, IsNil)
+	defer db.Close()
+
+	logFile := "mariadb-bin.000016"
+	pos := "475"
+	gtidSet := "0-1-2"
+	rows := sqlmock.NewRows([]string{"File", "Position", "Binlog_Do_DB", "Binlog_Ignore_DB"}).
+		AddRow(logFile, pos, "", "")
+	mock.ExpectQuery("SHOW MASTER STATUS").WillReturnRows(rows)
+	rows = sqlmock.NewRows([]string{"@@global.gtid_binlog_pos"}).
+		AddRow(gtidSet)
+	mock.ExpectQuery("SELECT @@global.gtid_binlog_pos").WillReturnRows(rows)
+	testFilePath := "/test"
+	m := newGlobalMetadata(testFilePath)
+	c.Assert(m.getGlobalMetaData(db, ServerTypeMariaDB), IsNil)
+	c.Assert(m.filePath, Equals, path.Join(testFilePath, metadataPath))
+
+	c.Assert(m.logFile, Equals, logFile)
+	c.Assert(m.pos, Equals, pos)
+	c.Assert(m.gtidSet, Equals, gtidSet)
+
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
+}
\ No newline at end of file