Repository: pingcap/dumpling
Pull request #17 (2019-12-28 10:52:46)

refactor: move the functions from dump.go to sql.go
Changes:
- add `sql.go` and move related functions into it.
- merge/split some functions.
- rename dump_test.go to sql_test.go

codecov-io: # [Codecov](https://codecov.io/gh/pingcap/dumpling/pull/17?src=pr&el=h1) Report
> Merging [#17](https://codecov.io/gh/pingcap/dumpling/pull/17?src=pr&el=desc) into [master](https://codecov.io/gh/pingcap/dumpling/commit/aa490c8648475174e20e42138b9de982a9d4ddf6?src=pr&el=desc) will **increase** coverage by `0.23%`.
> The diff coverage is `36.61%`.


```diff
@@            Coverage Diff             @@
##           master      #17      +/-   ##
==========================================
+ Coverage   38.96%   39.19%   +0.23%     
==========================================
  Files           9       10       +1     
  Lines         521      523       +2     
==========================================
+ Hits          203      205       +2     
  Misses        299      299              
  Partials       19       19
```


Patch:
diff --git a/v4/export/dump.go b/v4/export/dump.go
index 067628e..d63400c 100644
--- a/v4/export/dump.go
+++ b/v4/export/dump.go
@@ -3,7 +3,6 @@ package export
 import (
 	"context"
 	"database/sql"
-	"fmt"
 	"strings"
 	"sync"
 
@@ -35,105 +34,22 @@ func prepareMeta(conf *Config) (databases []string, info ServerInfo, err error)
 		return nil, ServerInfoUnknown, withStack(err)
 	}
 	defer pool.Close()
-	databases, err = getDumpingDatabaseNames(pool, conf)
-	if err != nil {
-		return nil, ServerInfoUnknown, withStack(err)
-	}
-	info, err = detectServerInfo(pool)
-	if err != nil {
-		return databases, ServerInfoUnknown, withStack(err)
-	}
-	return
-}
 
-func getDumpingDatabaseNames(pool *sql.DB, conf *Config) ([]string, error) {
 	if conf.Database == "" {
-		return showDatabases(pool)
-	}
-	return strings.Split(conf.Database, ","), nil
-}
-
-func detectServerInfo(db *sql.DB) (ServerInfo, error) {
-	var versionInfo string
-	handleOneRow := func(rows *sql.Rows) error {
-		return rows.Scan(&versionInfo)
-	}
-	err := simpleQuery(db, "SELECT version()", handleOneRow)
-	if err != nil {
-		return ServerInfoUnknown, withStack(err)
-	}
-	return ParseServerInfo(versionInfo), nil
-}
-
-func simpleQuery(db *sql.DB, sql string, handleOneRow func(*sql.Rows) error) error {
-	rows, err := db.Query(sql)
-	if err != nil {
-		return withStack(err)
-	}
-
-	for rows.Next() {
-		if err := handleOneRow(rows); err != nil {
-			rows.Close()
-			return withStack(err)
+		databases, err = ShowDatabases(pool)
+		if err != nil {
+			return nil, ServerInfoUnknown, withStack(err)
 		}
+	} else {
+		databases = strings.Split(conf.Database, ",")
 	}
-	return nil
-}
 
-type oneStrColumnTable struct {
-	data []string
-}
-
-func (o *oneStrColumnTable) handleOneRow(rows *sql.Rows) error {
-	var str string
-	if err := rows.Scan(&str); err != nil {
-		return withStack(err)
-	}
-	o.data = append(o.data, str)
-	return nil
-}
-
-func showDatabases(db *sql.DB) ([]string, error) {
-	var res oneStrColumnTable
-	if err := simpleQuery(db, "SHOW DATABASES", res.handleOneRow); err != nil {
-		return nil, err
-	}
-	return res.data, nil
-}
-
-// showTables shows the tables of a database, the caller should use the correct database.
-func showTables(db *sql.DB) ([]string, error) {
-	var res oneStrColumnTable
-	if err := simpleQuery(db, "SHOW TABLES", res.handleOneRow); err != nil {
-		return nil, err
-	}
-	return res.data, nil
-}
-
-func showCreateDatabase(db *sql.DB, database string) (string, error) {
-	var oneRow [2]string
-	handleOneRow := func(rows *sql.Rows) error {
-		return rows.Scan(&oneRow[0], &oneRow[1])
-	}
-	query := fmt.Sprintf("SHOW CREATE DATABASE %s", database)
-	err := simpleQuery(db, query, handleOneRow)
+	versionStr, err := SelectVersion(pool)
 	if err != nil {
-		return "", err
-	}
-	return oneRow[1], nil
-}
-
-func showCreateTable(db *sql.DB, database, table string) (string, error) {
-	var oneRow [2]string
-	handleOneRow := func(rows *sql.Rows) error {
-		return rows.Scan(&oneRow[0], &oneRow[1])
-	}
-	query := fmt.Sprintf("SHOW CREATE TABLE %s.%s", database, table)
-	err := simpleQuery(db, query, handleOneRow)
-	if err != nil {
-		return "", err
+		return databases, ServerInfoUnknown, withStack(err)
 	}
-	return oneRow[1], nil
+	info = ParseServerInfo(versionStr)
+	return
 }
 
 func dumpDatabase(ctx context.Context, conf *Config, dbName string, writer Writer) error {
@@ -144,7 +60,7 @@ func dumpDatabase(ctx context.Context, conf *Config, dbName string, writer Write
 	}
 	defer db.Close()
 
-	createDatabaseSQL, err := showCreateDatabase(db, dbName)
+	createDatabaseSQL, err := ShowCreateDatabase(db, dbName)
 	if err != nil {
 		return err
 	}
@@ -152,7 +68,7 @@ func dumpDatabase(ctx context.Context, conf *Config, dbName string, writer Write
 		return err
 	}
 
-	tables, err := showTables(db)
+	tables, err := ShowTables(db)
 	if err != nil {
 		return err
 	}
@@ -164,7 +80,7 @@ func dumpDatabase(ctx context.Context, conf *Config, dbName string, writer Write
 	for i, table := range tables {
 		go func(ith int, table string, wg *sync.WaitGroup, res []error) {
 			defer wg.Done()
-			createTableSQL, err := showCreateTable(db, dbName, table)
+			createTableSQL, err := ShowCreateTable(db, dbName, table)
 			if err != nil {
 				res[ith] = err
 				return
@@ -175,7 +91,7 @@ func dumpDatabase(ctx context.Context, conf *Config, dbName string, writer Write
 			}
 
 			rateLimit.getToken()
-			tableIR, err := dumpTable(conf, db, dbName, table)
+			tableIR, err := SelectAllFromTable(conf, db, dbName, table)
 			defer rateLimit.putToken()
 			if err != nil {
 				res[ith] = err
@@ -196,120 +112,3 @@ func dumpDatabase(ctx context.Context, conf *Config, dbName string, writer Write
 	}
 	return nil
 }
-
-func getColumnTypes(db *sql.DB, table string) ([]*sql.ColumnType, error) {
-	rows, err := db.Query(fmt.Sprintf("SELECT * FROM %s LIMIT 1", table))
-	if err != nil {
-		return nil, withStack(err)
-	}
-	defer rows.Close()
-	return rows.ColumnTypes()
-}
-
-type tableDumper interface {
-	handleOneRow(ctx context.Context, rows *sql.Rows) error
-	prepareColumns(ctx context.Context, colTypes []*sql.ColumnType)
-	finishTable(ctx context.Context)
-}
-
-func dumpTable(conf *Config, db *sql.DB, database, table string) (TableDataIR, error) {
-	colTypes, err := getColumnTypes(db, table)
-	if err != nil {
-		return nil, err
-	}
-
-	query, err := buildSelectAllQuery(conf, db, database, table)
-	if err != nil {
-		return nil, err
-	}
-	rows, err := db.Query(query)
-	if err != nil {
-		return nil, withStack(err)
-	}
-
-	return &tableData{
-		database: database,
-		table:    table,
-		rows:     rows,
-		colTypes: colTypes,
-	}, nil
-}
-
-func buildSelectAllQuery(conf *Config, db *sql.DB, database, table string) (string, error) {
-	var query strings.Builder
-	query.WriteString("SELECT * FROM ")
-	query.WriteString(database)
-	query.WriteString(".")
-	query.WriteString(table)
-	if conf.SortByPk {
-		orderByClause, err := buildOrderByClause(conf, db, database, table)
-		if err != nil {
-			return "", err
-		}
-		if orderByClause != "" {
-			query.WriteString(" ")
-			query.WriteString(orderByClause)
-		}
-	}
-	return query.String(), nil
-}
-
-const errBadFieldCode = 1054
-
-func buildOrderByClause(conf *Config, db *sql.DB, database, table string) (string, error) {
-	if conf.ServerInfo.ServerType == ServerTypeTiDB {
-		tiDBRowIDQuery := fmt.Sprintf("SELECT _tidb_rowid from %s.%s LIMIT 0", database, table)
-		isBadField, otherErr := queryReturnExpectedErrorCode(db, tiDBRowIDQuery, errBadFieldCode)
-		if otherErr != nil {
-			return "", otherErr
-		}
-		if isBadField {
-			return "", nil
-		}
-		return "ORDER BY _tidb_rowid", nil
-	}
-	pkName, err := getPrimaryKeyName(db, database, table)
-	if err != nil {
-		return "", err
-	}
-	tableContainsPriKey := pkName != ""
-	if tableContainsPriKey {
-		return fmt.Sprintf("ORDER BY %s", pkName), nil
-	}
-	return "", nil
-}
-
-func queryReturnExpectedErrorCode(db *sql.DB, sql string, errCode int) (bool, error) {
-	_, err := db.Exec(sql)
-	if err != nil {
-		errMsg := strings.ToLower(err.Error())
-		if strings.Contains(errMsg, fmt.Sprintf("%d", errCode)) {
-			return true, nil
-		}
-		return false, withStack(err)
-	}
-	return false, nil
-}
-
-func getPrimaryKeyName(db *sql.DB, database, table string) (string, error) {
-	priKeyQuery := `SELECT column_name FROM information_schema.columns
-		WHERE table_schema = ? AND table_name = ? AND column_key = 'PRI';`
-	stmt, err := db.Prepare(priKeyQuery)
-	if err != nil {
-		return "", err
-	}
-	defer stmt.Close()
-	rows, err := stmt.Query(database, table)
-	if err != nil {
-		return "", withStack(err)
-	}
-
-	var colName string
-	for rows.Next() {
-		if err := rows.Scan(&colName); err != nil {
-			rows.Close()
-			return "", withStack(err)
-		}
-	}
-	return colName, nil
-}
diff --git a/v4/export/sql.go b/v4/export/sql.go
new file mode 100644
index 0000000..0f7e26f
--- /dev/null
+++ b/v4/export/sql.go
@@ -0,0 +1,201 @@
+package export
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+func ShowDatabases(db *sql.DB) ([]string, error) {
+	var res oneStrColumnTable
+	if err := simpleQuery(db, "SHOW DATABASES", res.handleOneRow); err != nil {
+		return nil, err
+	}
+	return res.data, nil
+}
+
+// ShowTables shows the tables of a database, the caller should use the correct database.
+func ShowTables(db *sql.DB) ([]string, error) {
+	var res oneStrColumnTable
+	if err := simpleQuery(db, "SHOW TABLES", res.handleOneRow); err != nil {
+		return nil, err
+	}
+	return res.data, nil
+}
+
+func ShowCreateDatabase(db *sql.DB, database string) (string, error) {
+	var oneRow [2]string
+	handleOneRow := func(rows *sql.Rows) error {
+		return rows.Scan(&oneRow[0], &oneRow[1])
+	}
+	query := fmt.Sprintf("SHOW CREATE DATABASE %s", database)
+	err := simpleQuery(db, query, handleOneRow)
+	if err != nil {
+		return "", err
+	}
+	return oneRow[1], nil
+}
+
+func ShowCreateTable(db *sql.DB, database, table string) (string, error) {
+	var oneRow [2]string
+	handleOneRow := func(rows *sql.Rows) error {
+		return rows.Scan(&oneRow[0], &oneRow[1])
+	}
+	query := fmt.Sprintf("SHOW CREATE TABLE %s.%s", database, table)
+	err := simpleQuery(db, query, handleOneRow)
+	if err != nil {
+		return "", err
+	}
+	return oneRow[1], nil
+}
+
+func SelectVersion(db *sql.DB) (string, error) {
+	var versionInfo string
+	handleOneRow := func(rows *sql.Rows) error {
+		return rows.Scan(&versionInfo)
+	}
+	err := simpleQuery(db, "SELECT version()", handleOneRow)
+	if err != nil {
+		return "", withStack(err)
+	}
+	return versionInfo, nil
+}
+
+func SelectAllFromTable(conf *Config, db *sql.DB, database, table string) (TableDataIR, error) {
+	colTypes, err := GetColumnTypes(db, table)
+	if err != nil {
+		return nil, err
+	}
+
+	query, err := buildSelectAllQuery(conf, db, database, table)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := db.Query(query)
+	if err != nil {
+		return nil, withStack(err)
+	}
+
+	return &tableData{
+		database: database,
+		table:    table,
+		rows:     rows,
+		colTypes: colTypes,
+	}, nil
+}
+
+func buildSelectAllQuery(conf *Config, db *sql.DB, database, table string) (string, error) {
+	var query strings.Builder
+	query.WriteString("SELECT * FROM ")
+	query.WriteString(database)
+	query.WriteString(".")
+	query.WriteString(table)
+	if conf.SortByPk {
+		orderByClause, err := buildOrderByClause(conf, db, database, table)
+		if err != nil {
+			return "", err
+		}
+		if orderByClause != "" {
+			query.WriteString(" ")
+			query.WriteString(orderByClause)
+		}
+	}
+	return query.String(), nil
+}
+
+func buildOrderByClause(conf *Config, db *sql.DB, database, table string) (string, error) {
+	if conf.ServerInfo.ServerType == ServerTypeTiDB {
+		ok, err := SelectTiDBRowID(db, database, table)
+		if err != nil {
+			return "", err
+		}
+		if ok {
+			return "ORDER BY _tidb_rowid", nil
+		} else {
+			return "", nil
+		}
+	}
+	pkName, err := GetPrimaryKeyName(db, database, table)
+	if err != nil {
+		return "", err
+	}
+	tableContainsPriKey := pkName != ""
+	if tableContainsPriKey {
+		return fmt.Sprintf("ORDER BY %s", pkName), nil
+	}
+	return "", nil
+}
+
+func SelectTiDBRowID(db *sql.DB, database, table string) (bool, error) {
+	const errBadFieldCode = 1054
+	tiDBRowIDQuery := fmt.Sprintf("SELECT _tidb_rowid from %s.%s LIMIT 0", database, table)
+	_, err := db.Exec(tiDBRowIDQuery)
+	if err != nil {
+		errMsg := strings.ToLower(err.Error())
+		if strings.Contains(errMsg, fmt.Sprintf("%d", errBadFieldCode)) {
+			return false, nil
+		}
+		return false, withStack(err)
+	}
+	return true, nil
+}
+
+func GetColumnTypes(db *sql.DB, table string) ([]*sql.ColumnType, error) {
+	rows, err := db.Query(fmt.Sprintf("SELECT * FROM %s LIMIT 1", table))
+	if err != nil {
+		return nil, withStack(err)
+	}
+	defer rows.Close()
+	return rows.ColumnTypes()
+}
+
+func GetPrimaryKeyName(db *sql.DB, database, table string) (string, error) {
+	priKeyQuery := `SELECT column_name FROM information_schema.columns
+		WHERE table_schema = ? AND table_name = ? AND column_key = 'PRI';`
+	stmt, err := db.Prepare(priKeyQuery)
+	if err != nil {
+		return "", err
+	}
+	defer stmt.Close()
+	rows, err := stmt.Query(database, table)
+	if err != nil {
+		return "", withStack(err)
+	}
+
+	var colName string
+	for rows.Next() {
+		if err := rows.Scan(&colName); err != nil {
+			rows.Close()
+			return "", withStack(err)
+		}
+	}
+	return colName, nil
+}
+
+type oneStrColumnTable struct {
+	data []string
+}
+
+func (o *oneStrColumnTable) handleOneRow(rows *sql.Rows) error {
+	var str string
+	if err := rows.Scan(&str); err != nil {
+		return withStack(err)
+	}
+	o.data = append(o.data, str)
+	return nil
+}
+
+func simpleQuery(db *sql.DB, sql string, handleOneRow func(*sql.Rows) error) error {
+	rows, err := db.Query(sql)
+	if err != nil {
+		return withStack(err)
+	}
+
+	for rows.Next() {
+		if err := handleOneRow(rows); err != nil {
+			rows.Close()
+			return withStack(err)
+		}
+	}
+	return nil
+}


Test patch:
diff --git a/v4/export/dump_test.go b/v4/export/sql_test.go
similarity index 98%
rename from v4/export/dump_test.go
rename to v4/export/sql_test.go
index a196ce5..383d49b 100644
--- a/v4/export/dump_test.go
+++ b/v4/export/sql_test.go
@@ -35,8 +35,9 @@ func (s *testDumpSuite) TestDetectServerInfo(c *C) {
 		rows := sqlmock.NewRows([]string{"version"}).AddRow(r)
 		mock.ExpectQuery("SELECT version()").WillReturnRows(rows)
 
-		info, err := detectServerInfo(db)
+		verStr, err := SelectVersion(db)
 		c.Assert(err, IsNil, cmt)
+		info := ParseServerInfo(verStr)
 		c.Assert(info.ServerType, Equals, serverTp, cmt)
 		c.Assert(info.ServerVersion == nil, Equals, expectVer == nil, cmt)
 		if info.ServerVersion == nil {