Repository: pingcap/dumpling
Pull request #124 (2020-07-18 15:14:17)

parse tso for snapshot
<!--
Thank you for contributing to Dumpling! Please read the [CONTRIBUTING](https://github.com/pingcap/dumpling/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
-->

### What problem does this PR solve? <!--add issue link with summary if exists-->
dumpling should support `datetime` type snapshot. But dumpling doesn't parse tso for `datetime` so that it fails set pd gcTS.

### What is changed and how it works?
Use `SELECT unix_timestamp("%s")` to get unix timestamp to avoid time zone problem.

### Check List <!--REMOVE the items that are not applicable-->

Tests <!-- At least one of them must be included. -->

 - Unit test
 - Integration test

Related changes

 - Need to cherry-pick to the release branch
 - Need to update the documentation
 
### Release note

<!-- bugfixes or new feature need a release note, must in the form of a list, such as

- support -T/--tables-list argument

or if no need to be included in the release note, just add the following line

- No release note
-->
- fix bug dumpling fails to parse tso bug when tidb verison >= v4.0.0


codecov[bot]: # [Codecov](https://codecov.io/gh/pingcap/dumpling/pull/124?src=pr&el=h1) Report
> Merging [#124](https://codecov.io/gh/pingcap/dumpling/pull/124?src=pr&el=desc) into [master](https://codecov.io/gh/pingcap/dumpling/commit/6d1ba108d47483d35bc2c7dd74cdeec38a17e64e&el=desc) will **increase** coverage by `0.60%`.
> The diff coverage is `68.75%`.


```diff
@@            Coverage Diff             @@
##           master     #124      +/-   ##
==========================================
+ Coverage   51.59%   52.19%   +0.60%     
==========================================
  Files          17       17              
  Lines        1853     1868      +15     
==========================================
+ Hits          956      975      +19     
+ Misses        829      825       -4     
  Partials       68       68              
```


Patch:
diff --git a/v4/export/dump.go b/v4/export/dump.go
index f9c7a4c..4d9901e 100644
--- a/v4/export/dump.go
+++ b/v4/export/dump.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"database/sql"
 	"errors"
-	"strconv"
 	"strings"
 	"time"
 
@@ -89,7 +88,7 @@ func Dump(pCtx context.Context, conf *Config) (err error) {
 	}
 
 	if doPdGC {
-		snapshotTS, err := strconv.ParseUint(conf.Snapshot, 10, 64)
+		snapshotTS, err := parseSnapshotToTSO(pool, conf.Snapshot)
 		if err != nil {
 			return err
 		}
diff --git a/v4/export/sql.go b/v4/export/sql.go
index 94c7122..4a9396e 100644
--- a/v4/export/sql.go
+++ b/v4/export/sql.go
@@ -632,6 +632,28 @@ found:
 	return uint64(estRows)
 }
 
+func parseSnapshotToTSO(pool *sql.DB, snapshot string) (uint64, error) {
+	snapshotTS, err := strconv.ParseUint(snapshot, 10, 64)
+	if err == nil {
+		return snapshotTS, nil
+	}
+	var tso sql.NullInt64
+	err = simpleQueryWithArgs(pool, func(rows *sql.Rows) error {
+		err := rows.Scan(&tso)
+		if err != nil {
+			return err
+		}
+		if !tso.Valid {
+			return fmt.Errorf("snapshot %s format not supported. please use tso or '2006-01-02 15:04:05' format time", snapshot)
+		}
+		return nil
+	}, "SELECT unix_timestamp(?)", snapshot)
+	if err != nil {
+		return 0, withStack(err)
+	}
+	return (uint64(tso.Int64)<<18)*1000 + 1, nil
+}
+
 func buildWhereCondition(conf *Config, where string) string {
 	var query strings.Builder
 	separator := "WHERE"


Test patch:
diff --git a/v4/export/sql_test.go b/v4/export/sql_test.go
index 01e6f11..79bc2b1 100644
--- a/v4/export/sql_test.go
+++ b/v4/export/sql_test.go
@@ -190,6 +190,32 @@ func (s *testDumpSuite) TestBuildSelectField(c *C) {
 
 }
 
+func (s *testDumpSuite) TestParseSnapshotToTSO(c *C) {
+	db, mock, err := sqlmock.New()
+	c.Assert(err, IsNil)
+	defer db.Close()
+
+	snapshot := "2020/07/18 20:31:50"
+	var unixTimeStamp uint64 = 1595075510
+	// generate columns valid snapshot
+	mock.ExpectQuery(`SELECT unix_timestamp(?)`).
+		WithArgs(sqlmock.AnyArg()).
+		WillReturnRows(sqlmock.NewRows([]string{`unix_timestamp("2020/07/18 20:31:50")`}).AddRow(1595075510))
+	tso, err := parseSnapshotToTSO(db, snapshot)
+	c.Assert(err, IsNil)
+	c.Assert(tso, Equals, (unixTimeStamp<<18)*1000+1)
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
+
+	// generate columns not valid snapshot
+	mock.ExpectQuery(`SELECT unix_timestamp(?)`).
+		WithArgs(sqlmock.AnyArg()).
+		WillReturnRows(sqlmock.NewRows([]string{`unix_timestamp("XXYYZZ")`}).AddRow(nil))
+	tso, err = parseSnapshotToTSO(db, "XXYYZZ")
+	c.Assert(err, ErrorMatches, "snapshot XXYYZZ format not supported. please use tso or '2006-01-02 15:04:05' format time")
+	c.Assert(tso, Equals, uint64(0))
+	c.Assert(mock.ExpectationsWereMet(), IsNil)
+}
+
 func makeVersion(major, minor, patch int64, preRelease string) *semver.Version {
 	return &semver.Version{
 		Major:      major,