Repository: pingcap/dumpling
Pull request #99 (2020-06-09 12:11:15)

Resolve csv data mismatch
resolve https://github.com/pingcap/dumpling/issues/98

codecov-commenter: # [Codecov](https://codecov.io/gh/pingcap/dumpling/pull/99?src=pr&el=h1) Report
> Merging [#99](https://codecov.io/gh/pingcap/dumpling/pull/99?src=pr&el=desc) into [master](https://codecov.io/gh/pingcap/dumpling/commit/0175843056a6068dd2f64afca6277d890934b63c&el=desc) will **increase** coverage by `0.19%`.
> The diff coverage is `92.30%`.


```diff
@@            Coverage Diff             @@
##           master      #99      +/-   ##
==========================================
+ Coverage   56.19%   56.39%   +0.19%     
==========================================
  Files          16       16              
  Lines        1589     1596       +7     
==========================================
+ Hits          893      900       +7     
  Misses        623      623              
  Partials       73       73              
```


Patch:
diff --git a/cmd/dumpling/main.go b/cmd/dumpling/main.go
index ffa150d..5a79c7d 100644
--- a/cmd/dumpling/main.go
+++ b/cmd/dumpling/main.go
@@ -90,12 +90,12 @@ func main() {
 	pflag.StringVarP(&logFile, "logfile", "L", "", "Log file `path`, leave empty to write to console")
 	pflag.StringVar(&logFormat, "logfmt", "text", "Log `format`: {text|json}")
 	pflag.StringVar(&consistency, "consistency", "auto", "Consistency level during dumping: {auto|none|flush|lock|snapshot}")
-	pflag.StringVar(&snapshot, "snapshot", "", "Snapshot position. Valid only when consistency=snapshot")
+	pflag.StringVar(&snapshot, "snapshot", "", "Snapshot position (uint64 from pd timestamp for TiDB). Valid only when consistency=snapshot")
 	pflag.BoolVarP(&noViews, "no-views", "W", true, "Do not dump views")
 	pflag.StringVar(&statusAddr, "status-addr", ":8281", "dumpling API server and pprof addr")
 	pflag.Uint64VarP(&rows, "rows", "r", export.UnspecifiedSize, "Split table into chunks of this many rows, default unlimited")
 	pflag.StringVar(&where, "where", "", "Dump only selected records")
-	pflag.BoolVar(&escapeBackslash, "escape-backslash", true, "use backslash to escape quotation marks")
+	pflag.BoolVar(&escapeBackslash, "escape-backslash", true, "use backslash to escape special characters")
 	pflag.StringVar(&fileType, "filetype", "sql", "The type of export file (sql/csv)")
 	pflag.BoolVar(&noHeader, "no-header", false, "whether not to dump CSV table header")
 	pflag.BoolVarP(&noSchemas, "no-schemas", "m", false, "Do not dump table schemas with the data")
@@ -165,6 +165,7 @@ func main() {
 	conf.NoHeader = noHeader
 	conf.NoSchemas = noSchemas
 	conf.NoData = noData
+	conf.Snapshot = snapshot
 	conf.CsvNullValue = csvNullValue
 	conf.Sql = sql
 	conf.TableFilter = tableFilter
diff --git a/v4/export/sql_type.go b/v4/export/sql_type.go
index e0b7d8c..aabe668 100644
--- a/v4/export/sql_type.go
+++ b/v4/export/sql_type.go
@@ -11,8 +11,6 @@ var colTypeRowReceiverMap = map[string]func() RowReceiverStringer{}
 var nullValue = "NULL"
 var quotationMark byte = '\''
 var doubleQuotationMark byte = '"'
-var quotationMarkNotQuote = []byte{quotationMark}
-var quotationMarkQuote = []byte{quotationMark, quotationMark}
 
 func init() {
 	for _, s := range dataTypeString {
@@ -47,11 +45,19 @@ var dataTypeBin = []string{
 	"BIT",
 }
 
-func escape(s []byte, bf *bytes.Buffer, escapeBackslash bool) {
-	if !escapeBackslash {
-		bf.Write(bytes.ReplaceAll(s, quotationMarkNotQuote, quotationMarkQuote))
+func getEscapeQuotation(escapeBackSlash bool, escapeQuotation byte) byte {
+	if escapeBackSlash {
+		return 0
+	}
+	return escapeQuotation
+}
+
+func escape(s []byte, bf *bytes.Buffer, escapeQuotation byte) {
+	if escapeQuotation != 0 {
+		bf.Write(bytes.ReplaceAll(s, []byte{escapeQuotation}, []byte{escapeQuotation, escapeQuotation}))
 		return
 	}
+
 	var (
 		escape byte
 		last   = 0
@@ -193,7 +199,7 @@ func (s *SQLTypeString) ReportSize() uint64 {
 func (s *SQLTypeString) WriteToBuffer(bf *bytes.Buffer, escapeBackslash bool) {
 	if s.RawBytes != nil {
 		bf.WriteByte(quotationMark)
-		escape(s.RawBytes, bf, escapeBackslash)
+		escape(s.RawBytes, bf, getEscapeQuotation(escapeBackslash, quotationMark))
 		bf.WriteByte(quotationMark)
 	} else {
 		bf.WriteString(nullValue)
@@ -203,7 +209,7 @@ func (s *SQLTypeString) WriteToBuffer(bf *bytes.Buffer, escapeBackslash bool) {
 func (s *SQLTypeString) WriteToBufferInCsv(bf *bytes.Buffer, escapeBackslash bool, csvNullValue string) {
 	if s.RawBytes != nil {
 		bf.WriteByte(doubleQuotationMark)
-		escape(s.RawBytes, bf, escapeBackslash)
+		escape(s.RawBytes, bf, getEscapeQuotation(escapeBackslash, doubleQuotationMark))
 		bf.WriteByte(doubleQuotationMark)
 	} else {
 		bf.WriteString(csvNullValue)
diff --git a/v4/export/writer_util.go b/v4/export/writer_util.go
index ce50310..6c40c2f 100644
--- a/v4/export/writer_util.go
+++ b/v4/export/writer_util.go
@@ -220,7 +220,7 @@ func WriteInsertInCsv(tblIR TableDataIR, w io.Writer, noHeader bool, csvNullValu
 	if !noHeader && len(tblIR.ColumnNames()) != 0 {
 		for i, col := range tblIR.ColumnNames() {
 			bf.WriteByte(doubleQuotationMark)
-			escape([]byte(col), bf, escapeBackSlash)
+			escape([]byte(col), bf, getEscapeQuotation(escapeBackSlash, doubleQuotationMark))
 			bf.WriteByte(doubleQuotationMark)
 			if i != len(tblIR.ColumnTypes())-1 {
 				bf.WriteByte(',')


Test patch:
diff --git a/v4/export/sql_type_test.go b/v4/export/sql_type_test.go
index 3fd5fff..9716c7a 100644
--- a/v4/export/sql_type_test.go
+++ b/v4/export/sql_type_test.go
@@ -15,9 +15,16 @@ func (s *testSqlByteSuite) TestEscape(c *C) {
 	str := []byte(`MWQeWw""'\rNmtGxzGp`)
 	expectStrBackslash := `MWQeWw\"\"\'\\rNmtGxzGp`
 	expectStrWithoutBackslash := `MWQeWw""''\rNmtGxzGp`
-	escape(str, &bf, true)
+	expectStrBackslashDoubleQuote := `MWQeWw""""'\rNmtGxzGp`
+	escape(str, &bf, getEscapeQuotation(true, quotationMark))
 	c.Assert(bf.String(), Equals, expectStrBackslash)
 	bf.Reset()
-	escape(str, &bf, false)
+	escape(str, &bf, getEscapeQuotation(true, doubleQuotationMark))
+	c.Assert(bf.String(), Equals, expectStrBackslash)
+	bf.Reset()
+	escape(str, &bf, getEscapeQuotation(false, quotationMark))
 	c.Assert(bf.String(), Equals, expectStrWithoutBackslash)
+	bf.Reset()
+	escape(str, &bf, getEscapeQuotation(false, doubleQuotationMark))
+	c.Assert(bf.String(), Equals, expectStrBackslashDoubleQuote)
 }