Repository: pingcap/dumpling
Pull request #78 (2020-05-22 05:28:32)

check rows error after dump
After dump job is completed, we should check `rows.Err()` to make sure dumpling has successfully dumped all data.


codecov-commenter: # [Codecov](https://codecov.io/gh/pingcap/dumpling/pull/78?src=pr&el=h1) Report
> Merging [#78](https://codecov.io/gh/pingcap/dumpling/pull/78?src=pr&el=desc) into [master](https://codecov.io/gh/pingcap/dumpling/commit/0eb24b8cb0ae3a2823b77c8e6efd34870a940c6c&el=desc) will **increase** coverage by `0.46%`.
> The diff coverage is `100.00%`.


```diff
@@            Coverage Diff             @@
##           master      #78      +/-   ##
==========================================
+ Coverage   56.66%   57.12%   +0.46%     
==========================================
  Files          16       16              
  Lines        1576     1593      +17     
==========================================
+ Hits          893      910      +17     
  Misses        608      608              
  Partials       75       75              
```


Patch:
diff --git a/v4/export/ir.go b/v4/export/ir.go
index 72d2ecb..4b4e073 100644
--- a/v4/export/ir.go
+++ b/v4/export/ir.go
@@ -24,6 +24,7 @@ type TableDataIR interface {
 type SQLRowIter interface {
 	Decode(RowReceiver) error
 	Next()
+	Error() error
 	HasNext() bool
 	HasNextSQLRowIter() bool
 	NextSQLRowIter() SQLRowIter
diff --git a/v4/export/ir_impl.go b/v4/export/ir_impl.go
index e2b21fb..7fee859 100644
--- a/v4/export/ir_impl.go
+++ b/v4/export/ir_impl.go
@@ -38,6 +38,10 @@ func (iter *rowIter) Decode(row RowReceiver) error {
 	return decodeFromRows(iter.rows, iter.args, row)
 }
 
+func (iter *rowIter) Error() error {
+	return iter.rows.Err()
+}
+
 func (iter *rowIter) Next() {
 	iter.hasNext = iter.rows.Next()
 }
@@ -78,6 +82,10 @@ func (c *fileRowIter) Decode(row RowReceiver) error {
 	return nil
 }
 
+func (c *fileRowIter) Error() error {
+	return c.rowIter.Error()
+}
+
 func (c *fileRowIter) Next() {
 	c.rowIter.Next()
 }
diff --git a/v4/export/writer_util.go b/v4/export/writer_util.go
index d831480..0f371cb 100644
--- a/v4/export/writer_util.go
+++ b/v4/export/writer_util.go
@@ -179,6 +179,9 @@ func WriteInsert(tblIR TableDataIR, w io.Writer) error {
 	}
 	close(wp.input)
 	<-wp.closed
+	if err = fileRowIter.Error(); err != nil {
+		return err
+	}
 	return wp.Error()
 }
 
@@ -261,6 +264,9 @@ func WriteInsertInCsv(tblIR TableDataIR, w io.Writer, noHeader bool, csvNullValu
 	}
 	close(wp.input)
 	<-wp.closed
+	if err = fileRowIter.Error(); err != nil {
+		return err
+	}
 	return wp.Error()
 }
 


Test patch:
diff --git a/v4/export/test_util.go b/v4/export/test_util.go
index ee95e1f..799bdf3 100644
--- a/v4/export/test_util.go
+++ b/v4/export/test_util.go
@@ -84,6 +84,7 @@ type mockTableIR struct {
 	colTypes        []string
 	colNames        []string
 	escapeBackSlash bool
+	rowErr          error
 }
 
 func (m *mockTableIR) DatabaseName() string {
@@ -132,6 +133,9 @@ func (m *mockTableIR) Rows() SQLRowIter {
 	}
 	defer db.Close()
 	mock.ExpectQuery("select 1").WillReturnRows(mockRows)
+	if m.rowErr != nil {
+		mockRows.RowError(len(m.data)-1, m.rowErr)
+	}
 	rows, err := db.Query("select 1")
 	if err != nil {
 		panic(fmt.Sprintf("sqlmock.New return error: %v", err))
@@ -154,3 +158,15 @@ func newMockTableIR(databaseName, tableName string, data [][]driver.Value, speci
 		colTypes:      colTypes,
 	}
 }
+
+func newMockTableIRWithError(databaseName, tableName string, data [][]driver.Value, specialComments, colTypes []string, err error) TableDataIR {
+	return &mockTableIR{
+		dbName:        databaseName,
+		tblName:       tableName,
+		data:          data,
+		specCmt:       specialComments,
+		selectedField: "*",
+		colTypes:      colTypes,
+		rowErr:        err,
+	}
+}
diff --git a/v4/export/writer_util_test.go b/v4/export/writer_util_test.go
index 52dd1d0..7c9d879 100644
--- a/v4/export/writer_util_test.go
+++ b/v4/export/writer_util_test.go
@@ -2,6 +2,7 @@ package export
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"strings"
 	"testing"
@@ -71,6 +72,34 @@ func (s *testUtilSuite) TestWriteInsert(c *C) {
 	c.Assert(bf.String(), Equals, expected)
 }
 
+func (s *testUtilSuite) TestWriteInsertReturnsError(c *C) {
+	data := [][]driver.Value{
+		{"1", "male", "bob@mail.com", "020-1234", nil},
+		{"2", "female", "sarah@mail.com", "020-1253", "healthy"},
+		{"3", "male", "john@mail.com", "020-1256", "healthy"},
+		{"4", "female", "sarah@mail.com", "020-1235", "healthy"},
+	}
+	colTypes := []string{"INT", "SET", "VARCHAR", "VARCHAR", "TEXT"}
+	specCmts := []string{
+		"/*!40101 SET NAMES binary*/;",
+		"/*!40014 SET FOREIGN_KEY_CHECKS=0*/;",
+	}
+	// row errors at last line
+	rowErr := errors.New("mock row error")
+	tableIR := newMockTableIRWithError("test", "employee", data, specCmts, colTypes, rowErr)
+	bf := &bytes.Buffer{}
+
+	err := WriteInsert(tableIR, bf)
+	c.Assert(err, Equals, rowErr)
+	expected := "/*!40101 SET NAMES binary*/;\n" +
+		"/*!40014 SET FOREIGN_KEY_CHECKS=0*/;\n" +
+		"INSERT INTO `employee` VALUES\n" +
+		"(1,'male','bob@mail.com','020-1234',NULL),\n" +
+		"(2,'female','sarah@mail.com','020-1253','healthy'),\n" +
+		"(3,'male','john@mail.com','020-1256','healthy');\n"
+	c.Assert(bf.String(), Equals, expected)
+}
+
 func (s *testUtilSuite) TestWriteInsertInCsv(c *C) {
 	data := [][]driver.Value{
 		{"1", "male", "bob@mail.com", "020-1234", nil},