Repository: mailru/go-clickhouse
Pull request #114 (2021-04-08 15:13:49)

Using tsv reader
Reading lines as TSV, instead of using CSV.

Fixes bugs
https://github.com/mailru/go-clickhouse/issues/103
https://github.com/mailru/go-clickhouse/issues/110


Patch:
diff --git a/rows.go b/rows.go
index 9afb36c..5268254 100644
--- a/rows.go
+++ b/rows.go
@@ -2,7 +2,6 @@ package clickhouse
 
 import (
 	"database/sql/driver"
-	"encoding/csv"
 	"fmt"
 	"io"
 	"reflect"
@@ -10,10 +9,12 @@ import (
 	"time"
 )
 
+type dataReader interface {
+	Read() (record []string, err error)
+}
+
 func newTextRows(c *conn, body io.ReadCloser, location *time.Location, useDBLocation bool) (*textRows, error) {
-	tsvReader := csv.NewReader(body)
-	tsvReader.Comma = '\t'
-	tsvReader.LazyQuotes = true
+	tsvReader := newReader(body)
 
 	columns, err := tsvReader.Read()
 	if err != nil {
@@ -60,7 +61,7 @@ func newTextRows(c *conn, body io.ReadCloser, location *time.Location, useDBLoca
 type textRows struct {
 	c        *conn
 	respBody io.ReadCloser
-	tsv      *csv.Reader
+	tsv      dataReader
 	columns  []string
 	types    []string
 	parsers  []DataParser
@@ -81,6 +82,15 @@ func (r *textRows) Next(dest []driver.Value) error {
 		return err
 	}
 
+	// skip row before WITH TOTALS,
+	// not but do not skip an empty line if it is part of the result
+	if len(row) == 1 && row[0] == "" && len(dest) != 1 {
+		row, err = r.tsv.Read()
+		if err != nil {
+			return err
+		}
+	}
+
 	for i, s := range row {
 		reader := strings.NewReader(s)
 		v, err := r.parsers[i].Parse(reader)
diff --git a/tsv_reader.go b/tsv_reader.go
new file mode 100644
index 0000000..326f4a0
--- /dev/null
+++ b/tsv_reader.go
@@ -0,0 +1,47 @@
+package clickhouse
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+type tsvReader struct {
+	r         *bufio.Reader
+	rawBuffer []byte
+}
+
+func newReader(r io.Reader) *tsvReader {
+	return &tsvReader{
+		r: bufio.NewReader(r),
+	}
+}
+
+func (r *tsvReader) readLine() ([]byte, error) {
+	line, err := r.r.ReadSlice('\n')
+	if err == bufio.ErrBufferFull {
+		r.rawBuffer = append(r.rawBuffer[:0], line...)
+		for err == bufio.ErrBufferFull {
+			line, err = r.r.ReadSlice('\n')
+			r.rawBuffer = append(r.rawBuffer, line...)
+		}
+		line = r.rawBuffer
+	}
+	// drop trailing \n
+	if n := len(line); n > 0 && line[n-1] == '\n' {
+		line = line[:n-1]
+	}
+	// drop trailing \r
+	if n := len(line); n > 0 && line[n-1] == '\r' {
+		line = line[:n-1]
+	}
+	return line, err
+}
+
+func (r *tsvReader) Read() (record []string, err error) {
+	line, errRead := r.readLine()
+	if errRead != nil && errRead != io.EOF {
+		return nil, errRead
+	}
+	return strings.Split(string(line), "\t"), errRead
+}


Test patch:
diff --git a/rows_test.go b/rows_test.go
index e17848f..df7acec 100644
--- a/rows_test.go
+++ b/rows_test.go
@@ -54,7 +54,7 @@ func TestTextRows(t *testing.T) {
 }
 
 func TestTextRowsQuoted(t *testing.T) {
-	buf := bytes.NewReader([]byte("text\nArray(String)\n['Quote: \"here\"']"))
+	buf := bytes.NewReader([]byte("text\nArray(String)\n['Quote: \"here\"']\n"))
 	rows, err := newTextRows(&conn{}, &bufReadCloser{buf}, time.Local, false)
 	if !assert.NoError(t, err) {
 		return
@@ -69,7 +69,7 @@ func TestTextRowsQuoted(t *testing.T) {
 }
 
 func TestTextRowsNewLine(t *testing.T) {
-	buf := bytes.NewReader([]byte("text\nString\nHello\\nThere"))
+	buf := bytes.NewReader([]byte("text\nString\nHello\\nThere\n"))
 	rows, err := newTextRows(&conn{}, &bufReadCloser{buf}, time.Local, false)
 	if !assert.NoError(t, err) {
 		return
@@ -82,3 +82,52 @@ func TestTextRowsNewLine(t *testing.T) {
 	}
 	assert.Equal(t, []driver.Value{"Hello\nThere"}, dest)
 }
+
+func TestTextRowsEmpty(t *testing.T) {
+	buf := bytes.NewReader([]byte("text\nString\n\n"))
+	rows, err := newTextRows(&conn{}, &bufReadCloser{buf}, time.Local, false)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, []string{"text"}, rows.Columns())
+	assert.Equal(t, []string{"String"}, rows.types)
+	dest := make([]driver.Value, 1)
+	if !assert.NoError(t, rows.Next(dest)) {
+		return
+	}
+	assert.Equal(t, []driver.Value{""}, dest)
+}
+
+func TestTextRowsWithStartsDoubleQuotes(t *testing.T) {
+	buf := bytes.NewReader([]byte("text\nString\n\"\n"))
+	rows, err := newTextRows(&conn{}, &bufReadCloser{buf}, time.Local, false)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, []string{"text"}, rows.Columns())
+	assert.Equal(t, []string{"String"}, rows.types)
+	dest := make([]driver.Value, 1)
+	if !assert.NoError(t, rows.Next(dest)) {
+		return
+	}
+	assert.Equal(t, []driver.Value{`"`}, dest)
+}
+
+func TestTextRowsWithEmptyLine(t *testing.T) {
+	buf := bytes.NewReader([]byte("count\ttext\nInt32\tString\n1\t\n\n2\t\n"))
+	rows, err := newTextRows(&conn{}, &bufReadCloser{buf}, time.Local, false)
+	if !assert.NoError(t, err) {
+		return
+	}
+	assert.Equal(t, []string{"count", "text"}, rows.Columns())
+	assert.Equal(t, []string{"Int32", "String"}, rows.types)
+	dest := make([]driver.Value, 2)
+	if !assert.NoError(t, rows.Next(dest)) {
+		return
+	}
+	assert.Equal(t, []driver.Value{int32(1), ""}, dest)
+	if !assert.NoError(t, rows.Next(dest)) {
+		return
+	}
+	assert.Equal(t, []driver.Value{int32(2), ""}, dest)
+}