Repository: mailru/go-clickhouse
Pull request #109 (2020-12-17 23:06:30)

Support Array(Nullable(TypeName)) Parse() 
I've noticed some time ago that Array(Nullable(TypeName)) elements can not be read from ClickHouse database properly.
So there's a solution for some data types.
P.S. + some tests of course
P.S.2. + fixed nil array element Parse() panic (dataparser.go:320)

dataparser.go: Array(Nullable(TypeName)) Parse() support where TypeName:
- UInt8-64
- Int8-64
- Float32-64
- String (also with escape seqs)
- Date (also with timezone)
- DateTime (also with timezone)
- Decimal
- Enum
- Uuid
- LowCardinality


Patch:
diff --git a/dataparser.go b/dataparser.go
index ed3565c..547ddd8 100644
--- a/dataparser.go
+++ b/dataparser.go
@@ -51,13 +51,108 @@ func (p *nullableParser) Parse(s io.RuneScanner) (driver.Value, error) {
 	// Clickhouse returns `\N` string for `null` in tsv format.
 	// For checking this value we need to check first two runes in `io.RuneScanner`, but we can't reset `io.RuneScanner` after it.
 	// Copy io.RuneScanner to `bytes.Buffer` and use it twice (1st for casting to string and checking to null, 2nd for passing to original parser)
-	d := readRaw(s)
+	var dB *bytes.Buffer
 
-	if bytes.Equal(d.Bytes(), []byte(`\N`)) {
+	dType := p.DataParser.Type()
+
+	switch dType {
+	case reflectTypeInt8, reflectTypeInt16, reflectTypeInt32, reflectTypeInt64,
+		reflectTypeUInt8, reflectTypeUInt16, reflectTypeUInt32, reflectTypeUInt64,
+		reflectTypeFloat32, reflectTypeFloat64:
+		d, err := readNumber(s)
+		if err != nil {
+			return nil, fmt.Errorf("error: %v", err)
+		}
+
+		dB = bytes.NewBufferString(d)
+	case reflectTypeString:
+		runes := ""
+		iter := 0
+
+		isNotString := false
+		for {
+			r, _, err := s.ReadRune()
+			if err != nil {
+				return nil, fmt.Errorf("error: %v", err)
+			}
+
+			if r != '\'' && iter == 0 {
+				s.UnreadRune()
+				d := readRaw(s)
+				dB = d
+				isNotString = true
+				break
+			}
+
+			isEscaped := false
+			if r == '\\' {
+				escaped, err := readEscaped(s)
+				if err != nil {
+					return "", fmt.Errorf("incorrect escaping in string: %v", err)
+				}
+
+				isEscaped = true
+				r = escaped
+				if r == '\'' {
+					runes += string('\\')
+				}
+			}
+
+			runes += string(r)
+
+			if r == '\'' && iter != 0 && !isEscaped {
+				break
+			}
+			iter++
+		}
+
+		if bytes.Equal([]byte(runes), []byte(`'N'`)) {
+			return nil, nil
+		}
+
+		if !isNotString {
+			dB = bytes.NewBufferString(runes)
+		}
+	case reflectTypeTime:
+		runes := ""
+
+		iter := 0
+		for {
+			r, _, err := s.ReadRune()
+			if err != nil {
+				return nil, nil
+			}
+
+			runes += string(r)
+
+			if r == '\'' && iter != 0 {
+				break
+			}
+			iter++
+		}
+
+		if runes == "0000-00-00" || runes == "0000-00-00 00:00:00" {
+			return time.Time{}, nil
+		}
+
+		if bytes.Equal([]byte(runes), []byte(`'\N'`)) {
+			return nil, nil
+		}
+
+		dB = bytes.NewBufferString(runes)
+	case reflectTypeEmptyStruct:
+		d := readRaw(s)
+		dB = d
+	default:
+		d := readRaw(s)
+		dB = d
+	}
+
+	if bytes.Equal(dB.Bytes(), []byte(`\N`)) {
 		return nil, nil
 	}
 
-	return p.DataParser.Parse(d)
+	return p.DataParser.Parse(dB)
 }
 
 func readNumber(s io.RuneScanner) (string, error) {
@@ -231,7 +326,14 @@ func (p *arrayParser) Parse(s io.RuneScanner) (driver.Value, error) {
 			return nil, fmt.Errorf("failed to parse array element: %v", err)
 		}
 
-		slice = reflect.Append(slice, reflect.ValueOf(v))
+		if v == nil {
+			if reflect.TypeOf(p.arg) != reflect.TypeOf(&nullableParser{}) {
+				//need check if v is nil: panic otherwise
+				return nil, fmt.Errorf("unexpected nil element")
+			}
+		} else {
+			slice = reflect.Append(slice, reflect.ValueOf(v))
+		}
 
 		r = read(s)
 		if r != ',' {


Test patch:
diff --git a/dataparser_test.go b/dataparser_test.go
index 5c0ce86..26585f4 100644
--- a/dataparser_test.go
+++ b/dataparser_test.go
@@ -369,3 +369,394 @@ func TestParseData(t *testing.T) {
 		})
 	}
 }
+
+func TestParseDataNewNullableArray(t *testing.T) {
+	type testCase struct {
+		name          string
+		inputtype     string
+		inputopt      *DataParserOptions
+		inputdata     string
+		output        interface{}
+		failParseDesc bool
+		failNewParser bool
+		failParseData bool
+	}
+
+	losAngeles, err := time.LoadLocation("America/Los_Angeles")
+	if err != nil {
+		t.Fatalf("failed to load time zone America/Los_Angeles: %v", err)
+	}
+	moscow, err := time.LoadLocation("Europe/Moscow")
+	if err != nil {
+		t.Fatalf("failed to load time zone Europe/Moscow: %v", err)
+	}
+
+	t.Log(losAngeles, moscow)
+
+	testCases := []*testCase{
+		{
+			name:      "array of nullable null String",
+			inputtype: "Array(Nullable(String))",
+			inputdata: `['ss','\N','dd','ff']`,
+			output:    []string{"ss", "dd", "ff"},
+		},
+		{
+			name:      "array(nullable(uuid))",
+			inputtype: "Array(Nullable(UUID))",
+			inputdata: `['c79a9747-7cef-4b11-8177-380f7ed462a4','\N','00000000-0000-0000-0000-000000000000']`,
+			output:    []string{"c79a9747-7cef-4b11-8177-380f7ed462a4", "00000000-0000-0000-0000-000000000000"},
+		},
+		{
+			name:      "array of nullable null UInt64",
+			inputtype: "Array(Nullable(UInt64))",
+			inputdata: `[1,\N,5,9]`,
+			output:    []uint64{1, 5, 9},
+		},
+		{
+			name:      "array of nullable null UInt16",
+			inputtype: "Array(Nullable(UInt16))",
+			inputdata: `[1,2,\N]`,
+			output:    []uint16{1, 2},
+		},
+		{
+			name:          "malformed array(nullable(date))",
+			inputtype:     "Array(Nullable(Date))",
+			inputdata:     `['a000-00-00 00:00:00', '\N']`,
+			output:        []time.Time{{}},
+			failParseData: true,
+		},
+		{
+			name:      "array of nullable null Float64",
+			inputtype: "Array(Nullable(Float64))",
+			inputdata: `[1.9,\N,5.3,9.9]`,
+			output:    []float64{1.9, 5.3, 9.9},
+		},
+		{
+			name:      "array(nullable(datetime)), without options and argument",
+			inputtype: "Array(Nullable(DateTime))",
+			inputdata: `['2018-01-02 12:34:56','\N','0000-00-00 00:00:00']`,
+			output: []time.Time{
+				time.Date(2018, 1, 2, 12, 34, 56, 0, time.UTC),
+				{},
+			},
+		},
+		{
+			name:      "nullable null string",
+			inputtype: "Nullable(String)",
+			inputdata: `\N`,
+			output:    nil,
+		},
+		{
+			name:      "nullable null UInt64",
+			inputtype: "Nullable(UInt64)",
+			inputdata: `\N`,
+			output:    nil,
+		},
+		{
+			name:      "nullable null decimal",
+			inputtype: "Nullable(Decimal)",
+			inputdata: `\N`,
+			output:    nil,
+		},
+		{
+			name:      "nullable null datetime",
+			inputtype: "Nullable(DateTime)",
+			inputdata: `\N`,
+			output:    nil,
+		},
+		{
+			name:      "nullable null enum",
+			inputtype: "Nullable(Enum8('hello' = 1, 'world' = 2))",
+			inputdata: `\N`,
+			output:    nil,
+		},
+		{
+			name:      "nullable null uuid",
+			inputtype: "Nullable(UUID)",
+			inputdata: `\N`,
+			output:    nil,
+		},
+		{
+			name:      "nullable null Date",
+			inputtype: "Nullable(Date)",
+			inputdata: `\N`,
+			output:    nil,
+		},
+		{
+			name:      "nullable(decimal)",
+			inputtype: "Nullable(Decimal(9,4))",
+			inputdata: "123",
+			output:    "123",
+		},
+		{
+			name:      "array(nullable(decimal))",
+			inputtype: "Array(Nullable(Decimal(9,4)))",
+			inputdata: "['123','555.6']",
+			output:    []string{"123", "555.6"},
+		},
+		{
+			name:      "nullable(enum)",
+			inputtype: "Nullable(Enum8('hello' = 1, 'world' = 2))",
+			inputdata: "hello",
+			output:    "hello",
+		},
+		{
+			name:      "array(nullable(enum))",
+			inputtype: "Array(Nullable(Enum8('hello' = 1, 'world' = 2)))",
+			inputdata: "['hello','hi']",
+			output:    []string{"hello", "hi"},
+		},
+		{
+			name:      "nullable(uuid)",
+			inputtype: "Nullable(UUID)",
+			inputdata: "c79a9747-7cef-4b11-8177-380f7ed462a4",
+			output:    "c79a9747-7cef-4b11-8177-380f7ed462a4",
+		},
+		{
+			name:      "array(nullable(uuid))",
+			inputtype: "Array(Nullable(UUID))",
+			inputdata: "['c79a9747-7cef-4b11-8177-380f7ed462a4','00000000-0000-0000-0000-000000000000']",
+			output:    []string{"c79a9747-7cef-4b11-8177-380f7ed462a4", "00000000-0000-0000-0000-000000000000"},
+		},
+		{
+			name:      "nullable low cardinality string",
+			inputtype: "Nullable(LowCardinality(String))",
+			inputdata: "hello",
+			output:    "hello",
+		},
+		{
+			name:      "array of nullable low cardinality string",
+			inputtype: "Array(Nullable(LowCardinality(String)))",
+			inputdata: "['hello','hi']",
+			output:    []string{"hello", "hi"},
+		},
+		{
+			name:      "array of nullable low cardinality UInt64",
+			inputtype: "Array(Nullable(LowCardinality(UInt64)))",
+			inputdata: "[123,555]",
+			output:    []uint64{123, 555},
+		},
+		//////////////////////////////////////////////////////
+		{
+			name:      "array(nullable(datetime)), without options and argument",
+			inputtype: "Array(Nullable(DateTime))",
+			inputdata: "['2018-01-02 12:34:56','0000-00-00 00:00:00']",
+			output: []time.Time{
+				time.Date(2018, 1, 2, 12, 34, 56, 0, time.UTC),
+				{},
+			},
+		},
+		{
+			name:      "array(nullable(datetime)), with argument",
+			inputtype: "Array(Nullable(DateTime('America/Los_Angeles')))",
+			inputdata: "['2018-01-02 12:34:56','0000-00-00 00:00:00']",
+			output: []time.Time{
+				time.Date(2018, 1, 2, 12, 34, 56, 0, losAngeles),
+				{},
+			},
+		},
+		{
+			name:      "array(nullable(datetime)) with argument, but location nil",
+			inputtype: "Array(Nullable(DateTime('America/Los_Angeles')))",
+			inputdata: "['2018-01-02 12:34:56','0000-00-00 00:00:00']",
+			inputopt: &DataParserOptions{
+				Location: nil,
+			},
+			output: []time.Time{
+				time.Date(2018, 1, 2, 12, 34, 56, 0, losAngeles),
+				{},
+			},
+		},
+		{
+			name:      "array(nullable(datetime)) without argument, but use location",
+			inputtype: "Array(Nullable(DateTime))",
+			inputdata: "['2018-01-02 12:34:56','0000-00-00 00:00:00']",
+			inputopt: &DataParserOptions{
+				Location: moscow,
+			},
+			output: []time.Time{
+				time.Date(2018, 1, 2, 12, 34, 56, 0, moscow),
+				{},
+			},
+		},
+		{
+			name:      "array(nullable(datetime)) with argument and location, ingnore argument",
+			inputtype: "Array(Nullable(DateTime('America/Los_Angeles')))",
+			inputdata: "['2018-01-02 12:34:56','0000-00-00 00:00:00']",
+			inputopt: &DataParserOptions{
+				Location: moscow,
+			},
+			output: []time.Time{
+				time.Date(2018, 1, 2, 12, 34, 56, 0, moscow),
+				{},
+			},
+		},
+		{
+			name:      "array(nullable(datetime)) with argument and location, prefer argument",
+			inputtype: "Array(Nullable(DateTime('America/Los_Angeles')))",
+			inputdata: "['2018-01-02 12:34:56','0000-00-00 00:00:00']",
+			inputopt: &DataParserOptions{
+				Location:      moscow,
+				UseDBLocation: true,
+			},
+			output: []time.Time{
+				time.Date(2018, 1, 2, 12, 34, 56, 0, losAngeles),
+				{},
+			},
+		},
+		{
+			name:          "array(nullable(datetime)) in nowhere",
+			inputtype:     "Array(Nullable(DateTime('Nowhere')))",
+			inputdata:     "['2018-01-02 12:34:56','0000-00-00 00:00:00']",
+			failNewParser: true,
+		},
+		{
+			name:      "zero array(nullable(datetime))",
+			inputtype: "Array(Nullable(DateTime))",
+			inputdata: "['0000-00-00 00:00:00','0000-00-00 00:00:00']",
+			output: []time.Time{
+				{},
+				{},
+			},
+		},
+		{
+			name:      "short array(nullable(datetime))",
+			inputtype: "Array(Nullable(DateTime))",
+			inputdata: "['000-00-00 00:00:00','000-00-00 00:00:00']",
+			output: []time.Time{
+				{},
+				{},
+			},
+			failParseData: true,
+		},
+		{
+			name:      "malformed array(nullable(datetime))",
+			inputtype: "Array(Nullable(DateTime))",
+			inputdata: "['0000-00-00 00:00:00','a000-00-00 00:00:00']",
+			output: []time.Time{
+				{},
+				{},
+			},
+			failParseData: true,
+		},
+		////////////////////////////////////////////////////
+		{
+			name:      "array of dates",
+			inputtype: "Array(Date)",
+			inputdata: "['2018-01-02','0000-00-00']",
+			output: []time.Time{
+				time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC),
+				{},
+			},
+		},
+		{
+			name:      "array of nullable dates",
+			inputtype: "Array(Nullable(Date))",
+			inputdata: "['2018-01-02','0000-00-00']",
+			output: []time.Time{
+				time.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC),
+				{},
+			},
+		},
+		{
+			name:          "malformed array(nullable(date))",
+			inputtype:     "Array(Nullable(Date))",
+			inputdata:     "['a000-00-00 00:00:00']",
+			output:        []time.Time{{}},
+			failParseData: true,
+		},
+		{
+			name:      "array of nullable dates with special timezone",
+			inputtype: "Array(Nullable(Date))",
+			inputdata: "['2019-06-29','0000-00-00']",
+			inputopt: &DataParserOptions{
+				Location: losAngeles,
+			},
+			output: []time.Time{
+				time.Date(2019, 6, 29, 0, 0, 0, 0, losAngeles),
+				{},
+			},
+		},
+		//////////////////////////////////////////////////
+		{
+			name:      "array of ints",
+			inputtype: "Array(UInt64)",
+			inputdata: "[1,2,3]",
+			output:    []uint64{1, 2, 3},
+		},
+		{
+			name:      "array of nullable ints",
+			inputtype: "Array(Nullable(UInt64))",
+			inputdata: "[1,2,3]",
+			output:    []uint64{1, 2, 3},
+		},
+		{
+			name:      "empty array of nullable ints",
+			inputtype: "Array(Nullable(UInt64))",
+			inputdata: "[]",
+			output:    []uint64{},
+		},
+		{
+			name:          "bad array of nullable ints",
+			inputtype:     "Array(Nullable(UInt64))",
+			inputdata:     "[1,,]",
+			output:        []uint64{},
+			failParseData: true,
+		},
+		//////////////////////////////////////////////////
+		{
+			name:      "array of strings",
+			inputtype: "Array(String)",
+			inputdata: "['133','2']",
+			output:    []string{"133", "2"},
+		},
+		{
+			name:      "array of nullable strings",
+			inputtype: "Array(Nullable(String))",
+			inputdata: `['a\taa\',','255']`,
+			output:    []string{"a\taa',", "255"},
+		},
+		{
+			name:          "array of nullable strings",
+			inputtype:     "Array(Nullable(String))",
+			inputdata:     "['aaa,]",
+			output:        nil,
+			failParseData: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(tt *testing.T) {
+			desc, err := ParseTypeDesc(tc.inputtype)
+			if tc.failParseDesc {
+				assert.Error(tt, err)
+				return
+			}
+			if !assert.NoError(tt, err) {
+				return
+			}
+			t.Log(desc.Name, desc.Args)
+
+			parser, err := newDataParser(desc, false, tc.inputopt)
+			if tc.failNewParser {
+				assert.Error(tt, err)
+				return
+			}
+			if !assert.NoError(tt, err) {
+				return
+			}
+
+			t.Log(parser.Type(), tc.inputdata)
+			output, err := parser.Parse(strings.NewReader(tc.inputdata))
+			if tc.failParseData {
+				assert.Error(tt, err)
+				return
+			}
+			if !assert.NoError(tt, err) {
+				return
+			}
+
+			assert.Equal(tt, tc.output, output)
+		})
+	}
+}