Repository: genjidb/genji
Pull request #423 (2021-07-23 16:32:21)

Parse blob literal
We now have a literal representation for BLOBs. Any string literal starting
with '\x' is parsed as an hex encoded blob. This mimics PostgreSQL's behavior.

```sql
SELECT '\xAAFF';
```



Patch:
diff --git a/Makefile b/Makefile
index 631de637..7e8b7d80 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ build: $(NAME)
 $(NAME):
 	cd ./cmd/$@ && go install
 
-gen: $(NAME)
+gen:
 	go generate ./...
 
 test:
diff --git a/document/cast.go b/document/cast.go
index df7bf167..2c6271e4 100644
--- a/document/cast.go
+++ b/document/cast.go
@@ -121,8 +121,11 @@ func CastAsDouble(v types.Value) (types.Value, error) {
 // CastAsText returns a JSON representation of v.
 // If the representation is a string, it gets unquoted.
 func CastAsText(v types.Value) (types.Value, error) {
-	if v.Type() == types.TextValue {
+	switch v.Type() {
+	case types.TextValue:
 		return v, nil
+	case types.BlobValue:
+		return types.NewTextValue(base64.StdEncoding.EncodeToString(v.V().([]byte))), nil
 	}
 
 	d, err := ValueToJSON(v)
@@ -132,13 +135,6 @@ func CastAsText(v types.Value) (types.Value, error) {
 
 	s := string(d)
 
-	if v.Type() == types.BlobValue {
-		s, err = strconv.Unquote(s)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	return types.NewTextValue(s), nil
 }
 
@@ -151,9 +147,11 @@ func CastAsBlob(v types.Value) (types.Value, error) {
 	}
 
 	if v.Type() == types.TextValue {
-		b, err := base64.StdEncoding.DecodeString(v.V().(string))
+		// if the string starts with \x, read it as hex
+		s := v.V().(string)
+		b, err := base64.StdEncoding.DecodeString(s)
 		if err != nil {
-			return nil, stringutil.Errorf(`cannot cast %q as blob: %w`, v.V(), err)
+			return nil, err
 		}
 
 		return types.NewBlobValue(b), nil
diff --git a/document/json.go b/document/json.go
index 88de776c..3c1eaf26 100644
--- a/document/json.go
+++ b/document/json.go
@@ -175,8 +175,6 @@ func ValueToString(v types.Value) string {
 		return "NULL"
 	case types.TextValue:
 		return strconv.Quote(v.V().(string))
-	case types.BlobValue:
-		return stringutil.Sprintf("%v", v.V())
 	}
 
 	d, _ := ValueToJSON(v)
diff --git a/internal/database/constraint.go b/internal/database/constraint.go
index 9378d6b2..619bbe88 100644
--- a/internal/database/constraint.go
+++ b/internal/database/constraint.go
@@ -361,12 +361,7 @@ type ConversionFunc func(v types.Value, path document.Path, targetType types.Val
 
 // CastConversion is a ConversionFunc that casts the value to the target type.
 func CastConversion(v types.Value, path document.Path, targetType types.ValueType) (types.Value, error) {
-	newV, err := document.CastAs(v, targetType)
-	if err != nil {
-		return v, stringutil.Errorf("field %q must be of type %q, got %q", path, targetType, v.Type())
-	}
-
-	return newV, nil
+	return document.CastAs(v, targetType)
 }
 
 // ConvertValueAtPath converts the value using the field constraints that are applicable
diff --git a/internal/expr/functions/builtins.go b/internal/expr/functions/builtins.go
index 6f34024e..785e2a3e 100644
--- a/internal/expr/functions/builtins.go
+++ b/internal/expr/functions/builtins.go
@@ -93,51 +93,6 @@ func (k *PK) String() string {
 	return "pk()"
 }
 
-// Cast represents the CAST expression.
-type Cast struct {
-	Expr   expr.Expr
-	CastAs types.ValueType
-}
-
-// Eval returns the primary key of the current document.
-func (c Cast) Eval(env *environment.Environment) (types.Value, error) {
-	v, err := c.Expr.Eval(env)
-	if err != nil {
-		return v, err
-	}
-
-	return document.CastAs(v, c.CastAs)
-}
-
-// IsEqual compares this expression with the other expression and returns
-// true if they are equal.
-func (c Cast) IsEqual(other expr.Expr) bool {
-	if other == nil {
-		return false
-	}
-
-	o, ok := other.(Cast)
-	if !ok {
-		return false
-	}
-
-	if c.CastAs != o.CastAs {
-		return false
-	}
-
-	if c.Expr != nil {
-		return expr.Equal(c.Expr, o.Expr)
-	}
-
-	return o.Expr != nil
-}
-
-func (c Cast) Params() []expr.Expr { return []expr.Expr{c.Expr} }
-
-func (c Cast) String() string {
-	return stringutil.Sprintf("CAST(%v AS %v)", c.Expr, c.CastAs)
-}
-
 var _ expr.AggregatorBuilder = (*Count)(nil)
 
 // Count is the COUNT aggregator function. It counts the number of documents
diff --git a/internal/expr/literal.go b/internal/expr/literal.go
index 38f979e5..f3ed6c09 100644
--- a/internal/expr/literal.go
+++ b/internal/expr/literal.go
@@ -77,6 +77,9 @@ func (l LiteralExprList) String() string {
 // Eval evaluates all the expressions and returns a literalValueList. It implements the Expr interface.
 func (l LiteralExprList) Eval(env *environment.Environment) (types.Value, error) {
 	var err error
+	if len(l) == 0 {
+		return types.NewArrayValue(document.NewValueBuffer()), nil
+	}
 	values := make([]types.Value, len(l))
 	for i, e := range l {
 		values[i], err = e.Eval(env)
diff --git a/internal/expr/operator.go b/internal/expr/operator.go
index aa3e33c3..35f1f25a 100644
--- a/internal/expr/operator.go
+++ b/internal/expr/operator.go
@@ -3,6 +3,7 @@ package expr
 import (
 	"errors"
 
+	"github.com/genjidb/genji/document"
 	"github.com/genjidb/genji/internal/environment"
 	"github.com/genjidb/genji/internal/sql/scanner"
 	"github.com/genjidb/genji/internal/stringutil"
@@ -119,3 +120,48 @@ func (op *ConcatOperator) Eval(env *environment.Environment) (types.Value, error
 		return types.NewTextValue(a.V().(string) + b.V().(string)), nil
 	})
 }
+
+// Cast represents the CAST expression.
+type Cast struct {
+	Expr   Expr
+	CastAs types.ValueType
+}
+
+// Eval returns the primary key of the current document.
+func (c Cast) Eval(env *environment.Environment) (types.Value, error) {
+	v, err := c.Expr.Eval(env)
+	if err != nil {
+		return v, err
+	}
+
+	return document.CastAs(v, c.CastAs)
+}
+
+// IsEqual compares this expression with the other expression and returns
+// true if they are equal.
+func (c Cast) IsEqual(other Expr) bool {
+	if other == nil {
+		return false
+	}
+
+	o, ok := other.(Cast)
+	if !ok {
+		return false
+	}
+
+	if c.CastAs != o.CastAs {
+		return false
+	}
+
+	if c.Expr != nil {
+		return Equal(c.Expr, o.Expr)
+	}
+
+	return o.Expr != nil
+}
+
+func (c Cast) Params() []Expr { return []Expr{c.Expr} }
+
+func (c Cast) String() string {
+	return stringutil.Sprintf("CAST(%v AS %v)", c.Expr, c.CastAs)
+}
diff --git a/internal/sql/parser/expr.go b/internal/sql/parser/expr.go
index 7d627bb5..761bda57 100644
--- a/internal/sql/parser/expr.go
+++ b/internal/sql/parser/expr.go
@@ -1,7 +1,9 @@
 package parser
 
 import (
+	"encoding/hex"
 	"strconv"
+	"strings"
 
 	"github.com/genjidb/genji/document"
 	"github.com/genjidb/genji/internal/environment"
@@ -241,6 +243,17 @@ func (p *Parser) parseUnaryExpr(allowed ...scanner.Token) (expr.Expr, error) {
 		p.orderedParams++
 		return expr.PositionalParam(p.orderedParams), nil
 	case scanner.STRING:
+		if strings.HasPrefix(lit, `\x`) {
+			blob, err := hex.DecodeString(lit[2:])
+			if err != nil {
+				if bt, ok := err.(hex.InvalidByteError); ok {
+					return nil, stringutil.Errorf("invalid hexadecimal digit: %c", bt)
+				}
+
+				return nil, err
+			}
+			return expr.LiteralValue{Value: types.NewBlobValue(blob)}, nil
+		}
 		return expr.LiteralValue{Value: types.NewTextValue(lit)}, nil
 	case scanner.NUMBER:
 		v, err := strconv.ParseFloat(lit, 64)
@@ -698,7 +711,7 @@ func (p *Parser) parseCastExpression() (expr.Expr, error) {
 		return nil, err
 	}
 
-	return functions.Cast{Expr: e, CastAs: tp}, nil
+	return expr.Cast{Expr: e, CastAs: tp}, nil
 }
 
 // tokenIsAllowed is a helper function that determines if a token is allowed.
diff --git a/internal/sql/scanner/scanner.go b/internal/sql/scanner/scanner.go
index 61e309dc..1b0e1390 100644
--- a/internal/sql/scanner/scanner.go
+++ b/internal/sql/scanner/scanner.go
@@ -575,7 +575,7 @@ func scanString(r io.RuneReader) (string, error) {
 	}
 
 	var buf bytes.Buffer
-	for {
+	for i := 0; ; i++ {
 		ch0, _, err := r.ReadRune()
 		if ch0 == ending {
 			return buf.String(), nil
@@ -595,6 +595,8 @@ func scanString(r io.RuneReader) (string, error) {
 				_, _ = buf.WriteRune('`')
 			} else if ch1 == '\'' {
 				_, _ = buf.WriteRune('\'')
+			} else if ch1 == 'x' && i == 0 {
+				_, _ = buf.WriteString(`\x`)
 			} else {
 				return string(ch0) + string(ch1), errBadEscape
 			}


Test patch:
diff --git a/document/cast_test.go b/document/cast_test.go
index 587197b0..9785bbc4 100644
--- a/document/cast_test.go
+++ b/document/cast_test.go
@@ -17,7 +17,7 @@ func TestCastAs(t *testing.T) {
 	integerV := types.NewIntegerValue(10)
 	doubleV := types.NewDoubleValue(10.5)
 	textV := types.NewTextValue("foo")
-	blobV := types.NewBlobValue([]byte("abc"))
+	blobV := types.NewBlobValue([]byte("asdine"))
 	arrayV := types.NewArrayValue(NewValueBuffer().
 		Append(types.NewTextValue("bar")).
 		Append(integerV))
@@ -26,8 +26,12 @@ func TestCastAs(t *testing.T) {
 		Add("b", textV))
 
 	check := func(t *testing.T, targetType types.ValueType, tests []test) {
+		t.Helper()
+
 		for _, test := range tests {
 			t.Run(ValueToString(test.v), func(t *testing.T) {
+				t.Helper()
+
 				got, err := CastAs(test.v, targetType)
 				if test.fails {
 					require.Error(t, err)
@@ -89,7 +93,7 @@ func TestCastAs(t *testing.T) {
 			{integerV, types.NewTextValue("10"), false},
 			{doubleV, types.NewTextValue("10.5"), false},
 			{textV, textV, false},
-			{blobV, types.NewTextValue("YWJj"), false},
+			{blobV, types.NewTextValue(`YXNkaW5l`), false},
 			{arrayV, types.NewTextValue(`["bar", 10]`), false},
 			{docV,
 				types.NewTextValue(`{"a": 10, "b": "foo"}`),
@@ -102,8 +106,8 @@ func TestCastAs(t *testing.T) {
 			{boolV, nil, true},
 			{integerV, nil, true},
 			{doubleV, nil, true},
-			{types.NewTextValue("YWJj"), blobV, false},
-			{types.NewTextValue("   dww  "), nil, true},
+			{types.NewTextValue("YXNkaW5l"), types.NewBlobValue([]byte{0x61, 0x73, 0x64, 0x69, 0x6e, 0x65}), false},
+			{types.NewTextValue("not base64"), nil, true},
 			{blobV, blobV, false},
 			{arrayV, nil, true},
 			{docV, nil, true},
diff --git a/document/json_test.go b/document/json_test.go
index 6acb0d73..f9bce485 100644
--- a/document/json_test.go
+++ b/document/json_test.go
@@ -15,8 +15,8 @@ func TestValueString(t *testing.T) {
 		expected string
 	}{
 		{"null", types.NewNullValue(), "NULL"},
-		{"bytes", types.NewBlobValue([]byte("bar")), "[98 97 114]"},
-		{"string", types.NewTextValue("bar"), "\"bar\""},
+		{"blob", types.NewBlobValue([]byte("bar")), `"YmFy"`},
+		{"string", types.NewTextValue("bar"), `"bar"`},
 		{"bool", types.NewBoolValue(true), "true"},
 		{"int", types.NewIntegerValue(10), "10"},
 		{"double", types.NewDoubleValue(10.1), "10.1"},
diff --git a/internal/expr/operator_test.go b/internal/expr/operator_test.go
index 38a1e34c..b3adf69b 100644
--- a/internal/expr/operator_test.go
+++ b/internal/expr/operator_test.go
@@ -1,6 +1,7 @@
 package expr_test
 
 import (
+	"path/filepath"
 	"testing"
 
 	"github.com/genjidb/genji/internal/testutil"
@@ -25,3 +26,7 @@ func TestConcatExpr(t *testing.T) {
 		})
 	}
 }
+
+func TestCast(t *testing.T) {
+	testutil.ExprRunner(t, filepath.Join("testdata", "cast.sql"))
+}
diff --git a/internal/expr/testdata/arithmetic.sql b/internal/expr/testdata/arithmetic.sql
index eb5200ec..aa8d1df6 100644
--- a/internal/expr/testdata/arithmetic.sql
+++ b/internal/expr/testdata/arithmetic.sql
@@ -75,3 +75,41 @@ NULL
 
 ! 1 ^ a
 'field not found'
+
+-- test: division
+> 1 / 2
+0
+
+> 1 + 1 * 2 / 4
+1
+
+-- test: arithmetic with different types
+> 1 + 1.5
+2.5
+
+> 1 + '2'
+NULL
+
+> 1 + true
+NULL
+
+> 1 + [1]
+NULL
+
+> 1 + {a: 1}
+NULL
+
+> [1] + [1]
+NULL
+
+> {a: 1} + {a: 1}
+NULL
+
+> 4.5 + 4.5
+9.0
+
+> 1000000000 * 1000000000
+1000000000000000000
+
+> 1000000000000000000 * 1000000000000000000 * 1000000000000000000
+1000000000000000000000000000000000000000000000000000000
diff --git a/internal/expr/testdata/cast.sql b/internal/expr/testdata/cast.sql
new file mode 100644
index 00000000..6e869112
--- /dev/null
+++ b/internal/expr/testdata/cast.sql
@@ -0,0 +1,169 @@
+-- test: source(INT)
+> CAST (1 AS INTEGER)
+1
+
+> CAST (1 AS BOOL)
+true
+
+> CAST (1 AS DOUBLE)
+1.0
+
+> CAST (1 AS TEXT)
+'1'
+
+! CAST (1 AS BLOB)
+'cannot cast integer as blob'
+
+! CAST (1 AS ARRAY)
+'cannot cast integer as array'
+
+! CAST (1 AS DOCUMENT)
+'cannot cast integer as document'
+
+-- test: source(DOUBLE)
+> CAST (1.1 AS DOUBLE)
+1.1
+
+> CAST (1.1 AS INTEGER)
+1
+
+! CAST (1.1 AS BOOL)
+'cannot cast double as bool'
+
+> CAST (1.1 AS TEXT)
+'1.1'
+
+! CAST (1.1 AS BLOB)
+'cannot cast double as blob'
+
+! CAST (1.1 AS ARRAY)
+'cannot cast double as array'
+
+! CAST (1.1 AS DOCUMENT)
+'cannot cast double as document'
+
+-- test: source(BOOL)
+> CAST (true AS BOOL)
+true
+
+> CAST (true AS INTEGER)
+1
+
+> CAST (false AS INTEGER)
+0
+
+! CAST (true AS DOUBLE)
+'cannot cast bool as double'
+
+> CAST (true AS TEXT)
+'true'
+
+! CAST (true AS BLOB)
+'cannot cast bool as blob'
+
+! CAST (true AS ARRAY)
+'cannot cast bool as array'
+
+! CAST (true AS DOCUMENT)
+'cannot cast bool as document'
+
+-- test: source(TEXT)
+> CAST ('a' AS TEXT)
+'a'
+
+> CAST ('100' AS INTEGER)
+100
+
+> CAST ('100.5' AS INTEGER)
+100
+
+! CAST ('a' AS INTEGER)
+
+> CAST ('3.14' AS DOUBLE)
+3.14
+
+> CAST ('3' AS DOUBLE)
+3.0
+
+! CAST ('10.5wdwd' AS DOUBLE)
+
+> CAST ('true' AS BOOL)
+true
+
+> CAST ('false' AS BOOL)
+false
+
+! CAST ('falSe' AS BOOL)
+
+> CAST ('YXNkaW5l' AS BLOB)
+'\x617364696e65'
+
+> CAST ('[]' AS ARRAY)
+[]
+
+> CAST ('[1, true, [], {" a": 1}]' AS ARRAY)
+[1, true, [], {" a": 1}]
+
+! CAST ('[1, true, [], {" a": 1}' AS ARRAY)
+
+> CAST ('{"a": 1, "b": [1, true, [], {" a": 1}]}' AS DOCUMENT)
+{"a": 1, "b": [1, true, [], {" a": 1}]}
+
+! CAST ('{"a": 1' AS DOCUMENT)
+
+-- test: source(BLOB)
+> CAST ('\xAF' AS BLOB)
+'\xAF'
+
+! CAST ('\xAF' AS INT)
+'cannot cast blob as integer'
+
+! CAST ('\xAF' AS DOUBLE)
+'cannot cast blob as double'
+
+> CAST ('\x617364696e65' AS TEXT)
+'YXNkaW5l'
+
+! CAST ('\xAF' AS ARRAY)
+'cannot cast blob as array'
+
+! CAST ('\xAF' AS DOCUMENT)
+'cannot cast blob as document'
+
+-- test: source(ARRAY)
+> CAST ([1] AS ARRAY)
+[1]
+
+! CAST ([1] AS INTEGER)
+'cannot cast array as integer'
+
+! CAST ([1] AS DOUBLE)
+'cannot cast array as double'
+
+> CAST ([1, true, [], {" a": 1}] AS TEXT)
+'[1, true, [], {" a": 1}]'
+
+! CAST ([1] AS BLOB)
+'cannot cast array as blob'
+
+! CAST ([1] AS DOCUMENT)
+'cannot cast array as document'
+
+-- test: source(DOCUMENT)
+> CAST ({a: 1} AS DOCUMENT)
+{a: 1}
+
+! CAST ({a: 1} AS INTEGER)
+'cannot cast document as integer'
+
+! CAST ({a: 1} AS DOUBLE)
+'cannot cast document as double'
+
+> CAST ({"a": 1, "b": [1, true, [], {" a": 1}]} AS TEXT)
+'{"a": 1, "b": [1, true, [], {" a": 1}]}'
+
+! CAST ({a: 1} AS BLOB)
+'cannot cast document as blob'
+
+! CAST ({a: 1} AS ARRAY)
+'cannot cast document as array'
\ No newline at end of file
diff --git a/internal/sql/parser/expr_test.go b/internal/sql/parser/expr_test.go
index 6ac35c80..60a6604a 100644
--- a/internal/sql/parser/expr_test.go
+++ b/internal/sql/parser/expr_test.go
@@ -41,6 +41,10 @@ func TestParserExpr(t *testing.T) {
 		{"double quoted string", `"10.0"`, testutil.TextValue("10.0"), false},
 		{"single quoted string", "'-10.0'", testutil.TextValue("-10.0"), false},
 
+		// blobs
+		{"blob as hex string", `'\xff'`, testutil.BlobValue([]byte{255}), false},
+		{"invalid blob hex string", `'\xzz'`, nil, true},
+
 		// documents
 		{"empty document", `{}`, &expr.KVPairs{SelfReferenced: true}, false},
 		{"document values", `{a: 1, b: 1.0, c: true, d: 'string', e: "string", f: {foo: 'bar'}, g: h.i.j, k: [1, 2, 3]}`,
@@ -154,7 +158,7 @@ func TestParserExpr(t *testing.T) {
 		{"with NULL", "age > NULL", expr.Gt(testutil.ParsePath(t, "age"), testutil.NullValue()), false},
 
 		// unary operators
-		{"CAST", "CAST(a.b[1][0] AS TEXT)", functions.Cast{Expr: testutil.ParsePath(t, "a.b[1][0]"), CastAs: types.TextValue}, false},
+		{"CAST", "CAST(a.b[1][0] AS TEXT)", expr.Cast{Expr: testutil.ParsePath(t, "a.b[1][0]"), CastAs: types.TextValue}, false},
 		{"NOT", "NOT 10", expr.Not(testutil.IntegerValue(10)), false},
 		{"NOT", "NOT NOT", nil, true},
 		{"NOT", "NOT NOT 10", expr.Not(expr.Not(testutil.IntegerValue(10))), false},
diff --git a/internal/sql/scanner/scanner_test.go b/internal/sql/scanner/scanner_test.go
index 65822ee8..a88428d6 100644
--- a/internal/sql/scanner/scanner_test.go
+++ b/internal/sql/scanner/scanner_test.go
@@ -267,6 +267,7 @@ func TestScanString(t *testing.T) {
 		{in: `"foo\\bar"`, out: `foo\bar`},
 		{in: `"foo\"bar"`, out: `foo"bar`},
 		{in: `'foo\'bar'`, out: `foo'bar`},
+		{in: `'\xAF'`, out: `\xAF`},
 
 		{in: `"foo` + "\n", out: `foo`, err: "bad string"}, // newline in string
 		{in: `"foo`, out: `foo`, err: "bad string"},        // unclosed quotes
diff --git a/internal/testutil/expr.go b/internal/testutil/expr.go
index 02aa89ca..28b8e2dd 100644
--- a/internal/testutil/expr.go
+++ b/internal/testutil/expr.go
@@ -149,7 +149,7 @@ func ExprRunner(t *testing.T, testfile string) {
 						want, err := e.Eval(emptyEnv)
 						require.NoError(t, err)
 
-						// parse the given epxr
+						// parse the given expr
 						e, err = parser.NewParser(strings.NewReader(stmt.Expr)).ParseExpr()
 						require.NoError(t, err)