Repository: proullon/ramsql
Pull request #78 (2023-07-25 22:51:19)

add/scientific notation
We sqlx sends over values to the ramsql it doesn't support scientific notation

Joematpal: @proullon Thanks again for maintaining this project. There was still one test failing locally but I think it was there before.
proullon: Thanks !

I didn't even know it was supported by other engines :)

Patch:
diff --git a/engine/parser/lexer.go b/engine/parser/lexer.go
index c32f5da..8bd20e0 100644
--- a/engine/parser/lexer.go
+++ b/engine/parser/lexer.go
@@ -204,7 +204,7 @@ func (l *lexer) lex(instruction []byte) ([]Token, error) {
 	for l.pos < l.instructionLen {
 		r = false
 		for _, m := range matchers {
-			if r = m(); r == true {
+			if r = m(); r {
 				securityPos = l.pos
 				break
 			}
@@ -286,14 +286,16 @@ func (l *lexer) MatchStringToken() bool {
 func (l *lexer) MatchNumberToken() bool {
 
 	i := l.pos
-	for i < l.instructionLen && unicode.IsDigit(rune(l.instruction[i])) {
+	for i < l.instructionLen && (unicode.IsDigit(rune(l.instruction[i])) || l.instruction[i] == '.') {
 		i++
-	}
-	if i < l.instructionLen && l.instruction[i] == '.' {
-		i++
-		for i < l.instructionLen && unicode.IsDigit(rune(l.instruction[i])) {
+		// Checking for exponent/scientific notation; i.e. 32e-15, 55e15, 77e+100
+		if i < l.instructionLen && l.instruction[i] == 'e' {
 			i++
+			if i < l.instructionLen && (l.instruction[i] == '+' || l.instruction[i] == '-') {
+				i++
+			}
 		}
+
 	}
 
 	if i != l.pos {


Test patch:
diff --git a/driver/driver_test.go b/driver/driver_test.go
index 6845fe9..cbc3706 100644
--- a/driver/driver_test.go
+++ b/driver/driver_test.go
@@ -1199,3 +1199,69 @@ func TestSchema(t *testing.T) {
 		t.Fatalf("unexpected error dropping existing schema: %s", err)
 	}
 }
+
+func TestFloat(t *testing.T) {
+	log.UseTestLogger(t)
+
+	batch := []string{
+		`CREATE TABLE user (name TEXT, surname TEXT, age float(8));`,
+		`INSERT INTO user (name, surname, age) VALUES (Foo, Bar, 20.0);`,
+		`INSERT INTO user (name, surname, age) VALUES (John, Doe, 32.0);`,
+		`INSERT INTO user (name, surname, age) VALUES (Jane, Doe, 33.0939959238);`,
+		`INSERT INTO user (name, surname, age) VALUES (Joe, Doe, 1e-10);`,
+		`INSERT INTO user (name, surname, age) VALUES (Homer, Simpson, 40);`,
+		`INSERT INTO user (name, surname, age) VALUES (Marge, Simpson, 40);`,
+		`INSERT INTO user (name, surname, age) VALUES (Bruce, Wayne, 3333);`,
+	}
+
+	db, err := sql.Open("ramsql", "TestFloat")
+	if err != nil {
+		t.Fatalf("sql.Open : Error : %s\n", err)
+	}
+	defer db.Close()
+
+	for _, b := range batch {
+		_, err = db.Exec(b)
+		if err != nil {
+			t.Fatalf("sql.Exec: Error: %s\n", err)
+		}
+	}
+
+	query := `SELECT * FROM user
+			WHERE user.surname = Doe
+			AND user.name = Joe`
+
+	rows, err := db.Query(query)
+	if err != nil {
+		t.Fatalf("sql.Query: %s", err)
+	}
+
+	var nb int
+	for rows.Next() {
+		var name, surname string
+		var age float64
+		if err := rows.Scan(&name, &surname, &age); err != nil {
+			t.Fatalf("Cannot scan row: %s", err)
+		}
+		if surname != "Doe" && name != "Jane" {
+			t.Fatalf("Unwanted row: %s %s %f", name, surname, age)
+		}
+
+		nb++
+	}
+
+	if nb != 1 {
+		t.Fatalf("Expected 1 rows, got %d", nb)
+	}
+
+	query = `UPDATE user SET age = $3 WHERE name = $1 AND surname = $2`
+	var age float64 = 3450000000000
+
+	t.Logf("age in scientfic notation is 3.45e+12: %v", age)
+
+	_, err = db.Exec(query, "Bruce", "Wayne", age)
+	if err != nil {
+		t.Fatalf("Cannot run UPDATE query with AND: %s\n", err)
+	}
+
+}
diff --git a/engine/parser/lexer_test.go b/engine/parser/lexer_test.go
index e60661a..30d1538 100644
--- a/engine/parser/lexer_test.go
+++ b/engine/parser/lexer_test.go
@@ -6,7 +6,7 @@ import (
 )
 
 func TestLexerSimple(t *testing.T) {
-	query := `CREATE TABLE `+"`"+`account`+"`"+``
+	query := `CREATE TABLE ` + "`" + `account` + "`" + ``
 
 	lexer := lexer{}
 	decls, err := lexer.lex([]byte(query))
@@ -56,3 +56,57 @@ func TestLexerWithNEOperator(t *testing.T) {
 		t.Fatalf("Lexing failed, expected 21 tokens, got %d", len(decls))
 	}
 }
+
+func TestLexerWithInsertScientificNotation(t *testing.T) {
+	query := `INSERT INTO foo (substance, mass) values ('MnO2', 8694e-2)`
+
+	lexer := lexer{}
+	decls, err := lexer.lex([]byte(query))
+	if err != nil {
+		t.Fatalf("Cannot lex <%s> string", query)
+	}
+
+	if len(decls) != 23 {
+		t.Fatalf("Lexing failed, expected 21 tokens, got %d", len(decls))
+	}
+}
+
+func Test_lexer_MatchNumberToken(t *testing.T) {
+	tests := []struct {
+		name string
+		l    *lexer
+		want bool
+	}{
+		{
+			name: "should pass; thirty thousand",
+			l: &lexer{
+				instruction:    []byte("300000"),
+				instructionLen: len("300000"),
+			},
+			want: true,
+		},
+		{
+			name: "should pass; floating point",
+			l: &lexer{
+				instruction:    []byte("3.000000"),
+				instructionLen: len("3.000000"),
+			},
+			want: true,
+		},
+		{
+			name: "should pass; scientific notation",
+			l: &lexer{
+				instruction:    []byte("3e+6"),
+				instructionLen: len("3e+6"),
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.l.MatchNumberToken(); got != tt.want {
+				t.Errorf("lexer.MatchNumberToken() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}