Repository: tomarrell/lbadd
Pull request #61 (2020-02-26 05:12:59)

Added support for decimal points, exponents and hexadecimal numbers.
This PR adds the support for decimal points, exponents of the form `1.657E19` and `1.234E+21`, hexadecimals of the form `0xNUMBER` in unquoted numeric literals


Patch:
diff --git a/internal/parser/scanner/ruleset/ruleset_default.go b/internal/parser/scanner/ruleset/ruleset_default.go
index 48c5503..2cd2730 100644
--- a/internal/parser/scanner/ruleset/ruleset_default.go
+++ b/internal/parser/scanner/ruleset/ruleset_default.go
@@ -23,24 +23,36 @@ var (
 	// defaultLinefeedDetector is the linefeed detector that this ruleset allows
 	defaultLinefeedDetector   = matcher.RuneWithDesc("linefeed", '\n')
 	defaultStatementSeparator = matcher.RuneWithDesc("statement separator", ';')
+	defaultDecimalPoint       = matcher.RuneWithDesc("decimal point", '.')
+	defaultExponent           = matcher.RuneWithDesc("exponent indicator", 'E')
+	defaultExponentOperator   = matcher.String("+-")
+	defaultNumber             = matcher.New("number", unicode.Number)
 	// defaultLiteral matches the allowed letters of a literal
 	defaultLiteral = matcher.Merge(
-		matcher.New("Upper", unicode.Upper),
-		matcher.New("Lower", unicode.Lower),
-		matcher.New("Title", unicode.Title),
-		matcher.New("Number", unicode.Number),
+		matcher.New("upper", unicode.Upper),
+		matcher.New("lower", unicode.Lower),
+		matcher.New("title", unicode.Title),
+		defaultNumber,
+	)
+	defaultNumericLiteral = matcher.Merge(
+		defaultNumber,
+		defaultExponent,
+		defaultExponentOperator,
+		matcher.RuneWithDesc("X", 'x'),
 	)
 	defaultQuote          = matcher.String("'\"")
 	defaultUnaryOperator  = matcher.String("-+~")
 	defaultBinaryOperator = matcher.String("|*/%<>=&!")
 	defaultDelimiter      = matcher.String("(),")
-	defaultRules          = []Rule{
+	// the order of the rules are important for some cases. Beware
+	defaultRules = []Rule{
 		FuncRule(defaultStatementSeparatorRule),
 		FuncRule(defaultKeywordsRule),
 		FuncRule(defaultUnaryOperatorRule),
 		FuncRule(defaultBinaryOperatorRule),
 		FuncRule(defaultDelimiterRule),
 		FuncRule(defaultQuotedLiteralRule),
+		FuncRule(defaultNumericLiteralRule),
 		FuncRule(defaultUnquotedLiteralRule),
 	}
 	defaultKeywords = map[string]token.Type{"ABORT": token.KeywordAbort, "ACTION": token.KeywordAction, "ADD": token.KeywordAdd, "AFTER": token.KeywordAdd, "ALL": token.KeywordAll, "ALTER": token.KeywordAlter, "ANALYZE": token.KeywordAnalyze, "AND": token.KeywordAnd, "AS": token.KeywordAnd, "ASC": token.KeywordAsc, "ATTACH": token.KeywordAttach, "AUTOINCREMENT": token.KeywordAutoincrement, "BEFORE": token.KeywordBefore, "BEGIN": token.KeywordBegin, "BETWEEN": token.KeywordBetween, "BY": token.KeywordBy, "CASCADE": token.KeywordCascade, "CASE": token.KeywordCase, "CAST": token.KeywordCast, "CHECK": token.KeywordCheck, "COLLATE": token.KeywordCollate, "COLUMN": token.KeywordColumn, "COMMIT": token.KeywordCommit, "CONFLICT": token.KeywordConflict, "CONSTRAINT": token.KeywordConstraint, "CREATE": token.KeywordCreate, "CROSS": token.KeywordCross, "CURRENT": token.KeywordCurrent, "CURRENT_DATE": token.KeywordCurrentDate, "CURRENT_TIME": token.KeywordCurrentTime, "CURRENT_TIMESTAMP": token.KeywordCurrentTimestamp, "DATABASE": token.KeywordDatabase, "DEFAULT": token.KeywordDefault, "DEFERRABLE": token.KeywordDeferrable, "DEFERRED": token.KeywordDeferred, "DELETE": token.KeywordDelete, "DESC": token.KeywordDesc, "DETACH": token.KeywordDetach, "DISTINCT": token.KeywordDistinct, "DO": token.KeywordDo, "DROP": token.KeywordDrop, "EACH": token.KeywordEach, "ELSE": token.KeywordElse, "END": token.KeywordEnd, "ESCAPE": token.KeywordEscape, "EXCEPT": token.KeywordExcept, "EXCLUDE": token.KeywordExclude, "EXCLUSIVE": token.KeywordExclusive, "EXISTS": token.KeywordExists, "EXPLAIN": token.KeywordExplain, "FAIL": token.KeywordFail, "FILTER": token.KeywordFilter, "FIRST": token.KeywordFirst, "FOLLOWING": token.KeywordFollowing, "FOR": token.KeywordFor, "FOREIGN": token.KeywordForeign, "FROM": token.KeywordFrom, "FULL": token.KeywordFull, "GLOB": token.KeywordGlob, "GROUP": token.KeywordGroup, "GROUPS": token.KeywordGroups, "HAVING": token.KeywordHaving, "IF": token.KeywordIf, "IGNORE": token.KeywordIgnore, "IMMEDIATE": token.KeywordImmediate, "IN": token.KeywordIn, "INDEX": token.KeywordIndex, "INDEXED": token.KeywordIndexed, "INITIALLY": token.KeywordInitially, "INNER": token.KeywordInner, "INSERT": token.KeywordInsert, "INSTEAD": token.KeywordInstead, "INTERSECT": token.KeywordIntersect, "INTO": token.KeywordInto, "IS": token.KeywordIs, "ISNULL": token.KeywordIsnull, "JOIN": token.KeywordJoin, "KEY": token.KeywordKey, "LAST": token.KeywordLast, "LEFT": token.KeywordLeft, "LIKE": token.KeywordLike, "LIMIT": token.KeywordLimit, "MATCH": token.KeywordMatch, "NATURAL": token.KeywordNatural, "NO": token.KeywordNo, "NOT": token.KeywordNot, "NOTHING": token.KeywordNothing, "NOTNULL": token.KeywordNotnull, "NULL": token.KeywordNull, "OF": token.KeywordOf, "OFFSET": token.KeywordOffset, "ON": token.KeywordOn, "OR": token.KeywordOr, "ORDER": token.KeywordOrder, "OTHERS": token.KeywordOthers, "OUTER": token.KeywordOuter, "OVER": token.KeywordOver, "PARTITION": token.KeywordPartition, "PLAN": token.KeywordPlan, "PRAGMA": token.KeywordPragma, "PRECEDING": token.KeywordPreceding, "PRIMARY": token.KeywordPrimary, "QUERY": token.KeywordQuery, "RAISE": token.KeywordRaise, "RANGE": token.KeywordRange, "RECURSIVE": token.KeywordRecursive, "REFERENCES": token.KeywordReferences, "REGEXP": token.KeywordRegexp, "REINDEX": token.KeywordReindex, "RELEASE": token.KeywordRelease, "RENAME": token.KeywordRename, "REPLACE": token.KeywordReplace, "RESTRICT": token.KeywordRestrict, "RIGHT": token.KeywordRight, "ROLLBACK": token.KeywordRollback, "ROW": token.KeywordRow, "ROWS": token.KeywordRows, "SAVEPOINT": token.KeywordSavepoint, "SELECT": token.KeywordSelect, "SET": token.KeywordSet, "TABLE": token.KeywordTable, "TEMP": token.KeywordTemp, "TEMPORARY": token.KeywordTemporary, "THEN": token.KeywordThen, "TIES": token.KeywordTies, "TO": token.KeywordTo, "TRANSACTION": token.KeywordTransaction, "TRIGGER": token.KeywordTrigger, "UNBOUNDED": token.KeywordUnbounded, "UNION": token.KeywordUnion, "UNIQUE": token.KeywordUnique, "UPDATE": token.KeywordUpdate, "USING": token.KeywordUsing, "VACUUM": token.KeywordVacuum, "VALUES": token.KeywordValues, "VIEW": token.KeywordView, "VIRTUAL": token.KeywordVirtual, "WHEN": token.KeywordWhen, "WHERE": token.KeywordWhere, "WINDOW": token.KeywordWindow, "WITH": token.KeywordWith, "WITHOUT": token.KeywordWithout}
@@ -117,7 +129,6 @@ func defaultBinaryOperatorRule(s RuneScanner) (token.Type, bool) {
 					s.ConsumeRune()
 					return token.BinaryOperator, true
 				}
-				// return token.Unknown, false
 			}
 		}
 		// special cases where these operators can be single or can have a suffix
@@ -153,6 +164,7 @@ func defaultQuotedLiteralRule(s RuneScanner) (token.Type, bool) {
 		if !ok {
 			return token.Unknown, false
 		}
+		// allowing character escape
 		if next == '\\' {
 			s.ConsumeRune()
 			_, ok = s.Lookahead()
@@ -186,3 +198,67 @@ func defaultUnquotedLiteralRule(s RuneScanner) (token.Type, bool) {
 
 	return token.Literal, true
 }
+
+func defaultNumericLiteralRule(s RuneScanner) (token.Type, bool) {
+	decimalPointFlag := false
+	exponentFlag := false
+	exponentOperatorFlag := false
+	// Checking whether the first element is a number or a decimal point.
+	// If neither, an unknown token error is raised.
+	next, ok := s.Lookahead()
+	if !(ok && (defaultNumericLiteral.Matches(next) || defaultDecimalPoint.Matches(next))) {
+		return token.Unknown, false
+	}
+	// If the literal starts with a decimal point, it is recorded in the flag.
+	if defaultDecimalPoint.Matches(next) {
+		decimalPointFlag = true
+	}
+	s.ConsumeRune()
+	// case of hexadecimal numbers
+	if next == '0' {
+		for {
+			next, ok = s.Lookahead()
+			if !ok || next != 'x' {
+				break
+			}
+			if next == 'x' {
+				s.ConsumeRune()
+				return defaultUnquotedLiteralRule(s)
+			}
+		}
+	}
+	for {
+		// in the above case, if there was a `0.34` as a part of the string to read,
+		// the loop would have broken without consuming the rune; because the above
+		// loop consumes only hexadecimals of form `0xNUMBER`. Since all other cases
+		// are not consumed, the `LookAhead` below, gets the previous rune conveniently.
+		next, ok := s.Lookahead()
+		// continue on cases where the decimal point/exponent/exponent opeartor is already not found or not this particular rune.
+		if !(ok && (defaultNumericLiteral.Matches(next) || (!decimalPointFlag && defaultDecimalPoint.Matches(next)) || (!exponentFlag && defaultExponent.Matches(next)) || (!exponentOperatorFlag && defaultExponentOperator.Matches(next)))) {
+			break
+		}
+		if next == '.' {
+			if decimalPointFlag {
+				return token.Unknown, false
+			}
+			decimalPointFlag = true
+		}
+		if next == 'E' {
+			if exponentFlag {
+				return token.Unknown, false
+			}
+			exponentFlag = true
+		}
+		if next == '+' || next == '-' {
+			if exponentOperatorFlag {
+				return token.Unknown, false
+			}
+			exponentOperatorFlag = true
+		}
+		if defaultDecimalPoint.Matches(next) {
+			decimalPointFlag = true
+		}
+		s.ConsumeRune()
+	}
+	return token.Literal, true
+}


Test patch:
diff --git a/internal/parser/scanner/rule_based_scanner_test.go b/internal/parser/scanner/rule_based_scanner_test.go
index fb7f354..ac34999 100644
--- a/internal/parser/scanner/rule_based_scanner_test.go
+++ b/internal/parser/scanner/rule_based_scanner_test.go
@@ -46,7 +46,7 @@ func TestRuleBasedScanner(t *testing.T) {
 			},
 		},
 		{
-			"SELECT      FROM || & +7 59 \"foobar\"",
+			"SELECT      FROM || & +7 5 \"foobar\"",
 			ruleset.Default,
 			[]token.Token{
 				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
@@ -55,9 +55,24 @@ func TestRuleBasedScanner(t *testing.T) {
 				token.New(1, 21, 20, 1, token.BinaryOperator, "&"),
 				token.New(1, 23, 22, 1, token.UnaryOperator, "+"),
 				token.New(1, 24, 23, 1, token.Literal, "7"),
-				token.New(1, 26, 25, 2, token.Literal, "59"),
-				token.New(1, 29, 28, 8, token.Literal, "\"foobar\""),
-				token.New(1, 37, 36, 0, token.EOF, ""),
+				token.New(1, 26, 25, 1, token.Literal, "5"),
+				token.New(1, 28, 27, 8, token.Literal, "\"foobar\""),
+				token.New(1, 36, 35, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT      FROM || & +7 5.9 \"foobar\"",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 13, 12, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 18, 17, 2, token.BinaryOperator, "||"),
+				token.New(1, 21, 20, 1, token.BinaryOperator, "&"),
+				token.New(1, 23, 22, 1, token.UnaryOperator, "+"),
+				token.New(1, 24, 23, 1, token.Literal, "7"),
+				token.New(1, 26, 25, 3, token.Literal, "5.9"),
+				token.New(1, 30, 29, 8, token.Literal, "\"foobar\""),
+				token.New(1, 38, 37, 0, token.EOF, ""),
 			},
 		},
 		{
@@ -140,6 +155,48 @@ func TestRuleBasedScanner(t *testing.T) {
 				token.New(1, 53, 52, 0, token.EOF, ""),
 			},
 		},
+		{
+			`7 7.5 8.9.8 8.0 0.4 10 10000 18907.890 1890976.09.977`,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 1, token.Literal, "7"),
+				token.New(1, 3, 2, 3, token.Literal, "7.5"),
+				token.New(1, 7, 6, 3, token.Literal, "8.9"),
+				token.New(1, 10, 9, 2, token.Literal, ".8"),
+				token.New(1, 13, 12, 3, token.Literal, "8.0"),
+				token.New(1, 17, 16, 3, token.Literal, "0.4"),
+				token.New(1, 21, 20, 2, token.Literal, "10"),
+				token.New(1, 24, 23, 5, token.Literal, "10000"),
+				token.New(1, 30, 29, 9, token.Literal, "18907.890"),
+				token.New(1, 40, 39, 10, token.Literal, "1890976.09"),
+				token.New(1, 50, 49, 4, token.Literal, ".977"),
+				token.New(1, 54, 53, 0, token.EOF, ""),
+			},
+		},
+		{
+			`11.672E19 11.672E+19 11.657EE19 0xCAFEBABE 2.5E-1 1.2.3.4.5.6.7 5.hello something.4 `,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 9, token.Literal, "11.672E19"),
+				token.New(1, 11, 10, 10, token.Literal, "11.672E+19"),
+				token.New(1, 22, 21, 2, token.Literal, "11"),
+				token.New(1, 24, 23, 1, token.Error, "unexpected token: '.' at offset 23"),
+				token.New(1, 25, 24, 7, token.Literal, "657EE19"),
+				token.New(1, 33, 32, 10, token.Literal, "0xCAFEBABE"),
+				token.New(1, 44, 43, 6, token.Literal, "2.5E-1"),
+				token.New(1, 51, 50, 3, token.Literal, "1.2"),
+				token.New(1, 54, 53, 2, token.Literal, ".3"),
+				token.New(1, 56, 55, 2, token.Literal, ".4"),
+				token.New(1, 58, 57, 2, token.Literal, ".5"),
+				token.New(1, 60, 59, 2, token.Literal, ".6"),
+				token.New(1, 62, 61, 2, token.Literal, ".7"),
+				token.New(1, 65, 64, 2, token.Literal, "5."),
+				token.New(1, 67, 66, 5, token.Literal, "hello"),
+				token.New(1, 73, 72, 9, token.Literal, "something"),
+				token.New(1, 82, 81, 2, token.Literal, ".4"),
+				token.New(1, 85, 84, 0, token.EOF, ""),
+			},
+		},
 	}
 	for _, input := range inputs {
 		t.Run("ruleset=default/"+input.query, _TestRuleBasedScannerWithRuleset(input.query, input.ruleset, input.want))