Repository: tomarrell/lbadd
Pull request #48 (2020-02-20 16:42:21)

Rule based scanner
This PR when applied will be a complete implementation of a Rule Based Scanner.

Rule based scanner applies a set of rules one by one to figure out the matching rule for the set of runes, thus successfully scanning the input.


Patch:
diff --git a/go.mod b/go.mod
index 356bbef..58b9aaf 100644
--- a/go.mod
+++ b/go.mod
@@ -11,4 +11,5 @@ require (
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/text v0.3.2
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gotest.tools/gotestsum v0.4.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 02b3f91..1c90e77 100644
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,16 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.6.0 h1:66qjqZk8kalYAvDRtM1AdAJQI0tj4Wrue3Eq3B3pmFU=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -17,9 +25,26 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
 github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
@@ -27,6 +52,15 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94 h1:m5xBqfQdnzv6XuV/pJizrLOwUoGzyn1J249cA0cKL4o=
+golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc h1:ZMCWScCvS2fUVFw8LOpxyUUW5qiviqr4Dg5NdjLeiLU=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -35,9 +69,19 @@ golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw=
+gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gotest.tools/gotestsum v0.4.0 h1:vJsqvxHeCt1q6LQx9kLpiprZq/WvqvJfNoC6+zyCO+M=
+gotest.tools/gotestsum v0.4.0/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY=
diff --git a/internal/parser/scanner/ruleset/ruleset_default.go b/internal/parser/scanner/ruleset/ruleset_default.go
index b764a24..ef3e7c8 100644
--- a/internal/parser/scanner/ruleset/ruleset_default.go
+++ b/internal/parser/scanner/ruleset/ruleset_default.go
@@ -30,18 +30,30 @@ var (
 		matcher.New("Title", unicode.Title),
 		matcher.New("Number", unicode.Number),
 	)
-	defaultUnaryOperator = matcher.String("-+~")
-	defaultDelimiter     = matcher.String("(),")
-	defaultRules         = []Rule{
+	defaultQuote          = matcher.String("'\"")
+	defaultUnaryOperator  = matcher.String("-+~")
+	defaultBinaryOperator = matcher.String("|*/%<>=&!")
+	defaultDelimiter      = matcher.String("(),")
+	defaultRules          = []Rule{
 		FuncRule(defaultStatementSeparatorRule),
 		FuncRule(defaultKeywordsRule),
 		FuncRule(defaultUnaryOperatorRule),
+		FuncRule(defaultBinaryOperatorRule),
 		FuncRule(defaultDelimiterRule),
-		FuncRule(defaultUnquotedIdentifierRule),
+		FuncRule(defaultQuotedLiteralRule),
+		FuncRule(defaultUnquotedLiteralRule),
 	}
 	defaultKeywords = map[string]token.Type{"ABORT": token.KeywordAbort, "ACTION": token.KeywordAction, "ADD": token.KeywordAdd, "AFTER": token.KeywordAdd, "ALL": token.KeywordAll, "ALTER": token.KeywordAlter, "ANALYZE": token.KeywordAnalyze, "AND": token.KeywordAnd, "AS": token.KeywordAnd, "ASC": token.KeywordAsc, "ATTACH": token.KeywordAttach, "AUTOINCREMENT": token.KeywordAutoincrement, "BEFORE": token.KeywordBefore, "BEGIN": token.KeywordBegin, "BETWEEN": token.KeywordBetween, "BY": token.KeywordBy, "CASCADE": token.KeywordCascade, "CASE": token.KeywordCase, "CAST": token.KeywordCast, "CHECK": token.KeywordCheck, "COLLATE": token.KeywordCollate, "COLUMN": token.KeywordColumn, "COMMIT": token.KeywordCommit, "CONFLICT": token.KeywordConflict, "CONSTRAINT": token.KeywordConstraint, "CREATE": token.KeywordCreate, "CROSS": token.KeywordCross, "CURRENT": token.KeywordCurrent, "CURRENT_DATE": token.KeywordCurrentDate, "CURRENT_TIME": token.KeywordCurrentTime, "CURRENT_TIMESTAMP": token.KeywordCurrentTimestamp, "DATABASE": token.KeywordDatabase, "DEFAULT": token.KeywordDefault, "DEFERRABLE": token.KeywordDeferrable, "DEFERRED": token.KeywordDeferred, "DELETE": token.KeywordDelete, "DESC": token.KeywordDesc, "DETACH": token.KeywordDetach, "DISTINCT": token.KeywordDistinct, "DO": token.KeywordDo, "DROP": token.KeywordDrop, "EACH": token.KeywordEach, "ELSE": token.KeywordElse, "END": token.KeywordEnd, "ESCAPE": token.KeywordEscape, "EXCEPT": token.KeywordExcept, "EXCLUDE": token.KeywordExclude, "EXCLUSIVE": token.KeywordExclusive, "EXISTS": token.KeywordExists, "EXPLAIN": token.KeywordExplain, "FAIL": token.KeywordFail, "FILTER": token.KeywordFilter, "FIRST": token.KeywordFirst, "FOLLOWING": token.KeywordFollowing, "FOR": token.KeywordFor, "FOREIGN": token.KeywordForeign, "FROM": token.KeywordFrom, "FULL": token.KeywordFull, "GLOB": token.KeywordGlob, "GROUP": token.KeywordGroup, "GROUPS": token.KeywordGroups, "HAVING": token.KeywordHaving, "IF": token.KeywordIf, "IGNORE": token.KeywordIgnore, "IMMEDIATE": token.KeywordImmediate, "IN": token.KeywordIn, "INDEX": token.KeywordIndex, "INDEXED": token.KeywordIndexed, "INITIALLY": token.KeywordInitially, "INNER": token.KeywordInner, "INSERT": token.KeywordInsert, "INSTEAD": token.KeywordInstead, "INTERSECT": token.KeywordIntersect, "INTO": token.KeywordInto, "IS": token.KeywordIs, "ISNULL": token.KeywordIsnull, "JOIN": token.KeywordJoin, "KEY": token.KeywordKey, "LAST": token.KeywordLast, "LEFT": token.KeywordLeft, "LIKE": token.KeywordLike, "LIMIT": token.KeywordLimit, "MATCH": token.KeywordMatch, "NATURAL": token.KeywordNatural, "NO": token.KeywordNo, "NOT": token.KeywordNot, "NOTHING": token.KeywordNothing, "NOTNULL": token.KeywordNotnull, "NULL": token.KeywordNull, "OF": token.KeywordOf, "OFFSET": token.KeywordOffset, "ON": token.KeywordOn, "OR": token.KeywordOr, "ORDER": token.KeywordOrder, "OTHERS": token.KeywordOthers, "OUTER": token.KeywordOuter, "OVER": token.KeywordOver, "PARTITION": token.KeywordPartition, "PLAN": token.KeywordPlan, "PRAGMA": token.KeywordPragma, "PRECEDING": token.KeywordPreceding, "PRIMARY": token.KeywordPrimary, "QUERY": token.KeywordQuery, "RAISE": token.KeywordRaise, "RANGE": token.KeywordRange, "RECURSIVE": token.KeywordRecursive, "REFERENCES": token.KeywordReferences, "REGEXP": token.KeywordRegexp, "REINDEX": token.KeywordReindex, "RELEASE": token.KeywordRelease, "RENAME": token.KeywordRename, "REPLACE": token.KeywordReplace, "RESTRICT": token.KeywordRestrict, "RIGHT": token.KeywordRight, "ROLLBACK": token.KeywordRollback, "ROW": token.KeywordRow, "ROWS": token.KeywordRows, "SAVEPOINT": token.KeywordSavepoint, "SELECT": token.KeywordSelect, "SET": token.KeywordSet, "TABLE": token.KeywordTable, "TEMP": token.KeywordTemp, "TEMPORARY": token.KeywordTemporary, "THEN": token.KeywordThen, "TIES": token.KeywordTies, "TO": token.KeywordTo, "TRANSACTION": token.KeywordTransaction, "TRIGGER": token.KeywordTrigger, "UNBOUNDED": token.KeywordUnbounded, "UNION": token.KeywordUnion, "UNIQUE": token.KeywordUnique, "UPDATE": token.KeywordUpdate, "USING": token.KeywordUsing, "VACUUM": token.KeywordVacuum, "VALUES": token.KeywordValues, "VIEW": token.KeywordView, "VIRTUAL": token.KeywordVirtual, "WHEN": token.KeywordWhen, "WHERE": token.KeywordWhere, "WINDOW": token.KeywordWindow, "WITH": token.KeywordWith, "WITHOUT": token.KeywordWithout}
 )
 
+func defaultStatementSeparatorRule(s RuneScanner) (token.Type, bool) {
+	if next, ok := s.Lookahead(); ok && defaultStatementSeparator.Matches(next) {
+		s.ConsumeRune()
+		return token.StatementSeparator, true
+	}
+	return token.Unknown, false
+}
+
 func defaultKeywordsRule(s RuneScanner) (token.Type, bool) {
 	// read word
 	var buf bytes.Buffer
@@ -62,18 +74,60 @@ func defaultKeywordsRule(s RuneScanner) (token.Type, bool) {
 	return token.Unknown, false
 }
 
-func defaultStatementSeparatorRule(s RuneScanner) (token.Type, bool) {
-	if next, ok := s.Lookahead(); ok && defaultStatementSeparator.Matches(next) {
+func defaultUnaryOperatorRule(s RuneScanner) (token.Type, bool) {
+	if next, ok := s.Lookahead(); ok && defaultUnaryOperator.Matches(next) {
 		s.ConsumeRune()
-		return token.StatementSeparator, true
+		return token.UnaryOperator, true
 	}
 	return token.Unknown, false
 }
 
-func defaultUnaryOperatorRule(s RuneScanner) (token.Type, bool) {
-	if next, ok := s.Lookahead(); ok && defaultUnaryOperator.Matches(next) {
+// defaultBinaryOperatorRule scans binary operators. Since binary opeators
+// can be of length one and two, it first checks for former, then latter
+// and allows the best of the two conditions.
+func defaultBinaryOperatorRule(s RuneScanner) (token.Type, bool) {
+	if first, ok := s.Lookahead(); ok && defaultBinaryOperator.Matches(first) {
 		s.ConsumeRune()
-		return token.StatementSeparator, true
+		if first == '*' || first == '/' || first == '%' || first == '&' {
+			return token.BinaryOperator, true
+		} else if next, ok := s.Lookahead(); ok && defaultBinaryOperator.Matches(next) {
+			switch first {
+			case '|':
+				if next == '|' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '<':
+				if next == '<' || next == '=' || next == '>' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '>':
+				if next == '>' || next == '=' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '=':
+				if next == '=' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '!':
+				if next == '=' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+				// return token.Unknown, false
+			}
+		}
+		// special cases where these operators can be single or can have a suffix
+		// The switch case blocks have been designed such that only the cases where
+		// there's a meaningful operator, the runes are consumed and returned, in cases
+		// where the second operator is not meaningful, the first operator is consumed here.
+		// In cases where the second operator is not meaningful, it'll be processed again.
+		if first == '<' || first == '>' || first == '=' || first == '|' {
+			return token.BinaryOperator, true
+		}
 	}
 	return token.Unknown, false
 }
@@ -86,7 +140,37 @@ func defaultDelimiterRule(s RuneScanner) (token.Type, bool) {
 	return token.Unknown, false
 }
 
-func defaultUnquotedIdentifierRule(s RuneScanner) (token.Type, bool) {
+func defaultQuotedLiteralRule(s RuneScanner) (token.Type, bool) {
+	next, ok := s.Lookahead()
+	if !(ok && defaultQuote.Matches(next)) {
+		return token.Unknown, false
+	}
+	quoteType := next
+	s.ConsumeRune()
+
+	for {
+		next, ok := s.Lookahead()
+		if !ok {
+			return token.Unknown, false
+		}
+		if next == '\\' {
+			s.ConsumeRune()
+			next, ok = s.Lookahead()
+			if !ok {
+				return token.Unknown, false
+			}
+			s.ConsumeRune()
+		} else if next == quoteType {
+			break
+		} else {
+			s.ConsumeRune()
+		}
+	}
+	s.ConsumeRune()
+	return token.Literal, true
+}
+
+func defaultUnquotedLiteralRule(s RuneScanner) (token.Type, bool) {
 	if next, ok := s.Lookahead(); !(ok && defaultLiteral.Matches(next)) {
 		return token.Unknown, false
 	}


Test patch:
diff --git a/internal/parser/scanner/rule_based_scanner_test.go b/internal/parser/scanner/rule_based_scanner_test.go
index cf4f77b..09b875d 100644
--- a/internal/parser/scanner/rule_based_scanner_test.go
+++ b/internal/parser/scanner/rule_based_scanner_test.go
@@ -24,6 +24,122 @@ func TestRuleBasedScanner(t *testing.T) {
 				token.New(1, 18, 17, 0, token.EOF, ""),
 			},
 		},
+		{
+			"SELECT FROM \"WHERE\"",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 7, token.Literal, "\"WHERE\""),
+				token.New(1, 20, 19, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT FROM \"WHERE",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 1, token.Error, `unexpected token: '"' at offset 12`),
+				token.New(1, 14, 13, 5, token.KeywordWhere, "WHERE"),
+				token.New(1, 19, 18, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT      FROM || & +7 59 \"foobar\"",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 13, 12, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 18, 17, 2, token.BinaryOperator, "||"),
+				token.New(1, 21, 20, 1, token.BinaryOperator, "&"),
+				token.New(1, 23, 22, 1, token.UnaryOperator, "+"),
+				token.New(1, 24, 23, 1, token.Literal, "7"),
+				token.New(1, 26, 25, 2, token.Literal, "59"),
+				token.New(1, 29, 28, 8, token.Literal, "\"foobar\""),
+				token.New(1, 37, 36, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT FROM 'WHERE'",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 7, token.Literal, "'WHERE'"),
+				token.New(1, 20, 19, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT \"myCol FROM \"myTable\"",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 13, token.Literal, `"myCol FROM "`),
+				token.New(1, 21, 20, 7, token.Literal, "myTable"),
+				token.New(1, 28, 27, 1, token.Error, "unexpected token: '\"' at offset 27"),
+				token.New(1, 29, 28, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT \" FROM",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 1, token.Error, `unexpected token: '"' at offset 7`),
+				token.New(1, 10, 9, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 14, 13, 0, token.EOF, ""),
+			},
+		},
+		{
+			`SELECT FROM "this \" can be anything"`,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 25, token.Literal, `"this \" can be anything"`),
+				token.New(1, 38, 37, 0, token.EOF, ""),
+			},
+		},
+		{
+			`SELECT FROM 'this \' can be anything'`,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 25, token.Literal, `'this \' can be anything'`),
+				token.New(1, 38, 37, 0, token.EOF, ""),
+			},
+		},
+		{
+			`|| * / % + - ~ << >> & | < <= > >= = == != <> !>> >>`,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 2, token.BinaryOperator, "||"),
+				token.New(1, 4, 3, 1, token.BinaryOperator, "*"),
+				token.New(1, 6, 5, 1, token.BinaryOperator, "/"),
+				token.New(1, 8, 7, 1, token.BinaryOperator, "%"),
+				token.New(1, 10, 9, 1, token.UnaryOperator, "+"),
+				token.New(1, 12, 11, 1, token.UnaryOperator, "-"),
+				token.New(1, 14, 13, 1, token.UnaryOperator, "~"),
+				token.New(1, 16, 15, 2, token.BinaryOperator, "<<"),
+				token.New(1, 19, 18, 2, token.BinaryOperator, ">>"),
+				token.New(1, 22, 21, 1, token.BinaryOperator, "&"),
+				token.New(1, 24, 23, 1, token.BinaryOperator, "|"),
+				token.New(1, 26, 25, 1, token.BinaryOperator, "<"),
+				token.New(1, 28, 27, 2, token.BinaryOperator, "<="),
+				token.New(1, 31, 30, 1, token.BinaryOperator, ">"),
+				token.New(1, 33, 32, 2, token.BinaryOperator, ">="),
+				token.New(1, 36, 35, 1, token.BinaryOperator, "="),
+				token.New(1, 38, 37, 2, token.BinaryOperator, "=="),
+				token.New(1, 41, 40, 2, token.BinaryOperator, "!="),
+				token.New(1, 44, 43, 2, token.BinaryOperator, "<>"),
+				token.New(1, 47, 46, 1, token.Error, "unexpected token: '!' at offset 46"),
+				token.New(1, 48, 47, 2, token.BinaryOperator, ">>"),
+				token.New(1, 51, 50, 2, token.BinaryOperator, ">>"),
+				token.New(1, 53, 52, 0, token.EOF, ""),
+			},
+		},
 	}
 	for _, input := range inputs {
 		t.Run("ruleset=default/"+input.query, _TestRuleBasedScannerWithRuleset(input.query, input.ruleset, input.want))
@@ -47,8 +163,7 @@ func _TestRuleBasedScannerWithRuleset(input string, ruleset ruleset.Ruleset, wan
 				break
 			}
 		}
-
-		assert.Equalf(len(want), len(got), "did not receive as many tokens as expected (expected %d, but got %d)", len(want), len(got))
+		assert.Equalf(len(want), len(got), "did not receive as much tokens as expected (expected %d, but got %d)", len(want), len(got))
 
 		limit := len(want)
 		if len(got) < limit {