Repository: tomarrell/lbadd
Pull request #46 (2020-02-20 08:58:26)

Executor


Patch:
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..594bea6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+.DS_Store
+*.log
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 7a89690..c360d82 100644
--- a/Makefile
+++ b/Makefile
@@ -8,12 +8,10 @@ test: ## Runs the unit test suite
 
 .PHONY: lint
 lint: ## Runs the linters
-	golint;
-	errcheck;
-	gosec ./...;
-
-	# TODO re-enable staticcheck
-	# staticcheck;
+	golint ./...;
+	errcheck ./...;
+	gosec -quiet ./...;
+	staticcheck ./...;
 
 
 ## Help display.
diff --git a/cmd/repl/main.go b/cmd/repl/main.go
index a730fde..e7c1fdd 100644
--- a/cmd/repl/main.go
+++ b/cmd/repl/main.go
@@ -1,33 +1,100 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
 
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/diode"
 	"github.com/tomarrell/lbadd/internal/cli"
+	"github.com/tomarrell/lbadd/internal/executor"
+)
+
+const (
+	// ExitAbnormal is the exit code that the application will return upon
+	// internal abnormal exit.
+	ExitAbnormal = 1
+	// ExitInterrupt is the exit code that the application will return when
+	// aborted by the user.
+	ExitInterrupt = 2
 )
 
 func main() {
 	if err := run(os.Args, os.Stdin, os.Stdout, os.Stderr); err != nil {
 		_, _ = fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
+		os.Exit(ExitAbnormal)
 	}
 }
 
 func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error {
+	// setup flags
 	flags := flag.NewFlagSet(args[0], flag.ExitOnError)
 	var (
 		verbose = flags.Bool("verbose", false, "enable verbose output")
+		logfile = flags.String("logfile", "lbadd.cli.log", "define a log file to write messages to")
 	)
-	_ = *verbose // TODO: use *verbose to configure a logger
 
 	if err := flags.Parse(args[1:]); err != nil {
 		return fmt.Errorf("parse flags: %w", err)
 	}
 
-	cli := cli.New(stdin, stdout, nil) // TODO: pass in a functional executor
+	// programCtx is the context, that all components should run on. When
+	// invoking cancel, all started components should stop processing.
+	programCtx, cancel := context.WithCancel(context.Background())
+
+	// start listening for signals
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	defer func() {
+		signal.Stop(signalChan)
+		cancel()
+	}()
+	go func() {
+		select {
+		case <-signalChan: // first signal, cancel context
+			cancel()
+			_, _ = fmt.Fprintln(stdout, "Attempting graceful shutdown, press again to force quit")
+		case <-programCtx.Done():
+		}
+		<-signalChan // second signal, hard exit
+		os.Exit(ExitInterrupt)
+	}()
+
+	// Initialize a root logger
+	file, err := os.OpenFile(*logfile, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0600)
+	if err != nil {
+		return fmt.Errorf("open logfile: %w", err)
+	}
+
+	log := zerolog.New(
+		diode.NewWriter(
+			file,                // output writer
+			1000,                // pool size
+			10*time.Millisecond, // poll interval
+			func(missed int) {
+				_, _ = fmt.Fprintf(stderr, "Logger is falling behind, skipping %d messages\n", missed)
+			},
+		),
+	).With().
+		Timestamp().
+		Logger().
+		Level(zerolog.InfoLevel)
+
+	log.Info().Msg("start new session")
+
+	// apply the verbose flag
+	if *verbose {
+		log = log.Level(zerolog.TraceLevel)
+	}
+
+	// run the cli
+	cli := cli.New(programCtx, stdin, stdout, executor.New(log.With().Str("component", "executor").Logger()))
 	cli.Start()
 
 	return nil
diff --git a/codegen.go b/codegen.go
deleted file mode 100644
index 58e54cb..0000000
--- a/codegen.go
+++ /dev/null
@@ -1,1 +0,0 @@
-package lbadd
diff --git a/command.go b/command.go
deleted file mode 100644
index 9d0caf5..0000000
--- a/command.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package lbadd
-
-type command int
-
-const (
-	commandUnknown command = iota
-	commandInsert
-	commandSelect
-	commandDelete
-	commandCreateTable
-)
-
-func newCommand(cmd string) command {
-	switch toUp(cmd) {
-	case commandInsert.String():
-		return commandInsert
-	case commandSelect.String():
-		return commandSelect
-	case commandDelete.String():
-		return commandDelete
-	case commandCreateTable.String():
-		return commandCreateTable
-	default:
-		return commandUnknown
-	}
-}
-
-func (c command) String() string {
-	switch c {
-	case commandInsert:
-		return "INSERT"
-	case commandSelect:
-		return "SELECT"
-	case commandDelete:
-		return "DELETE"
-	case commandCreateTable:
-		return "CREATE TABLE"
-	default:
-		return "UNKNOWN"
-	}
-}
diff --git a/doc/internal/parser/scanner/scanner b/doc/internal/parser/scanner/scanner
deleted file mode 100644
index e69de29..0000000
diff --git a/executor.go b/executor.go
deleted file mode 100644
index b865ff7..0000000
--- a/executor.go
+++ /dev/null
@@ -1,165 +0,0 @@
-package lbadd
-
-import (
-	"fmt"
-	"regexp"
-)
-
-// Contains a command and associated information required to execute such command
-type instruction struct {
-	command command
-	table   string
-	params  []string
-}
-
-// A single column on a single row
-type record []byte
-
-// A row containing multiple records
-type row []record
-
-// A response from the executor
-type result struct {
-	columns      []column // columns in order of stored rows
-	rows         []row    // the set of rows
-	rowsAffected int      // the number of rows affected by execution
-	created      int      // the number of resources created
-}
-
-type exeConfig struct {
-	order int
-}
-
-// Execute executes an instruction against the database
-type executor struct {
-	db  *db
-	cfg exeConfig
-}
-
-func newExecutor(cfg exeConfig) *executor {
-	return &executor{
-		db:  newDB(),
-		cfg: cfg,
-	}
-}
-
-// The executor takes an instruction, and coordinates the operations which are
-// required to fulfill the instruction, executing these against the DB. It
-// also returns the result of the instruction.
-func (e *executor) execute(instr instruction) (result, error) {
-	switch instr.command {
-	case commandInsert:
-		return result{}, fmt.Errorf("unimplemented")
-	case commandSelect:
-		return e.executeSelect(instr)
-	case commandDelete:
-		return result{}, fmt.Errorf("unimplemented")
-	case commandCreateTable:
-		return e.executeCreateTable(instr)
-
-	default:
-		return result{}, fmt.Errorf("invalid executor command")
-	}
-}
-
-// Executes the select query instruction, returning the structure of the table
-// (columns) and the rows specified in the query.
-func (e *executor) executeSelect(instr instruction) (result, error) {
-	_, exists := e.db.tables[instr.table]
-	if !exists {
-		return result{}, fmt.Errorf("table %s does not exist", instr.table)
-	}
-
-	// TODO check if columns all exist in table
-	// TODO btree get all method
-
-	return result{}, fmt.Errorf("unimplemented")
-}
-
-// Executes the create table instruction, parses the columns given as arguments
-// and adds a new table record to the storage map.
-func (e *executor) executeCreateTable(instr instruction) (result, error) {
-	cols, err := parseInsertColumns(instr.params)
-	if err != nil {
-		return result{}, fmt.Errorf("failed to parse column params: %v", err)
-	}
-
-	e.db.tables[instr.table] = table{
-		name:    instr.table,
-		store:   newBtreeOrder(e.cfg.order),
-		columns: cols,
-	}
-
-	return result{created: 1}, nil
-}
-
-func parseInsertColumns(params []string) ([]column, error) {
-	// If there are no tables to be created, return early
-	if len(params) == 0 {
-		return []column{}, nil
-	}
-
-	// There should be a column name, type and nullable field for each column
-	// which is being declared
-	if len(params)%3 != 0 {
-		return []column{}, fmt.Errorf("invalid column pairs, every name must have a type")
-	}
-
-	cols := make([]column, 0, len(params)/3)
-
-	for i := 0; i < len(params); i += 3 {
-		p1, p2, p3 := params[i], params[i+1], params[i+2]
-		if err := validateTableName(p1); err != nil {
-			return cols, fmt.Errorf("invalid table name: %s: %v", p1, err)
-		}
-
-		colType := parseColumnType(p2)
-		if colType == columnTypeInvalid {
-			return cols, fmt.Errorf("found invalid column type: %s", p2)
-		}
-
-		b, err := parseBool(p3)
-		if err != nil {
-			return cols, fmt.Errorf("invalid value for field is_nullable: %s", err)
-		}
-
-		cols = append(cols, column{
-			name:       p1,
-			dataType:   colType,
-			isNullable: b,
-		})
-	}
-
-	return cols, nil
-}
-
-// The maximum number of characters allowed in a table name
-const tableNameMaxLen = 32
-
-// The validation pattern used to determine whether a table name is valid
-var tableNamePattern = regexp.MustCompile(`^[a-zA-Z]+$`)
-
-// Validates whether the string can be used as a valid table name identifier.
-// Returns an error if it is invalid, nil if valid.
-func validateTableName(name string) error {
-	if len(name) > tableNameMaxLen {
-		return fmt.Errorf("table name exceeds the character limit of %d", tableNameMaxLen)
-	}
-
-	if !tableNamePattern.MatchString(name) {
-		return fmt.Errorf("table name includes invalid characters")
-	}
-
-	return nil
-}
-
-func parseBool(str string) (bool, error) {
-	switch str {
-	case "true":
-		return true, nil
-	case "false":
-		return false, nil
-	default:
-		return false, fmt.Errorf("invalid boolean field")
-	}
-}
diff --git a/go.mod b/go.mod
index 8be9d5b..58b9aaf 100644
--- a/go.mod
+++ b/go.mod
@@ -3,10 +3,13 @@ module github.com/tomarrell/lbadd
 go 1.13
 
 require (
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
+	github.com/TimSatke/golden v0.1.1
+	github.com/davecgh/go-spew v1.1.1
+	github.com/google/go-cmp v0.4.0
+	github.com/kr/pretty v0.2.0 // indirect
+	github.com/rs/zerolog v1.18.0
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/text v0.3.2
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-	gopkg.in/yaml.v2 v2.2.7 // indirect
+	gotest.tools/gotestsum v0.4.0 // indirect
 )
diff --git a/go.sum b/go.sum
index e93bcc3..1c90e77 100644
--- a/go.sum
+++ b/go.sum
@@ -1,25 +1,87 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/TimSatke/golden v0.1.1 h1:hWYcD5ke6xM9wvah6TapbiNnWkQZV6yXhLfklf9FL2I=
+github.com/TimSatke/golden v0.1.1/go.mod h1:TkKBb5ZMuCM4Zy3X2wk8Apu7kpWgbrKJS05i4iiYJAY=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/fatih/color v1.6.0 h1:66qjqZk8kalYAvDRtM1AdAJQI0tj4Wrue3Eq3B3pmFU=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
+github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94 h1:m5xBqfQdnzv6XuV/pJizrLOwUoGzyn1J249cA0cKL4o=
+golang.org/x/crypto v0.0.0-20180426230345-b49d69b5da94/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc h1:ZMCWScCvS2fUVFw8LOpxyUUW5qiviqr4Dg5NdjLeiLU=
+golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
-gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw=
+gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gotest.tools/gotestsum v0.4.0 h1:vJsqvxHeCt1q6LQx9kLpiprZq/WvqvJfNoC6+zyCO+M=
+gotest.tools/gotestsum v0.4.0/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY=
diff --git a/internal/cli/cli.go b/internal/cli/cli.go
index a780bf8..8ddb73a 100644
--- a/internal/cli/cli.go
+++ b/internal/cli/cli.go
@@ -1,26 +1,23 @@
 package cli
 
 import (
+	"context"
 	"io"
 
-	"github.com/tomarrell/lbadd/internal/parser/ast"
+	"github.com/tomarrell/lbadd/internal/executor"
 )
 
-// Cli describes a command line interface that can be started and closed. It
-// should process commands from a defined input as long as it is running.
-// Processing must stop, when the cli is closed.
+// Cli describes a command line interface that can be started. A cli runs under
+// a context. Processing must start when the cli is started and stopped, when
+// the context is canceled.
+//
+// A cli is a way to interact with the database, with the indirection of an
+// (executor.Executor).
 type Cli interface {
 	Start()
-	io.Closer
-}
-
-// Executor describes a component that can execute the AST that is produced when
-// parsing the input.
-type Executor interface {
-	Execute(*ast.SQLStmt) error
 }
 
 // New creates a new Cli that can immediately be started.
-func New(in io.Reader, out io.Writer, exec Executor) Cli {
-	return newSimpleCli(in, out, exec)
+func New(ctx context.Context, in io.Reader, out io.Writer, exec executor.Executor) Cli {
+	return newSimpleCli(ctx, in, out, exec)
 }
diff --git a/internal/cli/simple_cli.go b/internal/cli/simple_cli.go
index 82e608d..a8d71d7 100644
--- a/internal/cli/simple_cli.go
+++ b/internal/cli/simple_cli.go
@@ -2,26 +2,31 @@ package cli
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"io"
 
+	"github.com/tomarrell/lbadd/internal/executor"
+	"github.com/tomarrell/lbadd/internal/executor/command"
 	"github.com/tomarrell/lbadd/internal/parser"
 )
 
 var _ Cli = (*simpleCli)(nil)
 
 type simpleCli struct {
+	ctx context.Context
+
 	in  io.Reader
 	out io.Writer
 
-	closed  bool
 	scanner *bufio.Scanner
 
-	exec Executor
+	exec executor.Executor
 }
 
-func newSimpleCli(in io.Reader, out io.Writer, exec Executor) *simpleCli {
+func newSimpleCli(ctx context.Context, in io.Reader, out io.Writer, exec executor.Executor) *simpleCli {
 	return &simpleCli{
+		ctx:     ctx,
 		in:      in,
 		out:     out,
 		scanner: bufio.NewScanner(in),
@@ -30,45 +35,75 @@ func newSimpleCli(in io.Reader, out io.Writer, exec Executor) *simpleCli {
 }
 
 func (c *simpleCli) Start() {
-	for !c.closed {
-		_, _ = fmt.Fprint(c.out, "$ ")
-		if !c.scanner.Scan() {
-			break
-		}
+	lines := make(chan string)
 
-		c.handleCommand(c.scanner.Text())
+	// read from the cli scanner
+	go func() {
+		for c.scanner.Scan() {
+			lines <- c.scanner.Text()
+		}
+	}()
 
-		_, _ = fmt.Fprintln(c.out, "")
+	for {
+		_, _ = fmt.Fprint(c.out, "$ ")
+		select {
+		case line := <-lines:
+			c.handleInput(line)
+		case <-c.ctx.Done():
+			return
+		}
 	}
 }
 
-func (c *simpleCli) handleCommand(command string) {
-	switch command {
+func (c *simpleCli) handleInput(input string) {
+	switch input {
 	case "help", "h", "?", "\\?":
-		fmt.Print("Available Commands:\n// TODO")
-		return
+		c.printHelp()
 	case "q", "exit", "\\q":
-		fmt.Print("Bye!")
-		return
+		c.quit()
+	default:
+		c.handleSQL(input)
 	}
+}
 
-	parser := parser.New(command)
-	for {
+func (c *simpleCli) printHelp() {
+	_, _ = fmt.Fprintln(c.out, "Available Commands:\n// TODO")
+}
+
+func (c *simpleCli) quit() {
+	_, _ = fmt.Fprintln(c.out, "Bye!")
+}
+
+func (c *simpleCli) handleSQL(sqlInput string) {
+	parser := parser.New(sqlInput)
+	for c.ctx.Err() == nil {
+		// parse the input statement
 		stmt, errs, ok := parser.Next()
 		if !ok {
 			break
 		}
+		// print errors to the output
 		for _, err := range errs {
 			_, _ = fmt.Fprintf(c.out, "error while parsing command: %v\n", err)
 		}
-		// TODO: define intermediary representation, convert and then execute
-		if err := c.exec.Execute(stmt); err != nil {
+		// if there were errors, abandon the statement
+		if len(errs) > 0 {
+			_, _ = fmt.Fprintf(c.out, "will skip statement, because there were %d parse errors\n", len(errs))
+			continue
+		}
+		// convert AST to IR
+		command, err := command.From(stmt)
+		if err != nil {
+			_, _ = fmt.Fprintf(c.out, "error while compiling command: %v\n", err)
+			continue
+		}
+		// execute the command
+		result, err := c.exec.Execute(command)
+		if err != nil {
 			_, _ = fmt.Fprintf(c.out, "error while executing command: %v\n", err)
+			continue
 		}
+		// print the result of the command execution to the output
+		_, _ = fmt.Fprintln(c.out, result)
 	}
 }
-
-func (c *simpleCli) Close() error {
-	c.closed = true
-	return nil
-}
diff --git a/column.go b/internal/database/column.go
similarity index 97%
rename from column.go
rename to internal/database/column.go
index 51c13bd..ce221f6 100644
--- a/column.go
+++ b/internal/database/column.go
@@ -1,4 +1,4 @@
-package lbadd
+package database
 
 // The set of data types that are can be used for columns
 type columnType int
diff --git a/db.go b/internal/database/database.go
similarity index 64%
rename from db.go
rename to internal/database/database.go
index aad31de..3576e3a 100644
--- a/db.go
+++ b/internal/database/database.go
@@ -1,8 +1,10 @@
-package lbadd
+package database
+
+import "github.com/tomarrell/lbadd/internal/database/storage"
 
 type table struct {
 	name    string
-	store   storage
+	store   storage.Storage
 	columns []column
 }
 
diff --git a/internal/database/doc.go b/internal/database/doc.go
new file mode 100644
index 0000000..2d17d0e
--- /dev/null
+++ b/internal/database/doc.go
@@ -0,0 +1,3 @@
+// Package database implements the database structure. Every method and function
+// defined in here works with arguments instead of an SQL statement.
+package database
diff --git a/btree.go b/internal/database/storage/btree/btree.go
similarity index 93%
rename from btree.go
rename to internal/database/storage/btree/btree.go
index eca7fb7..e8289bb 100644
--- a/btree.go
+++ b/internal/database/storage/btree/btree.go
@@ -1,19 +1,9 @@
-// Btree contains the btree struct, which is used as the primary data store of
-// the database.
-//
-// The btree supports 3 primary operations:
-// - get: given a key, retrieve the corresponding entry
-// - put: given a key and a value, create an entry in the btree
-// - remove: given a key, remove the corresponding entry in the tree if it
-// exists
-
-package lbadd
+package btree
 
 const defaultOrder = 3
 
-// storage defines the interface to be implemented by
-// the b-tree
-type storage interface {
+// Btree describes a btree.
+type Btree interface {
 	get(k key) (v *entry, exists bool)
 	insert(k key, v value)
 	remove(k key) (removed bool)
diff --git a/internal/database/storage/btree/doc.go b/internal/database/storage/btree/doc.go
new file mode 100644
index 0000000..f340118
--- /dev/null
+++ b/internal/database/storage/btree/doc.go
@@ -0,0 +1,9 @@
+// Package btree contains the btree struct, which is used as the primary data store of
+// the database.
+//
+// The btree supports 3 primary operations:
+// - get: given a key, retrieve the corresponding entry
+// - put: given a key and a value, create an entry in the btree
+// - remove: given a key, remove the corresponding entry in the tree if it
+// exists
+package btree
diff --git a/internal/database/storage/doc.go b/internal/database/storage/doc.go
new file mode 100644
index 0000000..597379e
--- /dev/null
+++ b/internal/database/storage/doc.go
@@ -0,0 +1,2 @@
+// Package storage does something. TODO(tomarrell) create doc.
+package storage
\ No newline at end of file
diff --git a/internal/database/storage/storage.go b/internal/database/storage/storage.go
new file mode 100644
index 0000000..4ec35fe
--- /dev/null
+++ b/internal/database/storage/storage.go
@@ -0,0 +1,4 @@
+package storage
+
+// Storage describes a storage component.
+type Storage interface{}
diff --git a/internal/executor/command/command.go b/internal/executor/command/command.go
new file mode 100644
index 0000000..3ecb287
--- /dev/null
+++ b/internal/executor/command/command.go
@@ -0,0 +1,13 @@
+package command
+
+import "github.com/tomarrell/lbadd/internal/parser/ast"
+
+// Command is the intermediate representation (IR) of an SQL ast.
+type Command struct {
+}
+
+// From converts the given (*ast.SQLStmt) to the IR, which is a
+// (command.Command).
+func From(stmt *ast.SQLStmt) (Command, error) {
+	return Command{}, nil
+}
diff --git a/internal/executor/command/doc.go b/internal/executor/command/doc.go
new file mode 100644
index 0000000..508b66c
--- /dev/null
+++ b/internal/executor/command/doc.go
@@ -0,0 +1,3 @@
+// Package command defined a command model, known as the intermediary
+// representation. It can be converted from an *ast.SQLStmt.
+package command
diff --git a/internal/executor/doc.go b/internal/executor/doc.go
new file mode 100644
index 0000000..0cda51b
--- /dev/null
+++ b/internal/executor/doc.go
@@ -0,0 +1,3 @@
+// Package executor implements executors, that can execute a command. The
+// command is converted from an *ast.SQLStmt.
+package executor
diff --git a/internal/executor/executor.go b/internal/executor/executor.go
new file mode 100644
index 0000000..5902d3f
--- /dev/null
+++ b/internal/executor/executor.go
@@ -0,0 +1,20 @@
+package executor
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/tomarrell/lbadd/internal/executor/command"
+)
+
+// Executor describes a component that can execute a command. A command is the
+// intermediate representation of an SQL statement, meaning that it has been
+// parsed.
+type Executor interface {
+	// Execute executes a command. The result of the computation is returned
+	// together with an error, if one occurred.
+	Execute(command.Command) (Result, error)
+}
+
+// New creates a new, ready to use Executor.
+func New(log zerolog.Logger) Executor {
+	return newSimpleExecutor(log)
+}
diff --git a/internal/executor/result.go b/internal/executor/result.go
new file mode 100644
index 0000000..bdb2c5e
--- /dev/null
+++ b/internal/executor/result.go
@@ -0,0 +1,11 @@
+package executor
+
+import "fmt"
+
+// Result describes the result of a command execution. The result is always a
+// table that has a header row. The smallest possible result table is a table
+// with one column and two rows, and is generated as a result of a single-value
+// computation, e.g. a sum().
+type Result interface {
+	fmt.Stringer
+}
diff --git a/internal/executor/simple_executor.go b/internal/executor/simple_executor.go
new file mode 100644
index 0000000..d6292f8
--- /dev/null
+++ b/internal/executor/simple_executor.go
@@ -0,0 +1,24 @@
+package executor
+
+import (
+	"fmt"
+
+	"github.com/rs/zerolog"
+	"github.com/tomarrell/lbadd/internal/executor/command"
+)
+
+var _ Executor = (*simpleExecutor)(nil)
+
+type simpleExecutor struct {
+	log zerolog.Logger
+}
+
+func newSimpleExecutor(log zerolog.Logger) *simpleExecutor {
+	return &simpleExecutor{
+		log: log,
+	}
+}
+
+func (e *simpleExecutor) Execute(cmd command.Command) (Result, error) {
+	return nil, fmt.Errorf("unimplemented")
+}
diff --git a/internal/parser/ast/statement.go b/internal/parser/ast/statement.go
index 95dafa0..f326913 100644
--- a/internal/parser/ast/statement.go
+++ b/internal/parser/ast/statement.go
@@ -761,6 +761,7 @@ type (
 	// ColumnDef as in the SQLite grammar.
 	ColumnDef struct {
 		ColumnName       token.Token
+		TypeName         *TypeName
 		ColumnConstraint []*ColumnConstraint
 	}
 
@@ -787,6 +788,11 @@ type (
 		Collate          token.Token
 		CollationName    token.Token
 		ForeignKeyClause *ForeignKeyClause
+		Generated        token.Token
+		Always           token.Token
+		As               token.Token
+		Stored           token.Token
+		Virtual          token.Token
 	}
 
 	// ColumnNameList as in the SQLite grammar.
@@ -811,12 +817,18 @@ type (
 	TypeName struct {
 		Name          []token.Token
 		LeftParen     token.Token
-		SignedNumber1 token.Token
+		SignedNumber1 *SignedNumber
 		Comma         token.Token
-		SignedNumber2 token.Token
+		SignedNumber2 *SignedNumber
 		RightParen    token.Token
 	}
 
+	// SignedNumber as in the SQLite grammar.
+	SignedNumber struct {
+		Sign           token.Token
+		NumericLiteral token.Token
+	}
+
 	// IndexedColumn as in the SQLite grammar.
 	IndexedColumn struct {
 		ColumnName    token.Token
diff --git a/internal/parser/ast/tool/cmp/cmp.go b/internal/parser/ast/tool/cmp/cmp.go
new file mode 100644
index 0000000..baf66f5
--- /dev/null
+++ b/internal/parser/ast/tool/cmp/cmp.go
@@ -0,0 +1,170 @@
+package cmp
+
+import (
+	"strings"
+
+	"github.com/tomarrell/lbadd/internal/parser/ast"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+//go:generate stringer -type=DeltaType
+
+// DeltaType describes the type of a delta, for example Nilness or TokenValue.
+type DeltaType uint16
+
+const (
+	// Unknown means, that the type in the delta has not been set properly. This
+	// must never be used. If you encounter this as a value, a developer has
+	// made an error. Please open an issue in that case.
+	Unknown DeltaType = iota
+	// Nilness describes, that two components in the same position (in the AST)
+	// have different nilness, i.e. one component is nil, the other is not.
+	Nilness
+	// TokenValue describes, that two tokens in the same position (in the AST)
+	// have different values.
+	TokenValue
+	// TokenPosition describes, that two tokens in the same position (in the
+	// AST) have different location information.
+	TokenPosition
+)
+
+// Delta describes a single difference between two components of an AST.
+type Delta struct {
+	// Path indicates the path within the struct, where the delta is present,
+	// for example:
+	//
+	//  SQLStmt.AlterTableStmt.ColumnDef.ColumnName
+	//
+	// This would indicate, that the delta describes a difference in the column
+	// name of the alter table statement of the sql statement. Effectively, this
+	// means, that the tokens in ColumnName in the first and second SQLStmt
+	// differ. See the message and the type (Typ) for additional difference
+	// information.
+	Path string
+	// Typ describes the type of this delta. For information on what type means
+	// what exactly, please refer to the documentation of the respective type.
+	Typ DeltaType
+	// Message contains a detailed description of what the delta describes, for
+	// example, if the difference is in a token value, or nilness of a member.
+	// This message is intended to be human readable, not machine processable.
+	Message string
+	// Left is the left hand side component of this delta.
+	Left interface{}
+	// Right is the right hand side component of this delta.
+	Right interface{}
+}
+
+// CompareAST compares two ASTs (specifically, two (*ast.SQLStmt)s) against each
+// other, and returns a list of deltas, which will be nil if the ASTs are equal.
+func CompareAST(left, right *ast.SQLStmt) (deltas []Delta) {
+	return compareSQLStmt(left, right, append(path{}, "SQLStmt"))
+}
+
+type path []string
+
+func (p path) String() string { return strings.Join(p, ".") }
+
+func compareSQLStmt(left, right *ast.SQLStmt, path path) (deltas []Delta) {
+	deltas = append(deltas, compareTokens(left.Explain, right.Explain, append(path, "Explain"))...)
+	deltas = append(deltas, compareTokens(left.Query, right.Query, append(path, "Query"))...)
+	deltas = append(deltas, compareTokens(left.Plan, right.Plan, append(path, "Plan"))...)
+	deltas = append(deltas, compareAlterTableStmt(left.AlterTableStmt, right.AlterTableStmt, append(path, "AlterTableStmt"))...)
+	// TODO(TimSatke) all other fields
+	return
+}
+
+func compareAlterTableStmt(left, right *ast.AlterTableStmt, path path) (deltas []Delta) {
+	deltas = append(deltas, compareTokens(left.Alter, right.Alter, append(path, "Alter"))...)
+	deltas = append(deltas, compareTokens(left.Table, right.Table, append(path, "Table"))...)
+	deltas = append(deltas, compareTokens(left.SchemaName, right.SchemaName, append(path, "SchemaName"))...)
+	deltas = append(deltas, compareTokens(left.Period, right.Period, append(path, "Period"))...)
+	deltas = append(deltas, compareTokens(left.TableName, right.TableName, append(path, "TableName"))...)
+	deltas = append(deltas, compareTokens(left.Rename, right.Rename, append(path, "Rename"))...)
+	deltas = append(deltas, compareTokens(left.To, right.To, append(path, "To"))...)
+	deltas = append(deltas, compareTokens(left.NewTableName, right.NewTableName, append(path, "NewTableName"))...)
+	deltas = append(deltas, compareTokens(left.Column, right.Column, append(path, "Column"))...)
+	deltas = append(deltas, compareTokens(left.ColumnName, right.ColumnName, append(path, "ColumnName"))...)
+	deltas = append(deltas, compareTokens(left.NewColumnName, right.NewColumnName, append(path, "NewColumnName"))...)
+	deltas = append(deltas, compareTokens(left.Add, right.Add, append(path, "Add"))...)
+	// TODO(TimSatke) compare column def
+	return
+}
+
+func compareTokens(left, right token.Token, path path) (deltas []Delta) {
+	if (left == nil && right != nil) ||
+		(left != nil && right == nil) {
+		deltas = append(deltas, Delta{
+			Path:    path.String(),
+			Typ:     Nilness,
+			Message: "one token was nil while the other one wasn't",
+			Left:    left,
+			Right:   right,
+		})
+	}
+
+	if left == right {
+		return
+	}
+
+	if left.Line() != right.Line() {
+		deltas = append(deltas, Delta{
+			Path:    path.String(),
+			Typ:     TokenPosition,
+			Message: "lines don't match",
+			Left:    left.Line(),
+			Right:   right.Line(),
+		})
+	}
+
+	if left.Col() != right.Col() {
+		deltas = append(deltas, Delta{
+			Path:    path.String(),
+			Typ:     TokenPosition,
+			Message: "cols don't match",
+			Left:    left.Col(),
+			Right:   right.Col(),
+		})
+	}
+
+	if left.Offset() != right.Offset() {
+		deltas = append(deltas, Delta{
+			Path:    path.String(),
+			Typ:     TokenPosition,
+			Message: "offsets don't match",
+			Left:    left.Offset(),
+			Right:   right.Offset(),
+		})
+	}
+
+	if left.Length() != right.Length() {
+		deltas = append(deltas, Delta{
+			Path:    path.String(),
+			Typ:     TokenValue,
+			Message: "lengths don't match",
+			Left:    left.Length(),
+			Right:   right.Length(),
+		})
+	}
+
+	if left.Type() != right.Type() {
+		deltas = append(deltas, Delta{
+			Path:    path.String(),
+			Typ:     TokenValue,
+			Message: "types don't match",
+			Left:    left.Type(),
+			Right:   right.Type(),
+		})
+	}
+
+	if left.Value() != right.Value() {
+		deltas = append(deltas, Delta{
+			Path:    path.String(),
+			Typ:     TokenValue,
+			Message: "values don't match",
+			Left:    left.Value(),
+			Right:   right.Value(),
+		})
+	}
+
+	return
+}
diff --git a/internal/parser/ast/tool/cmp/deltatype_string.go b/internal/parser/ast/tool/cmp/deltatype_string.go
new file mode 100644
index 0000000..7344e62
--- /dev/null
+++ b/internal/parser/ast/tool/cmp/deltatype_string.go
@@ -0,0 +1,26 @@
+// Code generated by "stringer -type=DeltaType"; DO NOT EDIT.
+
+package cmp
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Unknown-0]
+	_ = x[Nilness-1]
+	_ = x[TokenValue-2]
+	_ = x[TokenPosition-3]
+}
+
+const _DeltaType_name = "UnknownNilnessTokenValueTokenPosition"
+
+var _DeltaType_index = [...]uint8{0, 7, 14, 24, 37}
+
+func (i DeltaType) String() string {
+	if i >= DeltaType(len(_DeltaType_index)-1) {
+		return "DeltaType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _DeltaType_name[_DeltaType_index[i]:_DeltaType_index[i+1]]
+}
diff --git a/internal/parser/ast/tool/cmp/doc.go b/internal/parser/ast/tool/cmp/doc.go
new file mode 100644
index 0000000..08dde96
--- /dev/null
+++ b/internal/parser/ast/tool/cmp/doc.go
@@ -0,0 +1,4 @@
+// Package cmp implements a function (cmp.Diff), which compares two
+// (*ast.SQLStmt)s and returns a list of differences in the form of
+// (cmp.Delta)s.
+package cmp
diff --git a/internal/parser/parser.go b/internal/parser/parser.go
index 7842165..e1bb3cf 100644
--- a/internal/parser/parser.go
+++ b/internal/parser/parser.go
@@ -4,17 +4,19 @@ import (
 	"github.com/tomarrell/lbadd/internal/parser/ast"
 )
 
-type sentinel string
+// Error allows constant errors.
+type Error string
 
-func (s sentinel) Error() string { return string(s) }
+func (s Error) Error() string { return string(s) }
 
 // parser errors
 const (
-	ErrIncompleteStatement  = sentinel("incomplete statement")
-	ErrPrematureEOF         = sentinel("unexpectedly reached EOF")
-	ErrUnexpectedToken      = sentinel("unexpected token")
-	ErrUnknownToken         = sentinel("unknown token")
-	ErrUnsupportedConstruct = sentinel("unsupported construct")
+	ErrIncompleteStatement  = Error("incomplete statement")
+	ErrPrematureEOF         = Error("unexpectedly reached EOF")
+	ErrScanner              = Error("scanner")
+	ErrUnexpectedToken      = Error("unexpected token")
+	ErrUnknownToken         = Error("unknown token")
+	ErrUnsupportedConstruct = Error("unsupported construct")
 )
 
 // Parser describes a parser that returns (maybe multiple) SQLStatements from a
@@ -35,3 +37,9 @@ type Parser interface {
 	// stmt=nil, errs=nil, ok=false.
 	Next() (stmt *ast.SQLStmt, errs []error, ok bool)
 }
+
+// New creates a new, ready to use parser, that will parse the given input
+// string.
+func New(input string) Parser {
+	return NewSimpleParser(input)
+}
diff --git a/internal/parser/scanner/rule_based_scanner.go b/internal/parser/scanner/rule_based_scanner.go
new file mode 100644
index 0000000..4c44692
--- /dev/null
+++ b/internal/parser/scanner/rule_based_scanner.go
@@ -0,0 +1,161 @@
+package scanner
+
+import (
+	"fmt"
+
+	"github.com/tomarrell/lbadd/internal/parser/scanner/ruleset"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+var _ Scanner = (*ruleBasedScanner)(nil)
+
+type ruleBasedScanner struct {
+	input []rune
+
+	cache token.Token
+
+	whitespaceDetector ruleset.DetectorFunc
+	linefeedDetector   ruleset.DetectorFunc
+	rules              []ruleset.Rule
+
+	state
+}
+
+type state struct {
+	start     int
+	startLine int
+	startCol  int
+	pos       int
+	line      int
+	col       int
+}
+
+// NewRuleBased creates a new, ready to use rule based scanner with the given
+// ruleset, that will process the given input rune slice.
+func NewRuleBased(input []rune, ruleset ruleset.Ruleset) Scanner {
+	return &ruleBasedScanner{
+		input:              input,
+		cache:              nil,
+		whitespaceDetector: ruleset.WhitespaceDetector,
+		linefeedDetector:   ruleset.LinefeedDetector,
+		rules:              ruleset.Rules,
+		state: state{
+			start:     0,
+			startLine: 1,
+			startCol:  1,
+			pos:       0,
+			line:      1,
+			col:       1,
+		},
+	}
+}
+
+func (s *ruleBasedScanner) Next() token.Token {
+	tok := s.Peek()
+	s.cache = nil
+	return tok
+}
+
+func (s *ruleBasedScanner) Peek() token.Token {
+	if s.cache == nil {
+		s.cache = s.computeNext()
+	}
+	return s.cache
+}
+
+func (s *ruleBasedScanner) checkpoint() state {
+	return s.state
+}
+
+func (s *ruleBasedScanner) restore(chck state) {
+	s.state = chck
+}
+
+func (s *ruleBasedScanner) done() bool {
+	return s.pos >= len(s.input)
+}
+
+func (s *ruleBasedScanner) computeNext() token.Token {
+	if s.done() {
+		return s.eof()
+	}
+
+	s.drainWhitespace()
+	return s.applyRule()
+}
+
+func (s *ruleBasedScanner) applyRule() token.Token {
+	// try to apply all rules in the given order
+	for _, rule := range s.rules {
+		chck := s.checkpoint()
+		typ, ok := rule.Apply(s)
+		if ok {
+			return s.token(typ)
+		}
+		s.restore(chck)
+	}
+
+	// no rules matched, create an error token
+	s.ConsumeRune() // skip the one offending rune
+	return s.unexpectedToken()
+}
+
+func (s *ruleBasedScanner) drainWhitespace() {
+	for {
+		next, ok := s.Lookahead()
+		if !(ok && s.whitespaceDetector(next)) {
+			break
+		}
+		s.ConsumeRune()
+	}
+	_ = s.token(token.Unknown) // discard consumed tokens
+}
+
+func (s *ruleBasedScanner) candidate() string {
+	return string(s.input[s.start:s.pos])
+}
+
+func (s *ruleBasedScanner) eof() token.Token {
+	return s.token(token.EOF)
+}
+
+func (s *ruleBasedScanner) unexpectedToken() token.Token {
+	return s.errorToken(fmt.Errorf("%w: '%v' at offset %v", ErrUnexpectedToken, s.candidate(), s.start))
+}
+
+func (s *ruleBasedScanner) token(t token.Type) token.Token {
+	tok := token.New(s.startLine, s.startCol, s.start, s.pos-s.start, t, s.candidate())
+	s.updateStartPositions()
+	return tok
+}
+
+func (s *ruleBasedScanner) errorToken(err error) token.Token {
+	tok := token.NewErrorToken(s.startLine, s.startCol, s.start, s.pos-s.start, token.Error, err)
+	s.updateStartPositions()
+	return tok
+}
+
+func (s *ruleBasedScanner) updateStartPositions() {
+	s.start = s.pos
+	s.startLine = s.line
+	s.startCol = s.col
+}
+
+// runeScanner
+
+func (s *ruleBasedScanner) Lookahead() (rune, bool) {
+	if !s.done() {
+		return s.input[s.pos], true
+	}
+	return 0, false
+}
+
+func (s *ruleBasedScanner) ConsumeRune() {
+	if s.linefeedDetector(s.input[s.pos]) {
+		s.line++
+		s.col = 1
+	} else {
+		s.col++
+	}
+	s.pos++
+}
diff --git a/internal/parser/scanner/ruleset/doc.go b/internal/parser/scanner/ruleset/doc.go
new file mode 100644
index 0000000..0252580
--- /dev/null
+++ b/internal/parser/scanner/ruleset/doc.go
@@ -0,0 +1,3 @@
+// Package ruleset implements rules that can be used in combination with a rule
+// based scanner.
+package ruleset
diff --git a/internal/parser/scanner/ruleset/ruleset.go b/internal/parser/scanner/ruleset/ruleset.go
new file mode 100644
index 0000000..6e3b512
--- /dev/null
+++ b/internal/parser/scanner/ruleset/ruleset.go
@@ -0,0 +1,69 @@
+package ruleset
+
+import (
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+// DetectorFunc is a function that can detect if a given rune has a
+// implementation specific attribute.
+type DetectorFunc func(rune) bool
+
+// Ruleset is a collection of a whitespace detector, a linefeed detector and a
+// slice of rules. A rule based scanner can work with this to create tokens from
+// the given rules.
+type Ruleset struct {
+	WhitespaceDetector DetectorFunc
+	LinefeedDetector   DetectorFunc
+	Rules              []Rule
+}
+
+// Rule describes a single scanner rule that can theoretically be applied. If
+// the rule is applicable, is decided by the rule itself. If it returns
+// applicable=false, the it is the rule based scanner's responsibility, to reset
+// its position information to where it was, before the rule was applied.
+//
+// A rule takes a RuneScanner as parameter, which can be used to evaluate runes
+// and advance the position pointer of the rule based scanner. After consuming n
+// matching runes, return a token.Type. The rule based scanner will create a
+// token from its position information, the consumed runes and the returned
+// token.Type.
+//
+//  input := "hello"
+//  ...
+//  func (r myRule) Apply(s RuneScanner) (token.Type, bool) {
+//      r.ConsumeRune()
+//      r.ConsumeRune()
+//      r.ConsumeRune()
+//      r.ConsumeRune()
+//      r.ConsumeRune()
+//      return token.Literal, true
+//  }
+//
+// The above example will cause the rule based scanner to emit a token with
+// length=5, value=hello, offset=0.
+//
+// To use a func(RuneScanner) (token.Type, bool) as Rule, see ruleset.FuncRule.
+type Rule interface {
+	// Apply tries to apply this rule to the given RuneScanner. If the rule
+	// determines itself as not applicable to the current RuneScanner, return
+	// applicable=false, and the rule based scanner will handle everything else.
+	Apply(RuneScanner) (token token.Type, applicable bool)
+}
+
+// FuncRule is an type alias that works as implementation for a ruleset.Rule.
+//
+//  func myRule(s RuneScanner) (token.Type, bool) { ... }
+//  var _ Rule = FuncRule(myRule)
+type FuncRule func(RuneScanner) (token.Type, bool)
+
+// Apply implements (ruleset.Rule).Apply.
+func (r FuncRule) Apply(s RuneScanner) (token.Type, bool) { return r(s) }
+
+// RuneScanner is a capsule that limits the interaction of the scanner to a
+// subset of two core functions. It is not guaranteed that the value in this
+// interface will always be of the same type. This implies, that you shouldn't
+// place type assertions on this interface.
+type RuneScanner interface {
+	Lookahead() (rune, bool)
+	ConsumeRune()
+}
diff --git a/internal/parser/scanner/ruleset/ruleset_default.go b/internal/parser/scanner/ruleset/ruleset_default.go
new file mode 100644
index 0000000..ef3e7c8
--- /dev/null
+++ b/internal/parser/scanner/ruleset/ruleset_default.go
@@ -0,0 +1,188 @@
+package ruleset
+
+import (
+	"bytes"
+	"unicode"
+
+	"github.com/tomarrell/lbadd/internal/parser/scanner/matcher"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+var (
+	// Default is the ruleset that this application uses by default. The rules
+	// are inspired by Sqlite, however they do not care as much about
+	// compatibility with other database systems, and are therefore simpler to
+	// read and write.
+	Default = Ruleset{
+		WhitespaceDetector: defaultWhitespaceDetector.Matches,
+		LinefeedDetector:   defaultLinefeedDetector.Matches,
+		Rules:              defaultRules,
+	}
+	// defaultWhitespaceDetector matches all the the whitespaces that this ruleset allows
+	defaultWhitespaceDetector = matcher.New("whitespace", unicode.Space)
+	// defaultLinefeedDetector is the linefeed detector that this ruleset allows
+	defaultLinefeedDetector   = matcher.RuneWithDesc("linefeed", '\n')
+	defaultStatementSeparator = matcher.RuneWithDesc("statement separator", ';')
+	// defaultLiteral matches the allowed letters of a literal
+	defaultLiteral = matcher.Merge(
+		matcher.New("Upper", unicode.Upper),
+		matcher.New("Lower", unicode.Lower),
+		matcher.New("Title", unicode.Title),
+		matcher.New("Number", unicode.Number),
+	)
+	defaultQuote          = matcher.String("'\"")
+	defaultUnaryOperator  = matcher.String("-+~")
+	defaultBinaryOperator = matcher.String("|*/%<>=&!")
+	defaultDelimiter      = matcher.String("(),")
+	defaultRules          = []Rule{
+		FuncRule(defaultStatementSeparatorRule),
+		FuncRule(defaultKeywordsRule),
+		FuncRule(defaultUnaryOperatorRule),
+		FuncRule(defaultBinaryOperatorRule),
+		FuncRule(defaultDelimiterRule),
+		FuncRule(defaultQuotedLiteralRule),
+		FuncRule(defaultUnquotedLiteralRule),
+	}
+	defaultKeywords = map[string]token.Type{"ABORT": token.KeywordAbort, "ACTION": token.KeywordAction, "ADD": token.KeywordAdd, "AFTER": token.KeywordAdd, "ALL": token.KeywordAll, "ALTER": token.KeywordAlter, "ANALYZE": token.KeywordAnalyze, "AND": token.KeywordAnd, "AS": token.KeywordAnd, "ASC": token.KeywordAsc, "ATTACH": token.KeywordAttach, "AUTOINCREMENT": token.KeywordAutoincrement, "BEFORE": token.KeywordBefore, "BEGIN": token.KeywordBegin, "BETWEEN": token.KeywordBetween, "BY": token.KeywordBy, "CASCADE": token.KeywordCascade, "CASE": token.KeywordCase, "CAST": token.KeywordCast, "CHECK": token.KeywordCheck, "COLLATE": token.KeywordCollate, "COLUMN": token.KeywordColumn, "COMMIT": token.KeywordCommit, "CONFLICT": token.KeywordConflict, "CONSTRAINT": token.KeywordConstraint, "CREATE": token.KeywordCreate, "CROSS": token.KeywordCross, "CURRENT": token.KeywordCurrent, "CURRENT_DATE": token.KeywordCurrentDate, "CURRENT_TIME": token.KeywordCurrentTime, "CURRENT_TIMESTAMP": token.KeywordCurrentTimestamp, "DATABASE": token.KeywordDatabase, "DEFAULT": token.KeywordDefault, "DEFERRABLE": token.KeywordDeferrable, "DEFERRED": token.KeywordDeferred, "DELETE": token.KeywordDelete, "DESC": token.KeywordDesc, "DETACH": token.KeywordDetach, "DISTINCT": token.KeywordDistinct, "DO": token.KeywordDo, "DROP": token.KeywordDrop, "EACH": token.KeywordEach, "ELSE": token.KeywordElse, "END": token.KeywordEnd, "ESCAPE": token.KeywordEscape, "EXCEPT": token.KeywordExcept, "EXCLUDE": token.KeywordExclude, "EXCLUSIVE": token.KeywordExclusive, "EXISTS": token.KeywordExists, "EXPLAIN": token.KeywordExplain, "FAIL": token.KeywordFail, "FILTER": token.KeywordFilter, "FIRST": token.KeywordFirst, "FOLLOWING": token.KeywordFollowing, "FOR": token.KeywordFor, "FOREIGN": token.KeywordForeign, "FROM": token.KeywordFrom, "FULL": token.KeywordFull, "GLOB": token.KeywordGlob, "GROUP": token.KeywordGroup, "GROUPS": token.KeywordGroups, "HAVING": token.KeywordHaving, "IF": token.KeywordIf, "IGNORE": token.KeywordIgnore, "IMMEDIATE": token.KeywordImmediate, "IN": token.KeywordIn, "INDEX": token.KeywordIndex, "INDEXED": token.KeywordIndexed, "INITIALLY": token.KeywordInitially, "INNER": token.KeywordInner, "INSERT": token.KeywordInsert, "INSTEAD": token.KeywordInstead, "INTERSECT": token.KeywordIntersect, "INTO": token.KeywordInto, "IS": token.KeywordIs, "ISNULL": token.KeywordIsnull, "JOIN": token.KeywordJoin, "KEY": token.KeywordKey, "LAST": token.KeywordLast, "LEFT": token.KeywordLeft, "LIKE": token.KeywordLike, "LIMIT": token.KeywordLimit, "MATCH": token.KeywordMatch, "NATURAL": token.KeywordNatural, "NO": token.KeywordNo, "NOT": token.KeywordNot, "NOTHING": token.KeywordNothing, "NOTNULL": token.KeywordNotnull, "NULL": token.KeywordNull, "OF": token.KeywordOf, "OFFSET": token.KeywordOffset, "ON": token.KeywordOn, "OR": token.KeywordOr, "ORDER": token.KeywordOrder, "OTHERS": token.KeywordOthers, "OUTER": token.KeywordOuter, "OVER": token.KeywordOver, "PARTITION": token.KeywordPartition, "PLAN": token.KeywordPlan, "PRAGMA": token.KeywordPragma, "PRECEDING": token.KeywordPreceding, "PRIMARY": token.KeywordPrimary, "QUERY": token.KeywordQuery, "RAISE": token.KeywordRaise, "RANGE": token.KeywordRange, "RECURSIVE": token.KeywordRecursive, "REFERENCES": token.KeywordReferences, "REGEXP": token.KeywordRegexp, "REINDEX": token.KeywordReindex, "RELEASE": token.KeywordRelease, "RENAME": token.KeywordRename, "REPLACE": token.KeywordReplace, "RESTRICT": token.KeywordRestrict, "RIGHT": token.KeywordRight, "ROLLBACK": token.KeywordRollback, "ROW": token.KeywordRow, "ROWS": token.KeywordRows, "SAVEPOINT": token.KeywordSavepoint, "SELECT": token.KeywordSelect, "SET": token.KeywordSet, "TABLE": token.KeywordTable, "TEMP": token.KeywordTemp, "TEMPORARY": token.KeywordTemporary, "THEN": token.KeywordThen, "TIES": token.KeywordTies, "TO": token.KeywordTo, "TRANSACTION": token.KeywordTransaction, "TRIGGER": token.KeywordTrigger, "UNBOUNDED": token.KeywordUnbounded, "UNION": token.KeywordUnion, "UNIQUE": token.KeywordUnique, "UPDATE": token.KeywordUpdate, "USING": token.KeywordUsing, "VACUUM": token.KeywordVacuum, "VALUES": token.KeywordValues, "VIEW": token.KeywordView, "VIRTUAL": token.KeywordVirtual, "WHEN": token.KeywordWhen, "WHERE": token.KeywordWhere, "WINDOW": token.KeywordWindow, "WITH": token.KeywordWith, "WITHOUT": token.KeywordWithout}
+)
+
+func defaultStatementSeparatorRule(s RuneScanner) (token.Type, bool) {
+	if next, ok := s.Lookahead(); ok && defaultStatementSeparator.Matches(next) {
+		s.ConsumeRune()
+		return token.StatementSeparator, true
+	}
+	return token.Unknown, false
+}
+
+func defaultKeywordsRule(s RuneScanner) (token.Type, bool) {
+	// read word
+	var buf bytes.Buffer
+	for {
+		next, ok := s.Lookahead()
+		if !ok || defaultWhitespaceDetector.Matches(next) {
+			break
+		}
+		_, _ = buf.WriteRune(next)
+		s.ConsumeRune()
+	}
+	candidate := buf.String() // candidate is the next word that may be a keyword
+
+	// check if the candidate is a keyword
+	if typ, ok := defaultKeywords[candidate]; ok {
+		return typ, true
+	}
+	return token.Unknown, false
+}
+
+func defaultUnaryOperatorRule(s RuneScanner) (token.Type, bool) {
+	if next, ok := s.Lookahead(); ok && defaultUnaryOperator.Matches(next) {
+		s.ConsumeRune()
+		return token.UnaryOperator, true
+	}
+	return token.Unknown, false
+}
+
+// defaultBinaryOperatorRule scans binary operators. Since binary opeators
+// can be of length one and two, it first checks for former, then latter
+// and allows the best of the two conditions.
+func defaultBinaryOperatorRule(s RuneScanner) (token.Type, bool) {
+	if first, ok := s.Lookahead(); ok && defaultBinaryOperator.Matches(first) {
+		s.ConsumeRune()
+		if first == '*' || first == '/' || first == '%' || first == '&' {
+			return token.BinaryOperator, true
+		} else if next, ok := s.Lookahead(); ok && defaultBinaryOperator.Matches(next) {
+			switch first {
+			case '|':
+				if next == '|' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '<':
+				if next == '<' || next == '=' || next == '>' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '>':
+				if next == '>' || next == '=' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '=':
+				if next == '=' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+			case '!':
+				if next == '=' {
+					s.ConsumeRune()
+					return token.BinaryOperator, true
+				}
+				// return token.Unknown, false
+			}
+		}
+		// special cases where these operators can be single or can have a suffix
+		// The switch case blocks have been designed such that only the cases where
+		// there's a meaningful operator, the runes are consumed and returned, in cases
+		// where the second operator is not meaningful, the first operator is consumed here.
+		// In cases where the second operator is not meaningful, it'll be processed again.
+		if first == '<' || first == '>' || first == '=' || first == '|' {
+			return token.BinaryOperator, true
+		}
+	}
+	return token.Unknown, false
+}
+
+func defaultDelimiterRule(s RuneScanner) (token.Type, bool) {
+	if next, ok := s.Lookahead(); ok && defaultDelimiter.Matches(next) {
+		s.ConsumeRune()
+		return token.Delimiter, true
+	}
+	return token.Unknown, false
+}
+
+func defaultQuotedLiteralRule(s RuneScanner) (token.Type, bool) {
+	next, ok := s.Lookahead()
+	if !(ok && defaultQuote.Matches(next)) {
+		return token.Unknown, false
+	}
+	quoteType := next
+	s.ConsumeRune()
+
+	for {
+		next, ok := s.Lookahead()
+		if !ok {
+			return token.Unknown, false
+		}
+		if next == '\\' {
+			s.ConsumeRune()
+			next, ok = s.Lookahead()
+			if !ok {
+				return token.Unknown, false
+			}
+			s.ConsumeRune()
+		} else if next == quoteType {
+			break
+		} else {
+			s.ConsumeRune()
+		}
+	}
+	s.ConsumeRune()
+	return token.Literal, true
+}
+
+func defaultUnquotedLiteralRule(s RuneScanner) (token.Type, bool) {
+	if next, ok := s.Lookahead(); !(ok && defaultLiteral.Matches(next)) {
+		return token.Unknown, false
+	}
+	s.ConsumeRune()
+
+	for {
+		next, ok := s.Lookahead()
+		if !(ok && defaultLiteral.Matches(next)) {
+			break
+		}
+		s.ConsumeRune()
+	}
+
+	return token.Literal, true
+}
diff --git a/internal/parser/scanner/scanner.go b/internal/parser/scanner/scanner.go
index bb9cd59..9141e79 100644
--- a/internal/parser/scanner/scanner.go
+++ b/internal/parser/scanner/scanner.go
@@ -1,300 +1,21 @@
 package scanner
 
 import (
-	"fmt"
-
-	"github.com/tomarrell/lbadd/internal/parser/scanner/matcher"
 	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
 )
 
+// Error allows constant errors.
+type Error string
+
+func (s Error) Error() string { return string(s) }
+
+// Constant errors
+const (
+	ErrUnexpectedToken = Error("unexpected token")
+)
+
 // Scanner is the interface that describes a scanner.
 type Scanner interface {
-	HasNext() bool
 	Next() token.Token
 	Peek() token.Token
 }
-
-type scanner struct {
-	input          []rune
-	start          int
-	pos, line, col int
-	closed         bool
-}
-
-// checkpoint represents the state of a scanner at any given point in time. A
-// scanner can be restored to a checkpoint.
-//
-//	var s *Scanner
-//	...
-//	chck := s.checkpoint() // create a checkpoint
-//	...                     // accept(), next(), goback(), whatever
-//	s.restore(chck)        // scanner is in the same state as when the checkpoint was created
-//
-// This is useful when a state should not return an error if something
-// unexpected was read, but for example another grammar production should be
-// tried. To guarantee that both tries start with the same scanner state, a
-// checkpoint can be used.
-type checkpoint struct {
-	start int
-	pos   int
-
-	startLine, startCol int
-	line, lastCol, col  int
-}
-
-// New returns a new scanner
-func New(input []rune) Scanner {
-	return &scanner{
-		input: input,
-		start: 0,
-		pos:   0,
-
-		closed: false,
-	}
-}
-
-// HasNext checks for existance of next token, returns true if exists, false otherwise.
-func (s *scanner) HasNext() bool {
-	if s.done() {
-		return false
-	}
-	return true
-}
-
-// Next reads the next token. This is basically starting from the initial state until a
-// token gets emitted. If an error occurs, simply return an error token.
-func (s *scanner) Next() token.Token {
-	// if s.done() {
-	// 	return token.New(s.line, s.col, s.pos-s.start, s.pos, token.Error, fmt.Sprintf("Scanner closed. Can't read from it."))
-	// }
-	switch s.peek() {
-	case 'A':
-		return scanAKeyword(s)
-	case 'B':
-		return scanBKeyword(s)
-	case 'C':
-		return scanCKeyword(s)
-	case 'D':
-		return scanDKeyword(s)
-	case 'E':
-		return scanEKeyword(s)
-	case 'F':
-		return scanFKeyword(s)
-	case 'G':
-		return scanGKeyword(s)
-	case 'H':
-		return scanHKeyword(s)
-	case 'I':
-		return scanIKeyword(s)
-	case 'J':
-		return scanJKeyword(s)
-	case 'K':
-		return scanKKeyword(s)
-	case 'L':
-		return scanLKeyword(s)
-	case 'M':
-		return scanMKeyword(s)
-	case 'N':
-		return scanNKeyword(s)
-	case 'O':
-		return scanOKeyword(s)
-	case 'P':
-		return scanPKeyword(s)
-	case 'Q':
-		return scanQKeyword(s)
-	case 'R':
-		return scanRKeyword(s)
-	case 'S':
-		return scanSKeyword(s)
-	case 'T':
-		return scanTKeyword(s)
-	case 'U':
-		return scanUKeyword(s)
-	case 'V':
-		return scanVKeyword(s)
-	// case ' ':
-	// 	return scanSpace(s)
-	// case '"':
-	// 	return scanDoubleQuote(s)
-	// case '%':
-	// 	return scanPercent(s)
-	// case '&':
-	// 	return scanAmpersand(s)
-	// case '\'':
-	// 	return scanQuote(s)
-	// case '(':
-	// 	return scanLeftParanthesis(s)
-	// case ')':
-	// 	return scanRightParanthesis(s)
-	// case '*':
-	// 	return scanAsterisk(s)
-	// case '+':
-	// 	return scanPlusSign(s)
-	// case ',':
-	// 	return scanComma(s)
-	// case '-':
-	// 	return scanMinusSign(s)
-	// case '.':
-	// 	return scanPeriod(s)
-	// case '/':
-	// 	return scanSolidus(s)
-	// case '\\':
-	// 	return scanReverseSolidus(s)
-	// case ':':
-	// 	return scanColon(s)
-	// case ';':
-	// 	return scanSemiColon(s)
-	// case '<':
-	// 	return scanLessThanOperator(s)
-	// case '=':
-	// 	return scanEqualsOperator(s)
-	// case '>':
-	// 	return scanGreaterThanOperator(s)
-	// case '?':
-	// 	return scanQuestioMarkOrTrigraphs(s)
-	// case '[':
-	// 	return scanLeftBracket(s)
-	// case ']':
-	// 	return scanRightBracket(s)
-	// case '^':
-	// 	return scanCircumflex(s)
-	// case '_':
-	// 	return scanUnderscore(s)
-	// case '|':
-	// 	return scanVerticalBar(s)
-	// case '{':
-	// 	return scanLeftBrace(s)
-	// case '}':
-	// 	return scanRightBrace(s)
-	// case '$':
-	// 	return scanDollarSign(s)
-	default:
-		fmt.Println("TBI")
-	}
-	return nil
-}
-
-func (s *scanner) Peek() token.Token {
-	panic("implement me")
-}
-
-// Close will cause this scanner to not execute any more states. The execution
-// of the current state cannot be aborted, but the scanner will stop executing
-// states after the current state has finished.
-func (s *scanner) Close() error {
-	s.closed = true
-	return nil
-}
-
-// done determines whether the scanner is done with its work. This is the case,
-// if either the scanner was closed, or the scanner has reached the end of its
-// input.
-func (s *scanner) done() bool {
-	return s.closed ||
-		s.pos >= len(s.input)
-}
-
-// next returns the next rune of the scanners input and advances its pointer by
-// one position. This method also updates the line and col information of the
-// scanner. If the scanner.done()==true and this method is called, it will panic
-// with a syntax error.
-//
-// The process of advancing the pointer and returning the read rune is called
-// "consuming a rune" or "accepting a rune".
-func (s *scanner) next() rune {
-	// get the actual next rune
-	next := s.input[s.pos]
-	if next == '\n' {
-		s.line++
-		s.col = 1
-	}
-	// update current scanner position
-	s.pos++
-
-	return next
-}
-
-// peek returns the next rune of the scanners input without consuming it.
-func (s *scanner) peek() rune {
-	return s.input[s.pos]
-}
-
-// goback decrements the scanner's position by one and updates its line and col
-// information.
-func (s *scanner) goback() {
-	s.pos--
-}
-
-// ignore discards all accepted runes. This is done by simply setting the start
-// position of the last read token to the current scanner position.
-func (s *scanner) ignore() {
-	s.start = s.pos
-}
-
-// accept accepts exactly one rune matched by the given matcher. This means,
-// that: If the next rune is matched by the scanner, it is consumed and ok=true
-// is returned. If the next rune is NOT matched, it is unread and ok=false is
-// returned. This implies, that accept(Alphanumeric) will actually do nothing if
-// the next rune is not Alphanumeric. However, if the next rune is Alphanumeric,
-// it will be accepted.
-func (s *scanner) accept(m matcher.M) bool {
-	if m.Matches(s.next()) {
-		return true
-	}
-	s.goback()
-	return false
-}
-
-// acceptMultiple accepts multiple runes that are matched by the given matcher.
-// See the godoc on (*scanner).accept for more information. The amount of
-// matched runes is returned.
-func (s *scanner) acceptMultiple(m matcher.M) (matched uint) {
-	for s.accept(m) {
-		matched++
-	}
-	return
-}
-
-// acceptString accepts the exact sequence of runes that the given string
-// represents, or does nothing, if the string is not matched.
-//
-//	input := []rune(".hello")
-//	...
-//	s.acceptString("hello") // will do nothing, as the next rune is '.'
-//	s.next()                // advance the position by one (next rune is now 'h')
-//	s.acceptString("hello") // will accept 5 runes, the scanner has reached its EOF now
-func (s *scanner) acceptString(str string) bool {
-	if s.peekString(str) {
-		s.pos += len(str)
-		return true
-	}
-	return false
-}
-
-// peekString works like (*scanner).acceptString, except it doesn't consume any
-// runes. It just peeks, if the given string lays ahead.
-func (s *scanner) peekString(str string) bool {
-	for i, r := range str {
-		if r != s.input[s.pos+i] {
-			return false
-		}
-	}
-	return true
-}
-
-// createToken creates a token with given parameters
-func createToken(line, col, start, pos int, t token.Type, value string, s *scanner) token.Token {
-	token := token.New(line, col, start, pos-start, t, value)
-	s.start = pos
-	return token
-}
-
-// seekNext returns the position of the end of a keyword.
-// It takes the start position of the keyword.
-func (s *scanner) seekNext(start int) int {
-	start++
-	for s.input[start] != ' ' {
-		start++
-	}
-	return start
-}
diff --git a/internal/parser/scanner/states.go b/internal/parser/scanner/states.go
deleted file mode 100644
index 78bec70..0000000
--- a/internal/parser/scanner/states.go
+++ /dev/null
@@ -1,586 +0,0 @@
-package scanner
-
-import (
-	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
-)
-
-var keywordsWithA map[string]token.Type = map[string]token.Type{
-	"ABORT":          token.KeywordAbort,
-	"ACTION":         token.KeywordAction,
-	"ADD":            token.KeywordAdd,
-	"AFTER":          token.KeywordAdd,
-	"ALL":            token.KeywordAll,
-	"ALTER":          token.KeywordAlter,
-	"ANALYZE":        token.KeywordAnalyze,
-	"AND":            token.KeywordAnd,
-	"AS":             token.KeywordAnd,
-	"ASC":            token.KeywordAsc,
-	"ATTACH":         token.KeywordAttach,
-	"AUTO INCREMENT": token.KeywordAutoincrement,
-}
-
-var keywordsWithB map[string]token.Type = map[string]token.Type{
-	"BEFORE":  token.KeywordBefore,
-	"BEGIN":   token.KeywordBegin,
-	"BETWEEN": token.KeywordBetween,
-	"BY":      token.KeywordBy,
-}
-
-var keywordsWithC map[string]token.Type = map[string]token.Type{
-	"CASCADE":           token.KeywordCascade,
-	"CASE":              token.KeywordCase,
-	"CAST":              token.KeywordCast,
-	"CHECK":             token.KeywordCheck,
-	"COLLATE":           token.KeywordCollate,
-	"COLUMN":            token.KeywordColumn,
-	"COMMIT":            token.KeywordCommit,
-	"CONFLICT":          token.KeywordConflict,
-	"CONSTRAINT":        token.KeywordConstraint,
-	"CREATE":            token.KeywordCreate,
-	"CROSS":             token.KeywordCross,
-	"CURRENT":           token.KeywordCurrent,
-	"CURRENT_DATE":      token.KeywordCurrentDate,
-	"CURRENT_TIME":      token.KeywordCurrentTime,
-	"CURRENT_TIMESTAMP": token.KeywordCurrentTimestamp,
-}
-
-var keywordsWithD map[string]token.Type = map[string]token.Type{
-	"DATABASE":   token.KeywordDatabase,
-	"DEFAULT":    token.KeywordDefault,
-	"DEFERRABLE": token.KeywordDeferrable,
-	"DEFERRED":   token.KeywordDeferred,
-	"DELETE":     token.KeywordDelete,
-	"DESC":       token.KeywordDesc,
-	"DETACH":     token.KeywordDetach,
-	"DISTINCT":   token.KeywordDistinct,
-	"DO":         token.KeywordDo,
-	"DROP":       token.KeywordDrop,
-}
-
-var keywordsWithE map[string]token.Type = map[string]token.Type{
-	"EACH":      token.KeywordEach,
-	"ELSE":      token.KeywordElse,
-	"END":       token.KeywordEnd,
-	"ESCAPE":    token.KeywordEscape,
-	"EXCEPT":    token.KeywordExcept,
-	"EXCLUDE":   token.KeywordExclude,
-	"EXCLUSIVE": token.KeywordExclusive,
-	"EXISTS":    token.KeywordExists,
-	"EXPLAIN":   token.KeywordExplain,
-}
-
-var keywordsWithF map[string]token.Type = map[string]token.Type{
-	"FAIL":      token.KeywordFail,
-	"FILTER":    token.KeywordFilter,
-	"FIRST":     token.KeywordFirst,
-	"FOLLOWING": token.KeywordFollowing,
-	"FOR":       token.KeywordFor,
-	"FOREIGN":   token.KeywordForeign,
-	"FROM":      token.KeywordFrom,
-	"FULL":      token.KeywordFull,
-}
-
-var keywordsWithG map[string]token.Type = map[string]token.Type{
-	"GLOB":   token.KeywordGlob,
-	"GROUP":  token.KeywordGroup,
-	"GROUPS": token.KeywordGroups,
-}
-
-var keywordsWithH map[string]token.Type = map[string]token.Type{
-	"HAVING": token.KeywordHaving,
-}
-
-var keywordsWithI map[string]token.Type = map[string]token.Type{
-	"IF":        token.KeywordIf,
-	"IGNORE":    token.KeywordIgnore,
-	"IMMEDIATE": token.KeywordImmediate,
-	"IN":        token.KeywordIn,
-	"INDEX":     token.KeywordIndex,
-	"INDEXED":   token.KeywordIndexed,
-	"INITIALLY": token.KeywordInitially,
-	"INNER":     token.KeywordInner,
-	"INSERT":    token.KeywordInsert,
-	"INSTEAD":   token.KeywordInstead,
-	"INTERSECT": token.KeywordIntersect,
-	"INTO":      token.KeywordInto,
-	"IS":        token.KeywordIs,
-	"ISNULL":    token.KeywordIsnull,
-}
-
-var keywordsWithJ map[string]token.Type = map[string]token.Type{
-	"JOIN": token.KeywordJoin,
-}
-
-var keywordsWithK map[string]token.Type = map[string]token.Type{
-	"KEY": token.KeywordKey,
-}
-
-var keywordsWithL map[string]token.Type = map[string]token.Type{
-	"LAST":  token.KeywordLast,
-	"LEFT":  token.KeywordLeft,
-	"LIKE":  token.KeywordLike,
-	"LIMIT": token.KeywordLimit,
-}
-
-var keywordsWithM map[string]token.Type = map[string]token.Type{
-	"MATCH": token.KeywordMatch,
-}
-
-var keywordsWithN map[string]token.Type = map[string]token.Type{
-	"NATURAL": token.KeywordNatural,
-	"NO":      token.KeywordNo,
-	"NOT":     token.KeywordNot,
-	"NOTHING": token.KeywordNothing,
-	"NOTNULL": token.KeywordNotnull,
-	"NULL":    token.KeywordNull,
-}
-
-var keywordsWithO map[string]token.Type = map[string]token.Type{
-	"OF":     token.KeywordOf,
-	"OFFSET": token.KeywordOffset,
-	"ON":     token.KeywordOn,
-	"OR":     token.KeywordOr,
-	"ORDER":  token.KeywordOrder,
-	"OTHERS": token.KeywordOthers,
-	"OUTER":  token.KeywordOuter,
-	"OVER":   token.KeywordOver,
-}
-
-var keywordsWithP map[string]token.Type = map[string]token.Type{
-	"PARTITION": token.KeywordPartition,
-	"PLAN":      token.KeywordPlan,
-	"PRAGMA":    token.KeywordPragma,
-	"PRECEDING": token.KeywordPreceding,
-	"PRIMARY":   token.KeywordPrimary,
-}
-
-var keywordsWithQ map[string]token.Type = map[string]token.Type{
-	"QUERY": token.KeywordQuery,
-}
-
-var keywordsWithR map[string]token.Type = map[string]token.Type{
-	"RAISE":      token.KeywordRaise,
-	"RANGE":      token.KeywordRange,
-	"RECURSIVE":  token.KeywordRecursive,
-	"REFERENCES": token.KeywordReferences,
-	"REGEXP":     token.KeywordRegexp,
-	"REINDEX":    token.KeywordReindex,
-	"RELEASE":    token.KeywordRelease,
-	"RENAME":     token.KeywordRename,
-	"REPLACE":    token.KeywordReplace,
-	"RESTRICT":   token.KeywordRestrict,
-	"RIGHT":      token.KeywordRight,
-	"ROLLBACK":   token.KeywordRollback,
-	"ROW":        token.KeywordRow,
-	"ROWS":       token.KeywordRows,
-}
-
-var keywordsWithS map[string]token.Type = map[string]token.Type{
-	"SAVEPOINT": token.KeywordSavepoint,
-	"SELECT":    token.KeywordSelect,
-	"SET":       token.KeywordSet,
-}
-
-var keywordsWithT map[string]token.Type = map[string]token.Type{
-	"TABLE":       token.KeywordTable,
-	"TEMP":        token.KeywordTemp,
-	"TEMPORARY":   token.KeywordTemporary,
-	"THEN":        token.KeywordThen,
-	"TIES":        token.KeywordTies,
-	"TO":          token.KeywordTo,
-	"TRANSACTION": token.KeywordTransaction,
-	"TRIGGER":     token.KeywordTrigger,
-}
-
-var keywordsWithU map[string]token.Type = map[string]token.Type{
-	"UNBOUNDED": token.KeywordUnbounded,
-	"UNION":     token.KeywordUnion,
-	"UNIQUE":    token.KeywordUnique,
-	"UPDATE":    token.KeywordUpdate,
-	"USING":     token.KeywordUsing,
-}
-
-var keywordsWithV map[string]token.Type = map[string]token.Type{
-	"VACUUM":  token.KeywordVacuum,
-	"VALUES":  token.KeywordValues,
-	"VIEW":    token.KeywordView,
-	"VIRTUAL": token.KeywordVirtual,
-}
-
-var keywordsWithW map[string]token.Type = map[string]token.Type{
-	"WHEN":    token.KeywordWhen,
-	"WHERE":   token.KeywordWhere,
-	"WINDOW":  token.KeywordWindow,
-	"WITH":    token.KeywordWith,
-	"WITHOUT": token.KeywordWithout,
-}
-
-// func scanSpace(s *scanner) token.Token {
-// 	s.accept(matcher.String(" "))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanDoubleQuote(s *scanner) token.Token {
-// 	s.accept(matcher.String("\""))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanPercent(s *scanner) token.Token {
-// 	s.accept(matcher.String("%"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanAmpersand(s *scanner) token.Token {
-// 	s.accept(matcher.String("&"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanQuote(s *scanner) token.Token {
-// 	s.accept(matcher.String("'"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanLeftParanthesis(s *scanner) token.Token {
-// 	s.accept(matcher.String("("))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanRightParanthesis(s *scanner) token.Token {
-// 	s.accept(matcher.String(")"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanAsterisk(s *scanner) token.Token {
-// 	s.accept(matcher.String("*"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanPlusSign(s *scanner) token.Token {
-// 	s.accept(matcher.String("+"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanComma(s *scanner) token.Token {
-// 	s.accept(matcher.String(":true,"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanMinusSign(s *scanner) token.Token {
-// 	s.accept(matcher.String("-"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanPeriod(s *scanner) token.Token {
-// 	s.accept(matcher.String("."))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanSolidus(s *scanner) token.Token {
-// 	s.accept(matcher.String("/"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanReverseSolidus(s *scanner) token.Token {
-// 	s.accept(matcher.String("\\"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanColon(s *scanner) token.Token {
-// 	s.accept(matcher.String(":"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanSemiColon(s *scanner) token.Token {
-// 	s.accept(matcher.String(";"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanLessThanOperator(s *scanner) token.Token {
-// 	s.accept(matcher.String("<"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanEqualsOperator(s *scanner) token.Token {
-// 	s.accept(matcher.String("="))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanGreaterThanOperator(s *scanner) token.Token {
-// 	s.accept(matcher.String("<"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// // needs more work
-// func scanQuestioMarkOrTrigraphs(s *scanner) token.Token {
-// 	s.accept(matcher.String("="))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanLeftBracket(s *scanner) token.Token {
-// 	s.accept(matcher.String("["))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanRightBracket(s *scanner) token.Token {
-// 	s.accept(matcher.String("]"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanCircumflex(s *scanner) token.Token {
-// 	s.accept(matcher.String("^"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanUnderscore(s *scanner) token.Token {
-// 	s.accept(matcher.String("_"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanVerticalBar(s *scanner) token.Token {
-// 	s.accept(matcher.String("|"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanLeftBrace(s *scanner) token.Token {
-// 	s.accept(matcher.String("{"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanRightBrace(s *scanner) token.Token {
-// 	s.accept(matcher.String("}"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-// func scanDollarSign(s *scanner) token.Token {
-// 	s.accept(matcher.String("$"))
-// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
-// }
-
-func scanAKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithA[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithA[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanBKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithB[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithB[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanCKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithC[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithC[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanDKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithD[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithD[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanEKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithE[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithE[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanFKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithF[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithF[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanGKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithG[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithG[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanHKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithH[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithH[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanIKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithI[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithI[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanJKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithJ[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithJ[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanKKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithK[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithK[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanLKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithL[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithL[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanMKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithM[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithM[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanNKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithN[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithN[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanOKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithO[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithO[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanPKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithP[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithP[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanQKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithQ[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithQ[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanRKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithR[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithR[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanSKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithS[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithS[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanTKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithT[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithT[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanUKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithU[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithU[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-func scanVKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithV[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithV[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
-
-func scanWKeyword(s *scanner) token.Token {
-	input := string(s.input[s.start:s.seekNext(s.start)])
-	if _, ok := keywordsWithW[input]; ok {
-		s.acceptString(input)
-		return createToken(s.line, s.col, s.start, s.pos, keywordsWithW[input], input, s)
-	}
-	// return an error that this doesnt exist
-	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
-}
diff --git a/internal/parser/scanner/token/token.go b/internal/parser/scanner/token/token.go
index da10219..15d2604 100644
--- a/internal/parser/scanner/token/token.go
+++ b/internal/parser/scanner/token/token.go
@@ -10,6 +10,13 @@ type Token interface {
 	Valuer
 }
 
+// ErrorToken describes a single token that was produced by the scanner and
+// holds an error object.
+type ErrorToken interface {
+	Token
+	error
+}
+
 // Positioner describes something that has a 1-based line and col, and a 0-based
 // offset.
 type Positioner interface {
@@ -79,10 +86,28 @@ func (t tok) Value() string {
 	return t.value
 }
 
-func (t tok) IsLiteral() bool {
-	return t.IsLiteral()
-}
-
 func (t tok) String() string {
 	return fmt.Sprintf("%s(%s)", t.typ.String(), t.value)
 }
+
+type errorTok struct {
+	Token
+	err error
+}
+
+// NewErrorToken creates a new token with the given attributes and the given
+// error.
+func NewErrorToken(line, col, offset, length int, typ Type, err error) Token {
+	return errorTok{
+		Token: New(line, col, offset, length, typ, err.Error()),
+		err:   err,
+	}
+}
+
+func (t errorTok) Error() string {
+	return t.String()
+}
+
+func (t errorTok) String() string {
+	return fmt.Sprintf("%s(%s)", t.Type().String(), t.err)
+}
diff --git a/internal/parser/scanner/token/type.go b/internal/parser/scanner/token/type.go
index 46ae29d..db282b1 100644
--- a/internal/parser/scanner/token/type.go
+++ b/internal/parser/scanner/token/type.go
@@ -80,6 +80,7 @@ const (
 	KeywordForeign
 	KeywordFrom
 	KeywordFull
+	KeywordGenerated
 	KeywordGlob
 	KeywordGroup
 	KeywordGroups
@@ -167,4 +168,8 @@ const (
 	KeywordWithout
 
 	Literal
+
+	UnaryOperator
+	BinaryOperator
+	Delimiter
 )
diff --git a/internal/parser/scanner/token/type_string.go b/internal/parser/scanner/token/type_string.go
index 44f98bb..b0dd52b 100644
--- a/internal/parser/scanner/token/type_string.go
+++ b/internal/parser/scanner/token/type_string.go
@@ -70,97 +70,101 @@ func _() {
 	_ = x[KeywordForeign-59]
 	_ = x[KeywordFrom-60]
 	_ = x[KeywordFull-61]
-	_ = x[KeywordGlob-62]
-	_ = x[KeywordGroup-63]
-	_ = x[KeywordGroups-64]
-	_ = x[KeywordHaving-65]
-	_ = x[KeywordIf-66]
-	_ = x[KeywordIgnore-67]
-	_ = x[KeywordImmediate-68]
-	_ = x[KeywordIn-69]
-	_ = x[KeywordIndex-70]
-	_ = x[KeywordIndexed-71]
-	_ = x[KeywordInitially-72]
-	_ = x[KeywordInner-73]
-	_ = x[KeywordInsert-74]
-	_ = x[KeywordInstead-75]
-	_ = x[KeywordIntersect-76]
-	_ = x[KeywordInto-77]
-	_ = x[KeywordIs-78]
-	_ = x[KeywordIsnull-79]
-	_ = x[KeywordJoin-80]
-	_ = x[KeywordKey-81]
-	_ = x[KeywordLast-82]
-	_ = x[KeywordLeft-83]
-	_ = x[KeywordLike-84]
-	_ = x[KeywordLimit-85]
-	_ = x[KeywordMatch-86]
-	_ = x[KeywordNatural-87]
-	_ = x[KeywordNo-88]
-	_ = x[KeywordNot-89]
-	_ = x[KeywordNothing-90]
-	_ = x[KeywordNotnull-91]
-	_ = x[KeywordNull-92]
-	_ = x[KeywordNulls-93]
-	_ = x[KeywordOf-94]
-	_ = x[KeywordOffset-95]
-	_ = x[KeywordOn-96]
-	_ = x[KeywordOr-97]
-	_ = x[KeywordOrder-98]
-	_ = x[KeywordOthers-99]
-	_ = x[KeywordOuter-100]
-	_ = x[KeywordOver-101]
-	_ = x[KeywordPartition-102]
-	_ = x[KeywordPlan-103]
-	_ = x[KeywordPragma-104]
-	_ = x[KeywordPreceding-105]
-	_ = x[KeywordPrimary-106]
-	_ = x[KeywordQuery-107]
-	_ = x[KeywordRaise-108]
-	_ = x[KeywordRange-109]
-	_ = x[KeywordRecursive-110]
-	_ = x[KeywordReferences-111]
-	_ = x[KeywordRegexp-112]
-	_ = x[KeywordReindex-113]
-	_ = x[KeywordRelease-114]
-	_ = x[KeywordRename-115]
-	_ = x[KeywordReplace-116]
-	_ = x[KeywordRestrict-117]
-	_ = x[KeywordRight-118]
-	_ = x[KeywordRollback-119]
-	_ = x[KeywordRow-120]
-	_ = x[KeywordRows-121]
-	_ = x[KeywordSavepoint-122]
-	_ = x[KeywordSelect-123]
-	_ = x[KeywordSet-124]
-	_ = x[KeywordTable-125]
-	_ = x[KeywordTemp-126]
-	_ = x[KeywordTemporary-127]
-	_ = x[KeywordThen-128]
-	_ = x[KeywordTies-129]
-	_ = x[KeywordTo-130]
-	_ = x[KeywordTransaction-131]
-	_ = x[KeywordTrigger-132]
-	_ = x[KeywordUnbounded-133]
-	_ = x[KeywordUnion-134]
-	_ = x[KeywordUnique-135]
-	_ = x[KeywordUpdate-136]
-	_ = x[KeywordUsing-137]
-	_ = x[KeywordVacuum-138]
-	_ = x[KeywordValues-139]
-	_ = x[KeywordView-140]
-	_ = x[KeywordVirtual-141]
-	_ = x[KeywordWhen-142]
-	_ = x[KeywordWhere-143]
-	_ = x[KeywordWindow-144]
-	_ = x[KeywordWith-145]
-	_ = x[KeywordWithout-146]
-	_ = x[Literal-147]
+	_ = x[KeywordGenerated-62]
+	_ = x[KeywordGlob-63]
+	_ = x[KeywordGroup-64]
+	_ = x[KeywordGroups-65]
+	_ = x[KeywordHaving-66]
+	_ = x[KeywordIf-67]
+	_ = x[KeywordIgnore-68]
+	_ = x[KeywordImmediate-69]
+	_ = x[KeywordIn-70]
+	_ = x[KeywordIndex-71]
+	_ = x[KeywordIndexed-72]
+	_ = x[KeywordInitially-73]
+	_ = x[KeywordInner-74]
+	_ = x[KeywordInsert-75]
+	_ = x[KeywordInstead-76]
+	_ = x[KeywordIntersect-77]
+	_ = x[KeywordInto-78]
+	_ = x[KeywordIs-79]
+	_ = x[KeywordIsnull-80]
+	_ = x[KeywordJoin-81]
+	_ = x[KeywordKey-82]
+	_ = x[KeywordLast-83]
+	_ = x[KeywordLeft-84]
+	_ = x[KeywordLike-85]
+	_ = x[KeywordLimit-86]
+	_ = x[KeywordMatch-87]
+	_ = x[KeywordNatural-88]
+	_ = x[KeywordNo-89]
+	_ = x[KeywordNot-90]
+	_ = x[KeywordNothing-91]
+	_ = x[KeywordNotnull-92]
+	_ = x[KeywordNull-93]
+	_ = x[KeywordNulls-94]
+	_ = x[KeywordOf-95]
+	_ = x[KeywordOffset-96]
+	_ = x[KeywordOn-97]
+	_ = x[KeywordOr-98]
+	_ = x[KeywordOrder-99]
+	_ = x[KeywordOthers-100]
+	_ = x[KeywordOuter-101]
+	_ = x[KeywordOver-102]
+	_ = x[KeywordPartition-103]
+	_ = x[KeywordPlan-104]
+	_ = x[KeywordPragma-105]
+	_ = x[KeywordPreceding-106]
+	_ = x[KeywordPrimary-107]
+	_ = x[KeywordQuery-108]
+	_ = x[KeywordRaise-109]
+	_ = x[KeywordRange-110]
+	_ = x[KeywordRecursive-111]
+	_ = x[KeywordReferences-112]
+	_ = x[KeywordRegexp-113]
+	_ = x[KeywordReindex-114]
+	_ = x[KeywordRelease-115]
+	_ = x[KeywordRename-116]
+	_ = x[KeywordReplace-117]
+	_ = x[KeywordRestrict-118]
+	_ = x[KeywordRight-119]
+	_ = x[KeywordRollback-120]
+	_ = x[KeywordRow-121]
+	_ = x[KeywordRows-122]
+	_ = x[KeywordSavepoint-123]
+	_ = x[KeywordSelect-124]
+	_ = x[KeywordSet-125]
+	_ = x[KeywordTable-126]
+	_ = x[KeywordTemp-127]
+	_ = x[KeywordTemporary-128]
+	_ = x[KeywordThen-129]
+	_ = x[KeywordTies-130]
+	_ = x[KeywordTo-131]
+	_ = x[KeywordTransaction-132]
+	_ = x[KeywordTrigger-133]
+	_ = x[KeywordUnbounded-134]
+	_ = x[KeywordUnion-135]
+	_ = x[KeywordUnique-136]
+	_ = x[KeywordUpdate-137]
+	_ = x[KeywordUsing-138]
+	_ = x[KeywordVacuum-139]
+	_ = x[KeywordValues-140]
+	_ = x[KeywordView-141]
+	_ = x[KeywordVirtual-142]
+	_ = x[KeywordWhen-143]
+	_ = x[KeywordWhere-144]
+	_ = x[KeywordWindow-145]
+	_ = x[KeywordWith-146]
+	_ = x[KeywordWithout-147]
+	_ = x[Literal-148]
+	_ = x[UnaryOperator-149]
+	_ = x[BinaryOperator-150]
+	_ = x[Delimiter-151]
 }
 
-const _Type_name = "UnknownErrorEOFStatementSeparatorKeywordAbortKeywordActionKeywordAddKeywordAfterKeywordAllKeywordAlterKeywordAnalyzeKeywordAndKeywordAsKeywordAscKeywordAttachKeywordAutoincrementKeywordBeforeKeywordBeginKeywordBetweenKeywordByKeywordCascadeKeywordCaseKeywordCastKeywordCheckKeywordCollateKeywordColumnKeywordCommitKeywordConflictKeywordConstraintKeywordCreateKeywordCrossKeywordCurrentKeywordCurrentDateKeywordCurrentTimeKeywordCurrentTimestampKeywordDatabaseKeywordDefaultKeywordDeferrableKeywordDeferredKeywordDeleteKeywordDescKeywordDetachKeywordDistinctKeywordDoKeywordDropKeywordEachKeywordElseKeywordEndKeywordEscapeKeywordExceptKeywordExcludeKeywordExclusiveKeywordExistsKeywordExplainKeywordFailKeywordFilterKeywordFirstKeywordFollowingKeywordForKeywordForeignKeywordFromKeywordFullKeywordGlobKeywordGroupKeywordGroupsKeywordHavingKeywordIfKeywordIgnoreKeywordImmediateKeywordInKeywordIndexKeywordIndexedKeywordInitiallyKeywordInnerKeywordInsertKeywordInsteadKeywordIntersectKeywordIntoKeywordIsKeywordIsnullKeywordJoinKeywordKeyKeywordLastKeywordLeftKeywordLikeKeywordLimitKeywordMatchKeywordNaturalKeywordNoKeywordNotKeywordNothingKeywordNotnullKeywordNullKeywordNullsKeywordOfKeywordOffsetKeywordOnKeywordOrKeywordOrderKeywordOthersKeywordOuterKeywordOverKeywordPartitionKeywordPlanKeywordPragmaKeywordPrecedingKeywordPrimaryKeywordQueryKeywordRaiseKeywordRangeKeywordRecursiveKeywordReferencesKeywordRegexpKeywordReindexKeywordReleaseKeywordRenameKeywordReplaceKeywordRestrictKeywordRightKeywordRollbackKeywordRowKeywordRowsKeywordSavepointKeywordSelectKeywordSetKeywordTableKeywordTempKeywordTemporaryKeywordThenKeywordTiesKeywordToKeywordTransactionKeywordTriggerKeywordUnboundedKeywordUnionKeywordUniqueKeywordUpdateKeywordUsingKeywordVacuumKeywordValuesKeywordViewKeywordVirtualKeywordWhenKeywordWhereKeywordWindowKeywordWithKeywordWithoutLiteral"
+const _Type_name = "UnknownErrorEOFStatementSeparatorKeywordAbortKeywordActionKeywordAddKeywordAfterKeywordAllKeywordAlterKeywordAnalyzeKeywordAndKeywordAsKeywordAscKeywordAttachKeywordAutoincrementKeywordBeforeKeywordBeginKeywordBetweenKeywordByKeywordCascadeKeywordCaseKeywordCastKeywordCheckKeywordCollateKeywordColumnKeywordCommitKeywordConflictKeywordConstraintKeywordCreateKeywordCrossKeywordCurrentKeywordCurrentDateKeywordCurrentTimeKeywordCurrentTimestampKeywordDatabaseKeywordDefaultKeywordDeferrableKeywordDeferredKeywordDeleteKeywordDescKeywordDetachKeywordDistinctKeywordDoKeywordDropKeywordEachKeywordElseKeywordEndKeywordEscapeKeywordExceptKeywordExcludeKeywordExclusiveKeywordExistsKeywordExplainKeywordFailKeywordFilterKeywordFirstKeywordFollowingKeywordForKeywordForeignKeywordFromKeywordFullKeywordGeneratedKeywordGlobKeywordGroupKeywordGroupsKeywordHavingKeywordIfKeywordIgnoreKeywordImmediateKeywordInKeywordIndexKeywordIndexedKeywordInitiallyKeywordInnerKeywordInsertKeywordInsteadKeywordIntersectKeywordIntoKeywordIsKeywordIsnullKeywordJoinKeywordKeyKeywordLastKeywordLeftKeywordLikeKeywordLimitKeywordMatchKeywordNaturalKeywordNoKeywordNotKeywordNothingKeywordNotnullKeywordNullKeywordNullsKeywordOfKeywordOffsetKeywordOnKeywordOrKeywordOrderKeywordOthersKeywordOuterKeywordOverKeywordPartitionKeywordPlanKeywordPragmaKeywordPrecedingKeywordPrimaryKeywordQueryKeywordRaiseKeywordRangeKeywordRecursiveKeywordReferencesKeywordRegexpKeywordReindexKeywordReleaseKeywordRenameKeywordReplaceKeywordRestrictKeywordRightKeywordRollbackKeywordRowKeywordRowsKeywordSavepointKeywordSelectKeywordSetKeywordTableKeywordTempKeywordTemporaryKeywordThenKeywordTiesKeywordToKeywordTransactionKeywordTriggerKeywordUnboundedKeywordUnionKeywordUniqueKeywordUpdateKeywordUsingKeywordVacuumKeywordValuesKeywordViewKeywordVirtualKeywordWhenKeywordWhereKeywordWindowKeywordWithKeywordWithoutLiteralUnaryOperatorBinaryOperatorDelimiter"
 
-var _Type_index = [...]uint16{0, 7, 12, 15, 33, 45, 58, 68, 80, 90, 102, 116, 126, 135, 145, 158, 178, 191, 203, 217, 226, 240, 251, 262, 274, 288, 301, 314, 329, 346, 359, 371, 385, 403, 421, 444, 459, 473, 490, 505, 518, 529, 542, 557, 566, 577, 588, 599, 609, 622, 635, 649, 665, 678, 692, 703, 716, 728, 744, 754, 768, 779, 790, 801, 813, 826, 839, 848, 861, 877, 886, 898, 912, 928, 940, 953, 967, 983, 994, 1003, 1016, 1027, 1037, 1048, 1059, 1070, 1082, 1094, 1108, 1117, 1127, 1141, 1155, 1166, 1178, 1187, 1200, 1209, 1218, 1230, 1243, 1255, 1266, 1282, 1293, 1306, 1322, 1336, 1348, 1360, 1372, 1388, 1405, 1418, 1432, 1446, 1459, 1473, 1488, 1500, 1515, 1525, 1536, 1552, 1565, 1575, 1587, 1598, 1614, 1625, 1636, 1645, 1663, 1677, 1693, 1705, 1718, 1731, 1743, 1756, 1769, 1780, 1794, 1805, 1817, 1830, 1841, 1855, 1862}
+var _Type_index = [...]uint16{0, 7, 12, 15, 33, 45, 58, 68, 80, 90, 102, 116, 126, 135, 145, 158, 178, 191, 203, 217, 226, 240, 251, 262, 274, 288, 301, 314, 329, 346, 359, 371, 385, 403, 421, 444, 459, 473, 490, 505, 518, 529, 542, 557, 566, 577, 588, 599, 609, 622, 635, 649, 665, 678, 692, 703, 716, 728, 744, 754, 768, 779, 790, 806, 817, 829, 842, 855, 864, 877, 893, 902, 914, 928, 944, 956, 969, 983, 999, 1010, 1019, 1032, 1043, 1053, 1064, 1075, 1086, 1098, 1110, 1124, 1133, 1143, 1157, 1171, 1182, 1194, 1203, 1216, 1225, 1234, 1246, 1259, 1271, 1282, 1298, 1309, 1322, 1338, 1352, 1364, 1376, 1388, 1404, 1421, 1434, 1448, 1462, 1475, 1489, 1504, 1516, 1531, 1541, 1552, 1568, 1581, 1591, 1603, 1614, 1630, 1641, 1652, 1661, 1679, 1693, 1709, 1721, 1734, 1747, 1759, 1772, 1785, 1796, 1810, 1821, 1833, 1846, 1857, 1871, 1878, 1891, 1905, 1914}
 
 func (i Type) String() string {
 	if i >= Type(len(_Type_index)-1) {
diff --git a/internal/parser/simple_parser.go b/internal/parser/simple_parser.go
index a6d38d8..633de0d 100644
--- a/internal/parser/simple_parser.go
+++ b/internal/parser/simple_parser.go
@@ -2,8 +2,10 @@ package parser
 
 import (
 	"fmt"
+
 	"github.com/tomarrell/lbadd/internal/parser/ast"
 	"github.com/tomarrell/lbadd/internal/parser/scanner"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/ruleset"
 	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
 )
 
@@ -13,8 +15,12 @@ type errorReporter struct {
 	sealed bool
 }
 
+func (r *errorReporter) errorToken(t token.Token) {
+	r.errorf("%w: %s", ErrScanner, t.Value())
+}
+
 func (r *errorReporter) incompleteStatement() {
-	next, ok := r.p.lookahead()
+	next, ok := r.p.unsafeLowLevelLookahead()
 	if !ok {
 		r.errorf("%w: EOF", ErrIncompleteStatement)
 	} else {
@@ -31,7 +37,7 @@ func (r *errorReporter) unexpectedToken(expected ...token.Type) {
 	if r.sealed {
 		return
 	}
-	next, ok := r.p.lookahead()
+	next, ok := r.p.unsafeLowLevelLookahead()
 	if !ok || next.Type() == token.EOF {
 		// use this instead of r.prematureEOF() because we can add the
 		// information about what tokens were expected
@@ -43,6 +49,22 @@ func (r *errorReporter) unexpectedToken(expected ...token.Type) {
 	r.errorf("%w: got %s but expected one of %s at (%d:%d) offset %d length %d", ErrUnexpectedToken, next, expected, next.Line(), next.Col(), next.Offset(), next.Length())
 }
 
+func (r *errorReporter) unexpectedSingleRuneToken(typ token.Type, expected ...rune) {
+	if r.sealed {
+		return
+	}
+	next, ok := r.p.unsafeLowLevelLookahead()
+	if !ok || next.Type() == token.EOF {
+		// use this instead of r.prematureEOF() because we can add the
+		// information about what tokens were expected
+		r.errorf("%w: expected %s (more precisely one of %v)", ErrPrematureEOF, typ, expected)
+		r.sealed = true
+		return
+	}
+
+	r.errorf("%w: got %s but expected one of %s at (%d:%d) offset %d length %d", ErrUnexpectedToken, next, typ, next.Line(), next.Col(), next.Offset(), next.Length())
+}
+
 func (r *errorReporter) unhandledToken(t token.Token) {
 	r.errorf("%w: %s(%s) at (%d:%d) offset %d length %d", ErrUnknownToken, t.Type().String(), t.Value(), t.Line(), t.Col(), t.Offset(), t.Length())
 }
@@ -56,9 +78,11 @@ func (r *errorReporter) errorf(format string, args ...interface{}) {
 }
 
 type reporter interface {
+	errorToken(t token.Token)
 	incompleteStatement()
 	prematureEOF()
 	unexpectedToken(expected ...token.Type)
+	unexpectedSingleRuneToken(typ token.Type, expected ...rune)
 	unhandledToken(t token.Token)
 	unsupportedConstruct(t token.Token)
 }
@@ -69,15 +93,15 @@ type simpleParser struct {
 	scanner scanner.Scanner
 }
 
-// New creates new ready to use parser.
-func New(input string) Parser {
+// NewSimpleParser creates new ready to use parser.
+func NewSimpleParser(input string) Parser {
 	return &simpleParser{
-		scanner: scanner.New([]rune(input)),
+		scanner: scanner.NewRuleBased([]rune(input), ruleset.Default),
 	}
 }
 
 func (p *simpleParser) Next() (*ast.SQLStmt, []error, bool) {
-	if !p.scanner.HasNext() {
+	if p.scanner.Peek().Type() == token.EOF {
 		return nil, []error{}, false
 	}
 	errs := &errorReporter{
@@ -88,13 +112,13 @@ func (p *simpleParser) Next() (*ast.SQLStmt, []error, bool) {
 	return stmt, errs.errs, true
 }
 
-// skipUntil skips tokens until a token is of one of the given types. That token
-// will not be consumed, every other token will be consumed and an unexpected
-// token error will be reported.
-func (p *simpleParser) skipUntil(r reporter, types ...token.Type) {
+// searchNext skips tokens until a token is of one of the given types. That
+// token will not be consumed, every other token will be consumed and an
+// unexpected token error will be reported.
+func (p *simpleParser) searchNext(r reporter, types ...token.Type) {
 	for {
-		next, ok := p.lookahead()
-		if !ok || next.Type() == token.EOF || next.Type() == token.StatementSeparator {
+		next, ok := p.unsafeLowLevelLookahead()
+		if !ok {
 			return
 		}
 		for _, typ := range types {
@@ -107,10 +131,10 @@ func (p *simpleParser) skipUntil(r reporter, types ...token.Type) {
 	}
 }
 
-func (p *simpleParser) skipUntilNoError(types ...token.Type) {
+func (p *simpleParser) skipUntil(types ...token.Type) {
 	for {
-		next, ok := p.lookahead()
-		if !ok || next.Type() == token.EOF || next.Type() == token.StatementSeparator {
+		next, ok := p.unsafeLowLevelLookahead()
+		if !ok {
 			return
 		}
 		for _, typ := range types {
@@ -122,17 +146,49 @@ func (p *simpleParser) skipUntilNoError(types ...token.Type) {
 	}
 }
 
-func (p *simpleParser) lookahead() (next token.Token, hasNext bool) {
-	if !p.scanner.HasNext() {
-		return nil, false
-	}
-
+// unsafeLowLevelLookahead is a low level lookahead, only use if needed.
+// Remember to check for token.Error, token.EOF and token.StatementSeparator, as
+// this will only return hasNext=false if there are no more tokens (which only
+// should occur after an EOF token). Any other token will be returned with
+// next=<token>,hasNext=true.
+func (p *simpleParser) unsafeLowLevelLookahead() (next token.Token, hasNext bool) {
 	return p.scanner.Peek(), true
 }
 
-func (p *simpleParser) lookaheadWithType(typ token.Type) (token.Token, bool) {
-	next, hasNext := p.lookahead()
-	return next, hasNext && next.Type() == typ
+// lookahead performs a lookahead while consuming any error or statement
+// separator token, and reports an EOF, Error or IncompleteStatement if
+// appropriate. If this returns ok=false, return from your parse function
+// without reporting any more errors. If ok=false, this means that the next
+// token was either a StatementSeparator or EOF, and an error has been reported.
+//
+// To get any token, even EOF or the a StatementSeparator, use
+// (*simpleParser).optionalLookahead.
+func (p *simpleParser) lookahead(r reporter) (next token.Token, ok bool) {
+	next, ok = p.optionalLookahead(r)
+
+	if !ok || next.Type() == token.EOF {
+		r.prematureEOF()
+		ok = false
+	} else if next.Type() == token.StatementSeparator {
+		r.incompleteStatement()
+		ok = false
+	}
+	return
+}
+
+// optionalLookahead performs a lookahead while consuming any error token. If
+// this returns ok=false, no more tokens are available.
+func (p *simpleParser) optionalLookahead(r reporter) (next token.Token, ok bool) {
+	next, ok = p.unsafeLowLevelLookahead()
+
+	// drain all error tokens
+	for ok && next.Type() == token.Error {
+		r.errorToken(next)
+		p.consumeToken()
+		next, ok = p.unsafeLowLevelLookahead()
+	}
+
+	return next, ok
 }
 
 func (p *simpleParser) consumeToken() {
@@ -142,15 +198,15 @@ func (p *simpleParser) consumeToken() {
 func (p *simpleParser) parseSQLStatement(r reporter) (stmt *ast.SQLStmt) {
 	stmt = &ast.SQLStmt{}
 
-	if next, ok := p.lookaheadWithType(token.KeywordExplain); ok {
+	if next, ok := p.lookahead(r); ok && next.Type() == token.KeywordExplain {
 		stmt.Explain = next
 		p.consumeToken()
 
-		if next, ok := p.lookaheadWithType(token.KeywordQuery); ok {
+		if next, ok := p.lookahead(r); ok && next.Type() == token.KeywordQuery {
 			stmt.Query = next
 			p.consumeToken()
 
-			if next, ok := p.lookaheadWithType(token.KeywordPlan); ok {
+			if next, ok := p.lookahead(r); ok && next.Type() == token.KeywordPlan {
 				stmt.Plan = next
 				p.consumeToken()
 			} else {
@@ -164,14 +220,15 @@ func (p *simpleParser) parseSQLStatement(r reporter) (stmt *ast.SQLStmt) {
 	}
 
 	// according to the grammar, these are the tokens that initiate a statement
-	p.skipUntil(r, token.KeywordAlter, token.KeywordAnalyze, token.KeywordAttach, token.KeywordBegin, token.KeywordCommit, token.KeywordCreate, token.KeywordDelete, token.KeywordDetach, token.KeywordDrop, token.KeywordInsert, token.KeywordPragma, token.KeywordReindex, token.KeywordRelease, token.KeywordRollback, token.KeywordSavepoint, token.KeywordSelect, token.KeywordUpdate, token.KeywordVacuum)
+	p.searchNext(r, token.StatementSeparator, token.EOF, token.KeywordAlter, token.KeywordAnalyze, token.KeywordAttach, token.KeywordBegin, token.KeywordCommit, token.KeywordCreate, token.KeywordDelete, token.KeywordDetach, token.KeywordDrop, token.KeywordInsert, token.KeywordPragma, token.KeywordReindex, token.KeywordRelease, token.KeywordRollback, token.KeywordSavepoint, token.KeywordSelect, token.KeywordUpdate, token.KeywordVacuum)
 
-	next, ok := p.lookahead()
-	if !ok || next.Type() == token.EOF {
-		r.prematureEOF()
-		return stmt
+	next, ok := p.unsafeLowLevelLookahead()
+	if !ok {
+		r.incompleteStatement()
+		return
 	}
 
+	// lookahead processing to check what the statement ahead is
 	switch next.Type() {
 	case token.KeywordAlter:
 		stmt.AlterTableStmt = p.parseAlterTableStmt(r)
@@ -181,20 +238,555 @@ func (p *simpleParser) parseSQLStatement(r reporter) (stmt *ast.SQLStmt) {
 	case token.KeywordPragma:
 		// we don't support pragmas, as we don't need them yet
 		r.unsupportedConstruct(next)
-		p.skipUntilNoError(token.StatementSeparator, token.EOF)
+		p.skipUntil(token.StatementSeparator, token.EOF)
 	default:
 		r.unsupportedConstruct(next)
-		p.skipUntilNoError(token.StatementSeparator, token.EOF)
+		p.skipUntil(token.StatementSeparator, token.EOF)
 	}
 
-	next, ok = p.lookahead()
-	if ok && (next.Type() != token.StatementSeparator || next.Type() != token.EOF) {
-		r.unexpectedToken(token.StatementSeparator, token.EOF)
+	p.searchNext(r, token.StatementSeparator, token.EOF)
+	next, ok = p.unsafeLowLevelLookahead()
+	if !ok {
+		return
 	}
+	if next.Type() == token.StatementSeparator {
+		// if there's a statement separator, consume this token and get the next
+		// token, so that it can be checked if that next token is an EOF
+		p.consumeToken()
 
-	return stmt
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+	}
+	if next.Type() == token.EOF {
+		p.consumeToken()
+		return
+	}
+	return
 }
 
 func (p *simpleParser) parseAlterTableStmt(r reporter) (stmt *ast.AlterTableStmt) {
+	stmt = &ast.AlterTableStmt{}
+
+	p.searchNext(r, token.KeywordAlter)
+	next, ok := p.lookahead(r)
+	if !ok {
+		return
+	}
+	stmt.Alter = next
+	p.consumeToken()
+
+	next, ok = p.lookahead(r)
+	if !ok {
+		return
+	}
+	if next.Type() == token.KeywordTable {
+		stmt.Table = next
+		p.consumeToken()
+	} else {
+		r.unexpectedToken(token.KeywordTable)
+		// do not consume anything, report that there is no 'TABLE' keyword, and
+		// proceed as if we would have received it
+	}
+
+	schemaOrTableName, ok := p.lookahead(r)
+	if !ok {
+		return
+	}
+	if schemaOrTableName.Type() != token.Literal {
+		r.unexpectedToken(token.Literal)
+		return
+	}
+	p.consumeToken() // consume the literal
+
+	next, ok = p.lookahead(r)
+	if !ok {
+		return
+	}
+	if next.Value() == "." {
+		// schemaOrTableName is schema name
+		stmt.SchemaName = schemaOrTableName
+		stmt.Period = next
+		p.consumeToken()
+
+		tableName, ok := p.lookahead(r)
+		if !ok {
+			return
+		}
+		if tableName.Type() == token.Literal {
+			stmt.TableName = tableName
+			p.consumeToken()
+		}
+	} else {
+		stmt.TableName = schemaOrTableName
+	}
+	switch next.Type() {
+	case token.KeywordRename:
+		stmt.Rename = next
+		p.consumeToken()
+
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		switch next.Type() {
+		case token.KeywordTo:
+			stmt.To = next
+			p.consumeToken()
+
+			next, ok = p.lookahead(r)
+			if !ok {
+				return
+			}
+			if next.Type() != token.Literal {
+				r.unexpectedToken(token.Literal)
+				p.consumeToken()
+				return
+			}
+			stmt.NewTableName = next
+			p.consumeToken()
+		case token.KeywordColumn:
+			stmt.Column = next
+			p.consumeToken()
+
+			next, ok = p.lookahead(r)
+			if !ok {
+				return
+			}
+			if next.Type() != token.Literal {
+				r.unexpectedToken(token.Literal)
+				p.consumeToken()
+				return
+			}
+
+			fallthrough
+		case token.Literal:
+			stmt.ColumnName = next
+			p.consumeToken()
+
+			next, ok = p.lookahead(r)
+			if !ok {
+				return
+			}
+			if next.Type() != token.KeywordTo {
+				r.unexpectedToken(token.KeywordTo)
+				p.consumeToken()
+				return
+			}
+
+			stmt.To = next
+			p.consumeToken()
+
+			next, ok = p.lookahead(r)
+			if !ok {
+				return
+			}
+			if next.Type() != token.Literal {
+				r.unexpectedToken(token.Literal)
+				p.consumeToken()
+				return
+			}
+
+			stmt.NewColumnName = next
+			p.consumeToken()
+		default:
+			r.unexpectedToken(token.KeywordTo, token.KeywordColumn, token.Literal)
+		}
+	case token.KeywordAdd:
+		stmt.Add = next
+		p.consumeToken()
+
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		switch next.Type() {
+		case token.KeywordColumn:
+			stmt.Column = next
+			p.consumeToken()
+
+			next, ok = p.lookahead(r)
+			if !ok {
+				return
+			}
+			if next.Type() != token.Literal {
+				r.unexpectedToken(token.Literal)
+				p.consumeToken()
+				return
+			}
+			fallthrough
+		case token.Literal:
+			stmt.ColumnDef = p.parseColumnDef(r)
+		default:
+			r.unexpectedToken(token.KeywordColumn, token.Literal)
+		}
+	}
+
+	return
+}
+
+func (p *simpleParser) parseColumnDef(r reporter) (def *ast.ColumnDef) {
+	def = &ast.ColumnDef{}
+
+	if next, ok := p.lookahead(r); ok && next.Type() == token.Literal {
+		def.ColumnName = next
+		p.consumeToken()
+
+		if next, ok = p.lookahead(r); ok && next.Type() == token.Literal {
+			def.TypeName = p.parseTypeName(r)
+		}
+
+		for {
+			next, ok = p.optionalLookahead(r)
+			if !ok {
+				return
+			}
+			if next.Type() == token.KeywordConstraint ||
+				next.Type() == token.KeywordPrimary ||
+				next.Type() == token.KeywordNot ||
+				next.Type() == token.KeywordUnique ||
+				next.Type() == token.KeywordCheck ||
+				next.Type() == token.KeywordDefault ||
+				next.Type() == token.KeywordCollate ||
+				next.Type() == token.KeywordGenerated ||
+				next.Type() == token.KeywordReferences {
+				def.ColumnConstraint = append(def.ColumnConstraint, p.parseColumnConstraint(r))
+			} else {
+				break
+			}
+		}
+	}
+	return
+}
+
+func (p *simpleParser) parseTypeName(r reporter) (name *ast.TypeName) {
+	name = &ast.TypeName{}
+
+	// one or more name
+	if next, ok := p.lookahead(r); ok && next.Type() == token.Literal {
+		name.Name = append(name.Name, next)
+		p.consumeToken()
+	} else {
+		r.unexpectedToken(token.Literal)
+	}
+	for {
+		if next, ok := p.lookahead(r); ok && next.Type() == token.Literal {
+			name.Name = append(name.Name, next)
+			p.consumeToken()
+		} else {
+			break
+		}
+	}
+
+	if next, ok := p.lookahead(r); ok && next.Type() == token.Delimiter {
+		if next.Value() == "(" {
+			name.LeftParen = next
+			p.consumeToken()
+
+			name.SignedNumber1 = p.parseSignedNumber(r)
+		} else {
+			r.unexpectedToken(token.Delimiter)
+		}
+	} else {
+		return
+	}
+
+	if next, ok := p.lookahead(r); ok && next.Type() == token.Delimiter {
+		switch next.Value() {
+		case ",":
+			name.Comma = next
+			p.consumeToken()
+
+			name.SignedNumber2 = p.parseSignedNumber(r)
+			next, ok = p.lookahead(r)
+			if !ok {
+				return
+			}
+			if next.Type() != token.Delimiter {
+				r.unexpectedToken(token.Delimiter)
+			}
+			fallthrough
+		case ")":
+			name.RightParen = next
+			p.consumeToken()
+		}
+	} else {
+		return
+	}
+
+	return
+}
+
+func (p *simpleParser) parseSignedNumber(r reporter) (num *ast.SignedNumber) {
+	num = &ast.SignedNumber{}
+
+	next, ok := p.lookahead(r)
+	if !ok {
+		return
+	}
+	switch next.Type() {
+	case token.UnaryOperator:
+		num.Sign = next
+		p.consumeToken()
+
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() != token.Literal {
+			r.unexpectedToken(token.Literal)
+			return
+		}
+		fallthrough
+	case token.Literal:
+		num.NumericLiteral = next
+		p.consumeToken()
+	default:
+		r.unexpectedToken(token.UnaryOperator, token.Literal)
+		return
+	}
+	return
+}
+
+func (p *simpleParser) parseColumnConstraint(r reporter) (constr *ast.ColumnConstraint) {
+	constr = &ast.ColumnConstraint{}
+
+	next, ok := p.lookahead(r)
+	if !ok {
+		return
+	}
+	if next.Type() == token.KeywordConstraint {
+		constr.Constraint = next
+		p.consumeToken()
+
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.Literal {
+			constr.Name = next
+			p.consumeToken()
+		} else {
+			r.unexpectedToken(token.Literal)
+			// report that the token was unexpected, but continue as if the
+			// missing literal token was present
+		}
+	}
+
+	next, ok = p.lookahead(r)
+	if !ok {
+		return
+	}
+	switch next.Type() {
+	case token.KeywordPrimary:
+		// PRIMARY
+		constr.Primary = next
+		p.consumeToken()
+
+		// KEY
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.KeywordKey {
+			constr.Key = next
+			p.consumeToken()
+		} else {
+			r.unexpectedToken(token.KeywordKey)
+		}
+
+		// ASC, DESC
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.KeywordAsc {
+			constr.Asc = next
+			p.consumeToken()
+		} else if next.Type() == token.KeywordDesc {
+			constr.Desc = next
+			p.consumeToken()
+		}
+
+		// conflict clause
+		next, ok = p.optionalLookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.KeywordOn {
+			constr.ConflictClause = p.parseConflictClause(r)
+		}
+
+		// AUTOINCREMENT
+		next, ok = p.optionalLookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.KeywordAutoincrement {
+			constr.Autoincrement = next
+			p.consumeToken()
+		}
+
+	case token.KeywordNot:
+		// NOT
+		constr.Not = next
+		p.consumeToken()
+
+		// NULL
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.KeywordNull {
+			constr.Null = next
+			p.consumeToken()
+		} else {
+			r.unexpectedToken(token.KeywordNull)
+		}
+
+		// conflict clause
+		next, ok = p.optionalLookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.KeywordOn {
+			constr.ConflictClause = p.parseConflictClause(r)
+		}
+
+	case token.KeywordUnique:
+		// UNIQUE
+		constr.Unique = next
+		p.consumeToken()
+
+		// conflict clause
+		next, ok = p.optionalLookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.KeywordOn {
+			constr.ConflictClause = p.parseConflictClause(r)
+		}
+
+	case token.KeywordCheck:
+		// CHECK
+		constr.Check = next
+		p.consumeToken()
+
+		// left paren
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.Delimiter && next.Value() == "(" {
+			constr.LeftParen = next
+			p.consumeToken()
+		} else {
+			r.unexpectedSingleRuneToken(token.Delimiter, '(')
+			// assume that the opening paren has been omitted, report the
+			// error but proceed as if it was found
+		}
+
+		// expr
+		constr.Expr = p.parseExpression(r)
+
+		// right paren
+		next, ok = p.lookahead(r)
+		if !ok {
+			return
+		}
+		if next.Type() == token.Delimiter && next.Value() == ")" {
+			constr.RightParen = next
+			p.consumeToken()
+		} else {
+			r.unexpectedSingleRuneToken(token.Delimiter, ')')
+			// assume that the opening paren has been omitted, report the
+			// error but proceed as if it was found
+		}
+
+	case token.KeywordDefault:
+		constr.Default = next
+		p.consumeToken()
+
+	case token.KeywordCollate:
+		constr.Collate = next
+		p.consumeToken()
+
+	case token.KeywordGenerated:
+		constr.Generated = next
+		p.consumeToken()
+
+	case token.KeywordReferences:
+		constr.ForeignKeyClause = p.parseForeignKeyClause(r)
+	default:
+		r.unexpectedToken(token.KeywordPrimary, token.KeywordNot, token.KeywordUnique, token.KeywordCheck, token.KeywordDefault, token.KeywordCollate, token.KeywordGenerated, token.KeywordReferences)
+	}
+
+	return
+}
+
+func (p *simpleParser) parseForeignKeyClause(r reporter) (clause *ast.ForeignKeyClause) {
+	panic("implement me")
+}
+
+func (p *simpleParser) parseConflictClause(r reporter) (clause *ast.ConflictClause) {
+	clause = &ast.ConflictClause{}
+
+	next, ok := p.optionalLookahead(r)
+	if !ok {
+		return
+	}
+
+	// ON
+	if next.Type() == token.KeywordOn {
+		clause.On = next
+		p.consumeToken()
+	} else {
+		// if there's no 'ON' token, the empty production is assumed, which is
+		// why no error is reported here
+		return
+	}
+
+	// CONFLICT
+	next, ok = p.lookahead(r)
+	if !ok {
+		return
+	}
+	if next.Type() == token.KeywordConflict {
+		clause.Conflict = next
+		p.consumeToken()
+	} else {
+		r.unexpectedToken(token.KeywordConflict)
+		return
+	}
+
+	// ROLLBACK, ABORT, FAIL, IGNORE, REPLACE
+	next, ok = p.lookahead(r)
+	if !ok {
+		return
+	}
+	switch next.Type() {
+	case token.KeywordRollback:
+		clause.Rollback = next
+		p.consumeToken()
+	case token.KeywordAbort:
+		clause.Abort = next
+		p.consumeToken()
+	case token.KeywordFail:
+		clause.Fail = next
+		p.consumeToken()
+	case token.KeywordIgnore:
+		clause.Ignore = next
+		p.consumeToken()
+	case token.KeywordReplace:
+		clause.Replace = next
+		p.consumeToken()
+	default:
+		r.unexpectedToken(token.KeywordRollback, token.KeywordAbort, token.KeywordFail, token.KeywordIgnore, token.KeywordReplace)
+	}
+	return
+}
+
+func (p *simpleParser) parseExpression(r reporter) (expr *ast.Expr) {
 	panic("implement me")
 }
diff --git a/parser.go b/parser.go
deleted file mode 100644
index 578a586..0000000
--- a/parser.go
+++ /dev/null
@@ -1,170 +0,0 @@
-package lbadd
-
-import (
-	"fmt"
-	"log"
-	"strings"
-)
-
-func parse(sql string) (query, error) {
-	p := parser{
-		cursor: 0,
-		sql:    sql,
-		step:   stepInit,
-		query:  query{},
-		err:    nil,
-	}
-
-	return p.parse()
-}
-
-type parser struct {
-	cursor int
-	sql    string
-	step   step
-	query  query
-	err    error
-}
-
-func (p *parser) parse() (query, error) {
-	q, err := p.doParse()
-	if err != nil {
-		log.Printf("Err: failed to parse sql: %v", err)
-		return q, err
-	}
-
-	return q, nil
-}
-
-func (p *parser) doParse() (query, error) {
-	for {
-		// Check if we've hit the end of the query
-		if p.cursor >= len(p.sql) {
-			return p.query, p.err
-		}
-
-		fmt.Println("\nstep:", p.step.String())
-		fmt.Println("remaining:", p.sql[p.cursor:])
-		switch p.step {
-		case stepInit:
-			switch toUp(p.peek()) {
-			case selectQuery.String():
-				p.query.queryType = selectQuery
-				p.step = stepSelectField
-				p.pop()
-			default:
-				return p.query, fmt.Errorf("unrecognised query type")
-			}
-
-		case stepSelectField:
-			field, _ := p.pop()
-			if toUp(field) == "FROM" {
-				return p.query, fmt.Errorf("at SELECT: unexpected FROM after comma")
-			}
-
-			p.query.fields = append(p.query.fields, field)
-
-			maybeFrom := toUp(p.peek())
-			if maybeFrom == "FROM" {
-				p.step = stepSelectFrom
-				continue
-			}
-
-			p.step = stepSelectComma
-
-		case stepSelectComma:
-			maybeComma := p.sql[p.cursor]
-			if maybeComma != ',' {
-				return p.query, fmt.Errorf("at SELECT: expected comma or FROM")
-			}
-			p.cursor++
-			p.step = stepSelectField
-
-		case stepSelectFrom:
-			val, _ := p.pop()
-			if strings.ToLower(val) == "from" {
-				p.step = stepSelectTable
-				break
-			}
-			return p.query, fmt.Errorf("at SELECT: expected FROM")
-
-		case stepSelectTable:
-			val, _ := p.pop()
-			p.query.tableName = val
-			return p.query, nil
-
-		default:
-			return p.query, nil
-		}
-	}
-}
-
-var reservedWords = []string{
-	"(", ")", ">=", "<=", "!=", ",", "=", ">", "<",
-	"SELECT", "INSERT INTO", "VALUES", "UPDATE",
-	"DELETE FROM", "WHERE", "FROM", "SET",
-}
-
-func (p *parser) peek() string {
-	p.popWhitespace()
-	val, _ := p.peekWithCount()
-	return val
-}
-
-func (p *parser) pop() (string, int) {
-	p.popWhitespace()
-	val, adv := p.peekWithCount()
-	p.cursor += adv
-
-	return val, adv
-}
-
-func (p *parser) peekWithCount() (string, int) {
-	if p.cursor >= len(p.sql) {
-		return "", 0
-	}
-
-	// Advance the cursor until we reach the end of the
-	// input, or the desired character
-	buf := ""
-	i := p.cursor
-	for {
-		// Reached the end
-		if i == len(p.sql) {
-			fmt.Println("returning buf: ", buf)
-			return buf, i - p.cursor
-		}
-
-		// Reached our desired character
-		if p.sql[i] == ' ' || isReserved(string(p.sql[i])) {
-			fmt.Println("returning buf: ", buf)
-			return buf, i - p.cursor
-		}
-
-		buf += string(p.sql[i])
-		i++
-	}
-}
-
-func (p *parser) popWhitespace() {
-	for {
-		if p.sql[p.cursor] != ' ' {
-			break
-		}
-		p.cursor++
-	}
-}
-
-func isReserved(token string) bool {
-	for _, w := range reservedWords {
-		if w == token {
-			return true
-		}
-	}
-
-	return false
-}
-
-func toUp(str string) string {
-	return strings.ToUpper(str)
-}
diff --git a/query.go b/query.go
deleted file mode 100644
index b8f6971..0000000
--- a/query.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package lbadd
-
-type query struct {
-	queryType  queryType
-	tableName  string
-	conditions []condition
-	updates    map[string]string
-	inserts    [][]string
-	fields     []string
-}
-
-// The type of the parsed query
-// e.g. SELECT, INSERT etc
-type queryType int
-
-const (
-	queryUnknownType queryType = iota
-	selectQuery
-	updateQuery
-	insertQuery
-	deleteQuery
-)
-
-func (qt queryType) String() string {
-	switch qt {
-	case selectQuery:
-		return "SELECT"
-	case updateQuery:
-		return "UPDATE"
-	case insertQuery:
-		return "INSERT"
-	case deleteQuery:
-		return "DELETE"
-	default:
-		return "UNKNOWN"
-	}
-}
-
-// Operator
-type operatorType int
-
-const (
-	unknownOperator operatorType = iota
-	equal                        // =
-	notEqual                     // !=
-	greater                      // >
-	lesser                       // <
-	greaterOrEqual               // >=
-	lesserOrEqual                // <=
-)
-
-func (ot operatorType) String() string {
-	switch ot {
-	case equal:
-		return "equal"
-	case notEqual:
-		return "notEqual"
-	case greater:
-		return "greater"
-	case lesser:
-		return "lesser"
-	case greaterOrEqual:
-		return "greaterOrEqual"
-	case lesserOrEqual:
-		return "lesserOrEqual"
-	default:
-		return "unknownOperator"
-	}
-}
-
-// Condition
-type condition struct {
-	lhs        string
-	lhsIsField bool
-	operator   operatorType
-	rhs        string
-	rhsIsField bool
-}
diff --git a/step_string.go b/step_string.go
deleted file mode 100644
index a952469..0000000
--- a/step_string.go
+++ /dev/null
@@ -1,27 +0,0 @@
-// Code generated by "stringer -type=step"; DO NOT EDIT.
-
-package lbadd
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[stepInit-0]
-	_ = x[stepSelectField-1]
-	_ = x[stepSelectComma-2]
-	_ = x[stepSelectFrom-3]
-	_ = x[stepSelectTable-4]
-}
-
-const _step_name = "stepInitstepSelectFieldstepSelectCommastepSelectFromstepSelectTable"
-
-var _step_index = [...]uint8{0, 8, 23, 38, 52, 67}
-
-func (i step) String() string {
-	if i < 0 || i >= step(len(_step_index)-1) {
-		return "step(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _step_name[_step_index[i]:_step_index[i+1]]
-}
diff --git a/steps.go b/steps.go
deleted file mode 100644
index 4fe4110..0000000
--- a/steps.go
+++ /dev/null
@@ -1,13 +0,0 @@
-//go:generate stringer -type=step
-
-package lbadd
-
-type step int
-
-const (
-	stepInit step = iota
-	stepSelectField
-	stepSelectComma
-	stepSelectFrom
-	stepSelectTable
-)


Test patch:
diff --git a/codegen_test.go b/codegen_test.go
deleted file mode 100644
index 58e54cb..0000000
--- a/codegen_test.go
+++ /dev/null
@@ -1,1 +0,0 @@
-package lbadd
diff --git a/command_test.go b/command_test.go
deleted file mode 100644
index 1458ea8..0000000
--- a/command_test.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package lbadd
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_newCommand(t *testing.T) {
-	type args struct {
-		cmd string
-	}
-
-	tests := []struct {
-		name string
-		args args
-		want command
-	}{
-		{
-			name: "unknown command",
-			args: args{cmd: "uh oh"},
-			want: 0,
-		},
-		{
-			name: "insert",
-			args: args{cmd: "insert"},
-			want: 1,
-		},
-		{
-			name: "select",
-			args: args{cmd: "select"},
-			want: 2,
-		},
-		{
-			name: "delete",
-			args: args{cmd: "delete"},
-			want: 3,
-		},
-		{
-			name: "mixed casing insert",
-			args: args{cmd: "iNsErT"},
-			want: 1,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := newCommand(tt.args.cmd)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
-
-func Test_command_String(t *testing.T) {
-	tests := []struct {
-		name string
-		c    command
-		want string
-	}{
-		{
-			name: "unknown",
-			c:    0,
-			want: "UNKNOWN",
-		},
-		{
-			name: "insert",
-			c:    1,
-			want: "INSERT",
-		},
-		{
-			name: "select",
-			c:    2,
-			want: "SELECT",
-		},
-		{
-			name: "delete",
-			c:    3,
-			want: "DELETE",
-		},
-		{
-			name: "create table",
-			c:    4,
-			want: "CREATE TABLE",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := tt.c.String()
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
diff --git a/executor_test.go b/executor_test.go
deleted file mode 100644
index ad6ba46..0000000
--- a/executor_test.go
+++ /dev/null
@@ -1,233 +0,0 @@
-package lbadd
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_executor_execute(t *testing.T) {
-	type fields struct {
-		db *db
-	}
-	type args struct {
-		instr instruction
-	}
-
-	tests := []struct {
-		name    string
-		fields  fields
-		args    args
-		want    result
-		wantErr bool
-	}{
-		// TODO add test cases
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			e := &executor{
-				db: tt.fields.db,
-			}
-
-			got, err := e.execute(tt.args.instr)
-			if err != nil {
-				assert.Error(t, err)
-			}
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
-
-func Test_executor_executeCreateTable(t *testing.T) {
-	type fields struct {
-		db  *db
-		cfg exeConfig
-	}
-	type args struct {
-		instr instruction
-	}
-
-	order := 3
-
-	tests := []struct {
-		name       string
-		fields     fields
-		args       args
-		want       result
-		wantTables map[string]table
-		wantErr    bool
-	}{
-		{
-			name:    "creates a new empty table",
-			fields:  fields{db: &db{tables: map[string]table{}}, cfg: exeConfig{order: order}},
-			args:    args{instr: instruction{command: commandCreateTable, table: "users"}},
-			want:    result{created: 1},
-			wantErr: false,
-			wantTables: map[string]table{
-				"users": {
-					name:    "users",
-					store:   newBtreeOrder(order),
-					columns: []column{},
-				},
-			},
-		},
-		{
-			name:   "creates a new table with single column",
-			fields: fields{db: &db{tables: map[string]table{}}, cfg: exeConfig{order: order}},
-			args: args{instr: instruction{
-				command: commandCreateTable,
-				table:   "users",
-				params:  []string{"name", "string", "false"},
-			}},
-			want:    result{created: 1},
-			wantErr: false,
-			wantTables: map[string]table{
-				"users": {
-					name:  "users",
-					store: newBtreeOrder(order),
-					columns: []column{
-						{
-							dataType:   columnTypeString,
-							name:       "name",
-							isNullable: false,
-						},
-					},
-				},
-			},
-		},
-		{
-			name:   "creates a new table with multiple columns",
-			fields: fields{db: &db{tables: map[string]table{}}, cfg: exeConfig{order: order}},
-			args: args{instr: instruction{
-				command: commandCreateTable,
-				table:   "users",
-				params:  []string{"name", "string", "false", "age", "integer", "true"},
-			}},
-			want:    result{created: 1},
-			wantErr: false,
-			wantTables: map[string]table{
-				"users": {
-					name:  "users",
-					store: newBtreeOrder(order),
-					columns: []column{
-						{
-							dataType:   columnTypeString,
-							name:       "name",
-							isNullable: false,
-						},
-						{
-							dataType:   columnTypeInt,
-							name:       "age",
-							isNullable: true,
-						},
-					},
-				},
-			},
-		},
-		{
-			name:   "fails to create if datatype is unknown",
-			fields: fields{db: &db{tables: map[string]table{}}, cfg: exeConfig{order: order}},
-			args: args{instr: instruction{
-				command: commandCreateTable,
-				table:   "users",
-				params:  []string{"name", "unknown", "false"},
-			}},
-			want:       result{created: 0},
-			wantErr:    true,
-			wantTables: map[string]table{},
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			e := &executor{
-				db:  tt.fields.db,
-				cfg: tt.fields.cfg,
-			}
-
-			got, err := e.executeCreateTable(tt.args.instr)
-			if tt.wantErr {
-				assert.Error(t, err)
-			} else {
-				assert.NoError(t, err)
-			}
-			assert.Equal(t, tt.want, got)
-			assert.Equal(t, tt.wantTables, tt.fields.db.tables)
-		})
-	}
-}
-
-func Test_executor_executeSelect(t *testing.T) {
-	type fields struct {
-		db  *db
-		cfg exeConfig
-	}
-	type args struct {
-		instr instruction
-	}
-
-	order := 3
-	mockTable := table{
-		name:  "user",
-		store: newBtreeOrder(order),
-		columns: []column{
-			{
-				dataType:   columnTypeInt,
-				name:       "id",
-				isNullable: false,
-			},
-			{
-				dataType:   columnTypeString,
-				name:       "name",
-				isNullable: false,
-			},
-		},
-	}
-	mockTable.store.insert(0, "John Smith")
-
-	tests := []struct {
-		name    string
-		fields  fields
-		args    args
-		want    result
-		wantErr bool
-	}{
-		{
-			name: "error when table does not exist",
-			fields: fields{
-				db: &db{
-					tables: map[string]table{mockTable.name: mockTable},
-				},
-				cfg: exeConfig{order: order},
-			},
-			args: args{
-				instr: instruction{
-					command: commandSelect,
-					table:   "missing_table",
-					params:  []string{},
-				},
-			},
-			want:    result{},
-			wantErr: true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			e := &executor{
-				db:  tt.fields.db,
-				cfg: tt.fields.cfg,
-			}
-
-			got, err := e.executeSelect(tt.args.instr)
-			if tt.wantErr {
-				assert.Error(t, err)
-			} else {
-				assert.NoError(t, err)
-			}
-
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
diff --git a/btree_test.go b/internal/database/storage/btree/btree_test.go
similarity index 99%
rename from btree_test.go
rename to internal/database/storage/btree/btree_test.go
index bb1b555..7629da9 100644
--- a/btree_test.go
+++ b/internal/database/storage/btree/btree_test.go
@@ -1,4 +1,4 @@
-package lbadd
+package btree
 
 import (
 	"testing"
diff --git a/internal/parser/ast/tool/cmp/cmp_test.go b/internal/parser/ast/tool/cmp/cmp_test.go
new file mode 100644
index 0000000..7e60e51
--- /dev/null
+++ b/internal/parser/ast/tool/cmp/cmp_test.go
@@ -0,0 +1,52 @@
+package cmp
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/TimSatke/golden"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/tomarrell/lbadd/internal/parser/ast"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+var (
+	update bool
+)
+
+func TestMain(m *testing.M) {
+	flag.BoolVar(&update, "update", false, "enable to record tests and write results to disk as base for future comparisons")
+	flag.Parse()
+
+	os.Exit(m.Run())
+}
+
+func TestCompareAST(t *testing.T) {
+	left := &ast.SQLStmt{
+		AlterTableStmt: &ast.AlterTableStmt{
+			Alter:        token.New(1, 1, 0, 5, token.KeywordAlter, "ALTER"),
+			Table:        token.New(1, 7, 6, 5, token.KeywordTable, "TABLE"),
+			TableName:    token.New(1, 13, 12, 5, token.Literal, "users"),
+			Rename:       token.New(1, 19, 18, 6, token.KeywordRename, "RENAME"),
+			To:           token.New(1, 26, 25, 2, token.KeywordTo, "TO"),
+			NewTableName: token.New(1, 29, 28, 6, token.Literal, "admins"),
+		},
+	}
+	right := &ast.SQLStmt{
+		AlterTableStmt: &ast.AlterTableStmt{
+			Alter:        token.New(1, 1, 0, 5, token.KeywordAlter, "alter"),
+			Table:        token.New(1, 7, 6, 5, token.KeywordTable, "table"),
+			TableName:    token.New(1, 12, 11, 5, token.Literal, "users"),
+			Rename:       token.New(1, 19, 18, 6, token.KeywordRename, "rename"),
+			To:           token.New(1, 26, 25, 2, token.KeywordTo, "to"),
+			NewTableName: token.New(1, 29, 28, 6, token.Literal, "admins"),
+		},
+	}
+	output := spew.Sdump(CompareAST(left, right))
+	t.Log(output)
+
+	g := golden.New(t)
+	g.ShouldUpdate = update
+	g.Assert(t.Name(), []byte(output))
+}
diff --git a/internal/parser/ast/tool/cmp/testdata/TestCompareAST.golden b/internal/parser/ast/tool/cmp/testdata/TestCompareAST.golden
new file mode 100644
index 0000000..c71ee32
--- /dev/null
+++ b/internal/parser/ast/tool/cmp/testdata/TestCompareAST.golden
@@ -0,0 +1,44 @@
+([]cmp.Delta) (len=6 cap=6) {
+ (cmp.Delta) {
+  Path: (string) (len=28) "SQLStmt.AlterTableStmt.Alter",
+  Typ: (cmp.DeltaType) TokenValue,
+  Message: (string) (len=18) "values don't match",
+  Left: (string) (len=5) "ALTER",
+  Right: (string) (len=5) "alter"
+ },
+ (cmp.Delta) {
+  Path: (string) (len=28) "SQLStmt.AlterTableStmt.Table",
+  Typ: (cmp.DeltaType) TokenValue,
+  Message: (string) (len=18) "values don't match",
+  Left: (string) (len=5) "TABLE",
+  Right: (string) (len=5) "table"
+ },
+ (cmp.Delta) {
+  Path: (string) (len=32) "SQLStmt.AlterTableStmt.TableName",
+  Typ: (cmp.DeltaType) TokenPosition,
+  Message: (string) (len=16) "cols don't match",
+  Left: (int) 13,
+  Right: (int) 12
+ },
+ (cmp.Delta) {
+  Path: (string) (len=32) "SQLStmt.AlterTableStmt.TableName",
+  Typ: (cmp.DeltaType) TokenPosition,
+  Message: (string) (len=19) "offsets don't match",
+  Left: (int) 12,
+  Right: (int) 11
+ },
+ (cmp.Delta) {
+  Path: (string) (len=29) "SQLStmt.AlterTableStmt.Rename",
+  Typ: (cmp.DeltaType) TokenValue,
+  Message: (string) (len=18) "values don't match",
+  Left: (string) (len=6) "RENAME",
+  Right: (string) (len=6) "rename"
+ },
+ (cmp.Delta) {
+  Path: (string) (len=25) "SQLStmt.AlterTableStmt.To",
+  Typ: (cmp.DeltaType) TokenValue,
+  Message: (string) (len=18) "values don't match",
+  Left: (string) (len=2) "TO",
+  Right: (string) (len=2) "to"
+ }
+}
diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go
new file mode 100644
index 0000000..6653537
--- /dev/null
+++ b/internal/parser/parser_test.go
@@ -0,0 +1,216 @@
+package parser
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/TimSatke/golden"
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/tomarrell/lbadd/internal/parser/ast"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+var (
+	update bool
+)
+
+func TestMain(m *testing.M) {
+	flag.BoolVar(&update, "update", false, "enable to record tests and write results to disk as base for future comparisons")
+	flag.Parse()
+
+	os.Exit(m.Run())
+}
+
+func TestSingleStatementParse(t *testing.T) {
+	inputs := []struct {
+		Query string
+		Stmt  *ast.SQLStmt
+	}{
+		{
+			"ALTER TABLE users RENAME TO admins",
+			&ast.SQLStmt{
+				AlterTableStmt: &ast.AlterTableStmt{
+					Alter:        token.New(1, 1, 0, 5, token.KeywordAlter, "ALTER"),
+					Table:        token.New(1, 7, 6, 5, token.KeywordTable, "TABLE"),
+					TableName:    token.New(1, 13, 12, 5, token.Literal, "users"),
+					Rename:       token.New(1, 19, 18, 6, token.KeywordRename, "RENAME"),
+					To:           token.New(1, 26, 25, 2, token.KeywordTo, "TO"),
+					NewTableName: token.New(1, 29, 28, 6, token.Literal, "admins"),
+				},
+			},
+		},
+		{
+			"ALTER TABLE users RENAME COLUMN name TO username",
+			&ast.SQLStmt{
+				AlterTableStmt: &ast.AlterTableStmt{
+					Alter:         token.New(1, 1, 0, 5, token.KeywordAlter, "ALTER"),
+					Table:         token.New(1, 7, 6, 5, token.KeywordTable, "TABLE"),
+					TableName:     token.New(1, 13, 12, 5, token.Literal, "users"),
+					Rename:        token.New(1, 19, 18, 6, token.KeywordRename, "RENAME"),
+					Column:        token.New(1, 26, 25, 6, token.KeywordColumn, "COLUMN"),
+					ColumnName:    token.New(1, 33, 32, 4, token.Literal, "name"),
+					To:            token.New(1, 38, 37, 2, token.KeywordTo, "TO"),
+					NewColumnName: token.New(1, 41, 40, 8, token.Literal, "username"),
+				},
+			},
+		},
+		{
+			"ALTER TABLE users RENAME name TO username",
+			&ast.SQLStmt{
+				AlterTableStmt: &ast.AlterTableStmt{
+					Alter:         token.New(1, 1, 0, 5, token.KeywordAlter, "ALTER"),
+					Table:         token.New(1, 7, 6, 5, token.KeywordTable, "TABLE"),
+					TableName:     token.New(1, 13, 12, 5, token.Literal, "users"),
+					Rename:        token.New(1, 19, 18, 6, token.KeywordRename, "RENAME"),
+					ColumnName:    token.New(1, 26, 25, 4, token.Literal, "name"),
+					To:            token.New(1, 31, 30, 2, token.KeywordTo, "TO"),
+					NewColumnName: token.New(1, 34, 33, 8, token.Literal, "username"),
+				},
+			},
+		},
+		{
+			"ALTER TABLE users ADD COLUMN foo VARCHAR(15) CONSTRAINT pk PRIMARY KEY AUTOINCREMENT CONSTRAINT nn NOT NULL",
+			&ast.SQLStmt{
+				AlterTableStmt: &ast.AlterTableStmt{
+					Alter:     token.New(1, 1, 0, 5, token.KeywordAlter, "ALTER"),
+					Table:     token.New(1, 7, 6, 5, token.KeywordTable, "TABLE"),
+					TableName: token.New(1, 13, 12, 5, token.Literal, "users"),
+					Add:       token.New(1, 19, 18, 3, token.KeywordAdd, "ADD"),
+					Column:    token.New(1, 23, 22, 6, token.KeywordColumn, "COLUMN"),
+					ColumnDef: &ast.ColumnDef{
+						ColumnName: token.New(1, 30, 29, 3, token.Literal, "foo"),
+						TypeName: &ast.TypeName{
+							Name: []token.Token{
+								token.New(1, 34, 33, 7, token.Literal, "VARCHAR"),
+							},
+							LeftParen: token.New(1, 41, 40, 1, token.Delimiter, "("),
+							SignedNumber1: &ast.SignedNumber{
+								NumericLiteral: token.New(1, 42, 41, 2, token.Literal, "15"),
+							},
+							RightParen: token.New(1, 44, 43, 1, token.Delimiter, ")"),
+						},
+						ColumnConstraint: []*ast.ColumnConstraint{
+							{
+								Constraint:    token.New(1, 46, 45, 10, token.KeywordConstraint, "CONSTRAINT"),
+								Name:          token.New(1, 57, 56, 2, token.Literal, "pk"),
+								Primary:       token.New(1, 60, 59, 7, token.KeywordPrimary, "PRIMARY"),
+								Key:           token.New(1, 68, 67, 3, token.KeywordKey, "KEY"),
+								Autoincrement: token.New(1, 72, 71, 13, token.KeywordAutoincrement, "AUTOINCREMENT"),
+							},
+							{
+								Constraint: token.New(1, 86, 85, 10, token.KeywordConstraint, "CONSTRAINT"),
+								Name:       token.New(1, 97, 96, 2, token.Literal, "nn"),
+								Not:        token.New(1, 100, 99, 3, token.KeywordNot, "NOT"),
+								Null:       token.New(1, 104, 103, 4, token.KeywordNull, "NULL"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			"ALTER TABLE users ADD foo VARCHAR(15) CONSTRAINT pk PRIMARY KEY AUTOINCREMENT CONSTRAINT nn NOT NULL",
+			&ast.SQLStmt{
+				AlterTableStmt: &ast.AlterTableStmt{
+					Alter:     token.New(1, 1, 0, 5, token.KeywordAlter, "ALTER"),
+					Table:     token.New(1, 7, 6, 5, token.KeywordTable, "TABLE"),
+					TableName: token.New(1, 13, 12, 5, token.Literal, "users"),
+					Add:       token.New(1, 19, 18, 3, token.KeywordAdd, "ADD"),
+					ColumnDef: &ast.ColumnDef{
+						ColumnName: token.New(1, 23, 22, 3, token.Literal, "foo"),
+						TypeName: &ast.TypeName{
+							Name: []token.Token{
+								token.New(1, 27, 26, 7, token.Literal, "VARCHAR"),
+							},
+							LeftParen: token.New(1, 34, 33, 1, token.Delimiter, "("),
+							SignedNumber1: &ast.SignedNumber{
+								NumericLiteral: token.New(1, 35, 34, 2, token.Literal, "15"),
+							},
+							RightParen: token.New(1, 37, 36, 1, token.Delimiter, ")"),
+						},
+						ColumnConstraint: []*ast.ColumnConstraint{
+							{
+								Constraint:    token.New(1, 39, 38, 10, token.KeywordConstraint, "CONSTRAINT"),
+								Name:          token.New(1, 50, 49, 2, token.Literal, "pk"),
+								Primary:       token.New(1, 53, 52, 7, token.KeywordPrimary, "PRIMARY"),
+								Key:           token.New(1, 61, 60, 3, token.KeywordKey, "KEY"),
+								Autoincrement: token.New(1, 65, 64, 13, token.KeywordAutoincrement, "AUTOINCREMENT"),
+							},
+							{
+								Constraint: token.New(1, 79, 78, 10, token.KeywordConstraint, "CONSTRAINT"),
+								Name:       token.New(1, 90, 89, 2, token.Literal, "nn"),
+								Not:        token.New(1, 93, 92, 3, token.KeywordNot, "NOT"),
+								Null:       token.New(1, 97, 96, 4, token.KeywordNull, "NULL"),
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, input := range inputs {
+		t.Run(input.Query[0:11], func(t *testing.T) {
+			assert := assert.New(t)
+
+			p := New(input.Query)
+
+			stmt, errs, ok := p.Next()
+			assert.True(ok, "expected exactly one statement")
+			for _, err := range errs {
+				assert.Nil(err)
+			}
+
+			opts := []cmp.Option{
+				cmp.Comparer(func(t1, t2 token.Token) bool {
+					if t1 == t2 {
+						return true
+					}
+					if (t1 == nil && t2 != nil) ||
+						(t1 != nil && t2 == nil) {
+						return false
+					}
+					return t1.Line() == t2.Line() &&
+						t1.Col() == t2.Col() &&
+						t1.Offset() == t2.Offset() &&
+						t1.Length() == t2.Length() &&
+						t1.Type() == t2.Type() &&
+						t1.Value() == t2.Value()
+				}),
+			}
+
+			t.Log(cmp.Diff(input.Stmt, stmt, opts...))
+			assert.True(cmp.Equal(input.Stmt, stmt, opts...))
+
+			_, _, ok = p.Next()
+			assert.False(ok, "expected only one statement")
+		})
+	}
+}
+
+func TestParserGolden(t *testing.T) {
+	inputs := []struct {
+		Name  string
+		Query string
+	}{
+		{"empty", ""},
+	}
+	for _, input := range inputs {
+		t.Run(input.Name, func(t *testing.T) {
+			p := NewSimpleParser(input.Query)
+
+			for {
+				stmt, errs, ok := p.Next()
+				if !ok {
+					break
+				}
+
+				g := golden.New(t)
+				g.ShouldUpdate = update
+				g.AssertStruct(input.Name+"_ast", stmt)
+				g.AssertStruct(input.Name+"_errs", errs)
+			}
+		})
+	}
+}
diff --git a/internal/parser/scanner/rule_based_scanner_test.go b/internal/parser/scanner/rule_based_scanner_test.go
new file mode 100644
index 0000000..09b875d
--- /dev/null
+++ b/internal/parser/scanner/rule_based_scanner_test.go
@@ -0,0 +1,182 @@
+package scanner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/ruleset"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+func TestRuleBasedScanner(t *testing.T) {
+	inputs := []struct {
+		query   string
+		ruleset ruleset.Ruleset
+		want    []token.Token
+	}{
+		{
+			"SELECT FROM WHERE",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 5, token.KeywordWhere, "WHERE"),
+				token.New(1, 18, 17, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT FROM \"WHERE\"",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 7, token.Literal, "\"WHERE\""),
+				token.New(1, 20, 19, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT FROM \"WHERE",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 1, token.Error, `unexpected token: '"' at offset 12`),
+				token.New(1, 14, 13, 5, token.KeywordWhere, "WHERE"),
+				token.New(1, 19, 18, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT      FROM || & +7 59 \"foobar\"",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 13, 12, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 18, 17, 2, token.BinaryOperator, "||"),
+				token.New(1, 21, 20, 1, token.BinaryOperator, "&"),
+				token.New(1, 23, 22, 1, token.UnaryOperator, "+"),
+				token.New(1, 24, 23, 1, token.Literal, "7"),
+				token.New(1, 26, 25, 2, token.Literal, "59"),
+				token.New(1, 29, 28, 8, token.Literal, "\"foobar\""),
+				token.New(1, 37, 36, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT FROM 'WHERE'",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 7, token.Literal, "'WHERE'"),
+				token.New(1, 20, 19, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT \"myCol FROM \"myTable\"",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 13, token.Literal, `"myCol FROM "`),
+				token.New(1, 21, 20, 7, token.Literal, "myTable"),
+				token.New(1, 28, 27, 1, token.Error, "unexpected token: '\"' at offset 27"),
+				token.New(1, 29, 28, 0, token.EOF, ""),
+			},
+		},
+		{
+			"SELECT \" FROM",
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 1, token.Error, `unexpected token: '"' at offset 7`),
+				token.New(1, 10, 9, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 14, 13, 0, token.EOF, ""),
+			},
+		},
+		{
+			`SELECT FROM "this \" can be anything"`,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 25, token.Literal, `"this \" can be anything"`),
+				token.New(1, 38, 37, 0, token.EOF, ""),
+			},
+		},
+		{
+			`SELECT FROM 'this \' can be anything'`,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 6, token.KeywordSelect, "SELECT"),
+				token.New(1, 8, 7, 4, token.KeywordFrom, "FROM"),
+				token.New(1, 13, 12, 25, token.Literal, `'this \' can be anything'`),
+				token.New(1, 38, 37, 0, token.EOF, ""),
+			},
+		},
+		{
+			`|| * / % + - ~ << >> & | < <= > >= = == != <> !>> >>`,
+			ruleset.Default,
+			[]token.Token{
+				token.New(1, 1, 0, 2, token.BinaryOperator, "||"),
+				token.New(1, 4, 3, 1, token.BinaryOperator, "*"),
+				token.New(1, 6, 5, 1, token.BinaryOperator, "/"),
+				token.New(1, 8, 7, 1, token.BinaryOperator, "%"),
+				token.New(1, 10, 9, 1, token.UnaryOperator, "+"),
+				token.New(1, 12, 11, 1, token.UnaryOperator, "-"),
+				token.New(1, 14, 13, 1, token.UnaryOperator, "~"),
+				token.New(1, 16, 15, 2, token.BinaryOperator, "<<"),
+				token.New(1, 19, 18, 2, token.BinaryOperator, ">>"),
+				token.New(1, 22, 21, 1, token.BinaryOperator, "&"),
+				token.New(1, 24, 23, 1, token.BinaryOperator, "|"),
+				token.New(1, 26, 25, 1, token.BinaryOperator, "<"),
+				token.New(1, 28, 27, 2, token.BinaryOperator, "<="),
+				token.New(1, 31, 30, 1, token.BinaryOperator, ">"),
+				token.New(1, 33, 32, 2, token.BinaryOperator, ">="),
+				token.New(1, 36, 35, 1, token.BinaryOperator, "="),
+				token.New(1, 38, 37, 2, token.BinaryOperator, "=="),
+				token.New(1, 41, 40, 2, token.BinaryOperator, "!="),
+				token.New(1, 44, 43, 2, token.BinaryOperator, "<>"),
+				token.New(1, 47, 46, 1, token.Error, "unexpected token: '!' at offset 46"),
+				token.New(1, 48, 47, 2, token.BinaryOperator, ">>"),
+				token.New(1, 51, 50, 2, token.BinaryOperator, ">>"),
+				token.New(1, 53, 52, 0, token.EOF, ""),
+			},
+		},
+	}
+	for _, input := range inputs {
+		t.Run("ruleset=default/"+input.query, _TestRuleBasedScannerWithRuleset(input.query, input.ruleset, input.want))
+	}
+}
+
+func _TestRuleBasedScannerWithRuleset(input string, ruleset ruleset.Ruleset, want []token.Token) func(*testing.T) {
+	return func(t *testing.T) {
+		assert := assert.New(t)
+
+		var got []token.Token
+
+		// create the scanner to be tested
+		sc := NewRuleBased([]rune(input), ruleset)
+
+		// collect all whitespaces
+		for {
+			next := sc.Next()
+			got = append(got, next)
+			if next.Type() == token.EOF {
+				break
+			}
+		}
+		assert.Equalf(len(want), len(got), "did not receive as much tokens as expected (expected %d, but got %d)", len(want), len(got))
+
+		limit := len(want)
+		if len(got) < limit {
+			limit = len(got)
+		}
+
+		for i := 0; i < limit; i++ {
+			assert.Equal(want[i].Line(), got[i].Line(), "Line doesn't match")
+			assert.Equal(want[i].Col(), got[i].Col(), "Col doesn't match")
+			assert.Equal(want[i].Offset(), got[i].Offset(), "Offset doesn't match")
+			assert.Equal(want[i].Length(), got[i].Length(), "Length doesn't match")
+			assert.Equal(want[i].Type(), got[i].Type(), "Type doesn't match")
+			assert.Equal(want[i].Value(), got[i].Value(), "Value doesn't match")
+		}
+	}
+}
diff --git a/internal/parser/scanner/scanner_test.go b/internal/parser/scanner/scanner_test.go
deleted file mode 100644
index d92e5dc..0000000
--- a/internal/parser/scanner/scanner_test.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package scanner
-
-import (
-	"testing"
-)
-
-func Test_hasNext(t *testing.T) {
-	t.SkipNow()
-
-	input := "SELECT "
-	scanner := New([]rune(input))
-	for scanner.HasNext() {
-		scanner.Next()
-	}
-}
diff --git a/internal/parser/scanner/test/gen.go b/internal/parser/scanner/test/gen.go
new file mode 100644
index 0000000..77e299b
--- /dev/null
+++ b/internal/parser/scanner/test/gen.go
@@ -0,0 +1,1473 @@
+package scanner
+
+import (
+	"bytes"
+	"math/rand"
+	"time"
+	"unicode"
+
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+var _ token.Token = (*genTok)(nil)
+
+var rng = rand.New(rand.NewSource(time.Now().Unix()))
+
+type genTok struct {
+	offset int
+	value  string
+	typ    token.Type
+}
+
+func (t genTok) Line() int {
+	return 1
+}
+
+func (t genTok) Col() int {
+	return t.offset + 1
+}
+
+func (t genTok) Offset() int {
+	return t.offset
+}
+
+func (t genTok) Length() int {
+	return len(t.value)
+}
+
+func (t genTok) Type() token.Type {
+	return t.typ
+}
+
+func (t genTok) Value() string {
+	return t.value
+}
+
+func generateEOF(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.EOF,
+	}
+}
+func generateStatementSeparator(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.StatementSeparator,
+		value:  ";",
+	}
+}
+func generateKeywordAbort(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAbort,
+		value:  caseShuffle("Abort"),
+	}
+}
+func generateKeywordAction(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAction,
+		value:  caseShuffle("Action"),
+	}
+}
+func generateKeywordAdd(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAdd,
+		value:  caseShuffle("Add"),
+	}
+}
+func generateKeywordAfter(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAfter,
+		value:  caseShuffle("After"),
+	}
+}
+func generateKeywordAll(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAll,
+		value:  caseShuffle("All"),
+	}
+}
+func generateKeywordAlter(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAlter,
+		value:  caseShuffle("Alter"),
+	}
+}
+func generateKeywordAnalyze(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAnalyze,
+		value:  caseShuffle("Analyze"),
+	}
+}
+func generateKeywordAnd(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAnd,
+		value:  caseShuffle("And"),
+	}
+}
+func generateKeywordAs(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAs,
+		value:  caseShuffle("As"),
+	}
+}
+func generateKeywordAsc(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAsc,
+		value:  caseShuffle("Asc"),
+	}
+}
+func generateKeywordAttach(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAttach,
+		value:  caseShuffle("Attach"),
+	}
+}
+func generateKeywordAutoincrement(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordAutoincrement,
+		value:  caseShuffle("Autoincrement"),
+	}
+}
+func generateKeywordBefore(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordBefore,
+		value:  caseShuffle("Before"),
+	}
+}
+func generateKeywordBegin(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordBegin,
+		value:  caseShuffle("Begin"),
+	}
+}
+func generateKeywordBetween(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordBetween,
+		value:  caseShuffle("Between"),
+	}
+}
+func generateKeywordBy(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordBy,
+		value:  caseShuffle("By"),
+	}
+}
+func generateKeywordCascade(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCascade,
+		value:  caseShuffle("Cascade"),
+	}
+}
+func generateKeywordCase(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCase,
+		value:  caseShuffle("Case"),
+	}
+}
+func generateKeywordCast(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCast,
+		value:  caseShuffle("Cast"),
+	}
+}
+func generateKeywordCheck(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCheck,
+		value:  caseShuffle("Check"),
+	}
+}
+func generateKeywordCollate(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCollate,
+		value:  caseShuffle("Collate"),
+	}
+}
+func generateKeywordColumn(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordColumn,
+		value:  caseShuffle("Column"),
+	}
+}
+func generateKeywordCommit(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCommit,
+		value:  caseShuffle("Commit"),
+	}
+}
+func generateKeywordConflict(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordConflict,
+		value:  caseShuffle("Conflict"),
+	}
+}
+func generateKeywordConstraint(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordConstraint,
+		value:  caseShuffle("Constraint"),
+	}
+}
+func generateKeywordCreate(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCreate,
+		value:  caseShuffle("Create"),
+	}
+}
+func generateKeywordCross(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCross,
+		value:  caseShuffle("Cross"),
+	}
+}
+func generateKeywordCurrent(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCurrent,
+		value:  caseShuffle("Current"),
+	}
+}
+func generateKeywordCurrentDate(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCurrentDate,
+		value:  caseShuffle("CurrentDate"),
+	}
+}
+func generateKeywordCurrentTime(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCurrentTime,
+		value:  caseShuffle("CurrentTime"),
+	}
+}
+func generateKeywordCurrentTimestamp(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordCurrentTimestamp,
+		value:  caseShuffle("CurrentTimestamp"),
+	}
+}
+func generateKeywordDatabase(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDatabase,
+		value:  caseShuffle("Database"),
+	}
+}
+func generateKeywordDefault(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDefault,
+		value:  caseShuffle("Default"),
+	}
+}
+func generateKeywordDeferrable(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDeferrable,
+		value:  caseShuffle("Deferrable"),
+	}
+}
+func generateKeywordDeferred(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDeferred,
+		value:  caseShuffle("Deferred"),
+	}
+}
+func generateKeywordDelete(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDelete,
+		value:  caseShuffle("Delete"),
+	}
+}
+func generateKeywordDesc(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDesc,
+		value:  caseShuffle("Desc"),
+	}
+}
+func generateKeywordDetach(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDetach,
+		value:  caseShuffle("Detach"),
+	}
+}
+func generateKeywordDistinct(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDistinct,
+		value:  caseShuffle("Distinct"),
+	}
+}
+func generateKeywordDo(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDo,
+		value:  caseShuffle("Do"),
+	}
+}
+func generateKeywordDrop(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordDrop,
+		value:  caseShuffle("Drop"),
+	}
+}
+func generateKeywordEach(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordEach,
+		value:  caseShuffle("Each"),
+	}
+}
+func generateKeywordElse(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordElse,
+		value:  caseShuffle("Else"),
+	}
+}
+func generateKeywordEnd(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordEnd,
+		value:  caseShuffle("End"),
+	}
+}
+func generateKeywordEscape(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordEscape,
+		value:  caseShuffle("Escape"),
+	}
+}
+func generateKeywordExcept(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordExcept,
+		value:  caseShuffle("Except"),
+	}
+}
+func generateKeywordExclude(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordExclude,
+		value:  caseShuffle("Exclude"),
+	}
+}
+func generateKeywordExclusive(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordExclusive,
+		value:  caseShuffle("Exclusive"),
+	}
+}
+func generateKeywordExists(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordExists,
+		value:  caseShuffle("Exists"),
+	}
+}
+func generateKeywordExplain(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordExplain,
+		value:  caseShuffle("Explain"),
+	}
+}
+func generateKeywordFail(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordFail,
+		value:  caseShuffle("Fail"),
+	}
+}
+func generateKeywordFilter(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordFilter,
+		value:  caseShuffle("Filter"),
+	}
+}
+func generateKeywordFirst(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordFirst,
+		value:  caseShuffle("First"),
+	}
+}
+func generateKeywordFollowing(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordFollowing,
+		value:  caseShuffle("Following"),
+	}
+}
+func generateKeywordFor(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordFor,
+		value:  caseShuffle("For"),
+	}
+}
+func generateKeywordForeign(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordForeign,
+		value:  caseShuffle("Foreign"),
+	}
+}
+func generateKeywordFrom(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordFrom,
+		value:  caseShuffle("From"),
+	}
+}
+func generateKeywordFull(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordFull,
+		value:  caseShuffle("Full"),
+	}
+}
+func generateKeywordGenerated(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordGenerated,
+		value:  caseShuffle("Generated"),
+	}
+}
+func generateKeywordGlob(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordGlob,
+		value:  caseShuffle("Glob"),
+	}
+}
+func generateKeywordGroup(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordGroup,
+		value:  caseShuffle("Group"),
+	}
+}
+func generateKeywordGroups(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordGroups,
+		value:  caseShuffle("Groups"),
+	}
+}
+func generateKeywordHaving(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordHaving,
+		value:  caseShuffle("Having"),
+	}
+}
+func generateKeywordIf(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIf,
+		value:  caseShuffle("If"),
+	}
+}
+func generateKeywordIgnore(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIgnore,
+		value:  caseShuffle("Ignore"),
+	}
+}
+func generateKeywordImmediate(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordImmediate,
+		value:  caseShuffle("Immediate"),
+	}
+}
+func generateKeywordIn(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIn,
+		value:  caseShuffle("In"),
+	}
+}
+func generateKeywordIndex(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIndex,
+		value:  caseShuffle("Index"),
+	}
+}
+func generateKeywordIndexed(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIndexed,
+		value:  caseShuffle("Indexed"),
+	}
+}
+func generateKeywordInitially(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordInitially,
+		value:  caseShuffle("Initially"),
+	}
+}
+func generateKeywordInner(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordInner,
+		value:  caseShuffle("Inner"),
+	}
+}
+func generateKeywordInsert(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordInsert,
+		value:  caseShuffle("Insert"),
+	}
+}
+func generateKeywordInstead(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordInstead,
+		value:  caseShuffle("Instead"),
+	}
+}
+func generateKeywordIntersect(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIntersect,
+		value:  caseShuffle("Intersect"),
+	}
+}
+func generateKeywordInto(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordInto,
+		value:  caseShuffle("Into"),
+	}
+}
+func generateKeywordIs(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIs,
+		value:  caseShuffle("Is"),
+	}
+}
+func generateKeywordIsnull(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordIsnull,
+		value:  caseShuffle("Isnull"),
+	}
+}
+func generateKeywordJoin(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordJoin,
+		value:  caseShuffle("Join"),
+	}
+}
+func generateKeywordKey(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordKey,
+		value:  caseShuffle("Key"),
+	}
+}
+func generateKeywordLast(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordLast,
+		value:  caseShuffle("Last"),
+	}
+}
+func generateKeywordLeft(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordLeft,
+		value:  caseShuffle("Left"),
+	}
+}
+func generateKeywordLike(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordLike,
+		value:  caseShuffle("Like"),
+	}
+}
+func generateKeywordLimit(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordLimit,
+		value:  caseShuffle("Limit"),
+	}
+}
+func generateKeywordMatch(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordMatch,
+		value:  caseShuffle("Match"),
+	}
+}
+func generateKeywordNatural(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordNatural,
+		value:  caseShuffle("Natural"),
+	}
+}
+func generateKeywordNo(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordNo,
+		value:  caseShuffle("No"),
+	}
+}
+func generateKeywordNot(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordNot,
+		value:  caseShuffle("Not"),
+	}
+}
+func generateKeywordNothing(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordNothing,
+		value:  caseShuffle("Nothing"),
+	}
+}
+func generateKeywordNotnull(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordNotnull,
+		value:  caseShuffle("Notnull"),
+	}
+}
+func generateKeywordNull(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordNull,
+		value:  caseShuffle("Null"),
+	}
+}
+func generateKeywordNulls(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordNulls,
+		value:  caseShuffle("Nulls"),
+	}
+}
+func generateKeywordOf(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOf,
+		value:  caseShuffle("Of"),
+	}
+}
+func generateKeywordOffset(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOffset,
+		value:  caseShuffle("Offset"),
+	}
+}
+func generateKeywordOn(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOn,
+		value:  caseShuffle("On"),
+	}
+}
+func generateKeywordOr(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOr,
+		value:  caseShuffle("Or"),
+	}
+}
+func generateKeywordOrder(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOrder,
+		value:  caseShuffle("Order"),
+	}
+}
+func generateKeywordOthers(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOthers,
+		value:  caseShuffle("Others"),
+	}
+}
+func generateKeywordOuter(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOuter,
+		value:  caseShuffle("Outer"),
+	}
+}
+func generateKeywordOver(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordOver,
+		value:  caseShuffle("Over"),
+	}
+}
+func generateKeywordPartition(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordPartition,
+		value:  caseShuffle("Partition"),
+	}
+}
+func generateKeywordPlan(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordPlan,
+		value:  caseShuffle("Plan"),
+	}
+}
+func generateKeywordPragma(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordPragma,
+		value:  caseShuffle("Pragma"),
+	}
+}
+func generateKeywordPreceding(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordPreceding,
+		value:  caseShuffle("Preceding"),
+	}
+}
+func generateKeywordPrimary(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordPrimary,
+		value:  caseShuffle("Primary"),
+	}
+}
+func generateKeywordQuery(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordQuery,
+		value:  caseShuffle("Query"),
+	}
+}
+func generateKeywordRaise(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRaise,
+		value:  caseShuffle("Raise"),
+	}
+}
+func generateKeywordRange(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRange,
+		value:  caseShuffle("Range"),
+	}
+}
+func generateKeywordRecursive(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRecursive,
+		value:  caseShuffle("Recursive"),
+	}
+}
+func generateKeywordReferences(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordReferences,
+		value:  caseShuffle("References"),
+	}
+}
+func generateKeywordRegexp(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRegexp,
+		value:  caseShuffle("Regexp"),
+	}
+}
+func generateKeywordReindex(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordReindex,
+		value:  caseShuffle("Reindex"),
+	}
+}
+func generateKeywordRelease(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRelease,
+		value:  caseShuffle("Release"),
+	}
+}
+func generateKeywordRename(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRename,
+		value:  caseShuffle("Rename"),
+	}
+}
+func generateKeywordReplace(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordReplace,
+		value:  caseShuffle("Replace"),
+	}
+}
+func generateKeywordRestrict(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRestrict,
+		value:  caseShuffle("Restrict"),
+	}
+}
+func generateKeywordRight(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRight,
+		value:  caseShuffle("Right"),
+	}
+}
+func generateKeywordRollback(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRollback,
+		value:  caseShuffle("Rollback"),
+	}
+}
+func generateKeywordRow(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRow,
+		value:  caseShuffle("Row"),
+	}
+}
+func generateKeywordRows(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordRows,
+		value:  caseShuffle("Rows"),
+	}
+}
+func generateKeywordSavepoint(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordSavepoint,
+		value:  caseShuffle("Savepoint"),
+	}
+}
+func generateKeywordSelect(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordSelect,
+		value:  caseShuffle("Select"),
+	}
+}
+func generateKeywordSet(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordSet,
+		value:  caseShuffle("Set"),
+	}
+}
+func generateKeywordTable(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordTable,
+		value:  caseShuffle("Table"),
+	}
+}
+func generateKeywordTemp(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordTemp,
+		value:  caseShuffle("Temp"),
+	}
+}
+func generateKeywordTemporary(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordTemporary,
+		value:  caseShuffle("Temporary"),
+	}
+}
+func generateKeywordThen(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordThen,
+		value:  caseShuffle("Then"),
+	}
+}
+func generateKeywordTies(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordTies,
+		value:  caseShuffle("Ties"),
+	}
+}
+func generateKeywordTo(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordTo,
+		value:  caseShuffle("To"),
+	}
+}
+func generateKeywordTransaction(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordTransaction,
+		value:  caseShuffle("Transaction"),
+	}
+}
+func generateKeywordTrigger(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordTrigger,
+		value:  caseShuffle("Trigger"),
+	}
+}
+func generateKeywordUnbounded(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordUnbounded,
+		value:  caseShuffle("Unbounded"),
+	}
+}
+func generateKeywordUnion(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordUnion,
+		value:  caseShuffle("Union"),
+	}
+}
+func generateKeywordUnique(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordUnique,
+		value:  caseShuffle("Unique"),
+	}
+}
+func generateKeywordUpdate(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordUpdate,
+		value:  caseShuffle("Update"),
+	}
+}
+func generateKeywordUsing(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordUsing,
+		value:  caseShuffle("Using"),
+	}
+}
+func generateKeywordVacuum(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordVacuum,
+		value:  caseShuffle("Vacuum"),
+	}
+}
+func generateKeywordValues(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordValues,
+		value:  caseShuffle("Values"),
+	}
+}
+func generateKeywordView(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordView,
+		value:  caseShuffle("View"),
+	}
+}
+func generateKeywordVirtual(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordVirtual,
+		value:  caseShuffle("Virtual"),
+	}
+}
+func generateKeywordWhen(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordWhen,
+		value:  caseShuffle("When"),
+	}
+}
+func generateKeywordWhere(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordWhere,
+		value:  caseShuffle("Where"),
+	}
+}
+func generateKeywordWindow(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordWindow,
+		value:  caseShuffle("Window"),
+	}
+}
+func generateKeywordWith(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordWith,
+		value:  caseShuffle("With"),
+	}
+}
+func generateKeywordWithout(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.KeywordWithout,
+		value:  caseShuffle("Without"),
+	}
+}
+
+var alphabet = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ -_=+[]{};:'\\|,<.>/?!@#$%^&*()§±`~¡™£¢∞§¶•ªº–≠§“‘…æ«≤≥÷`∑´®†¥¨ˆøπåß∂ƒ©˙∆˚¬≈ç√∫˜µ")
+
+func generateLiteral(offset int) token.Token {
+	var buf bytes.Buffer
+	limit := rng.Intn(40)
+	for i := 0; i < limit; i++ {
+		buf.WriteRune(alphabet[rng.Intn(len(alphabet))])
+	}
+
+	return genTok{
+		offset: offset,
+		typ:    token.Literal,
+		value:  "\"" + buf.String() + "\"",
+	}
+}
+
+var unaryOperators = []string{"-", "+", "~"}
+
+func generateUnaryOperator(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.UnaryOperator,
+		value:  unaryOperators[rng.Intn(len(unaryOperators))],
+	}
+}
+
+var binaryOperators = []string{"||", "*", "/", "%", "+", "-", "<<", ">>", "&", "|", "<", "<=", ">", ">=", "=", "==", "!=", "<>"}
+
+func generateBinaryOperator(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.BinaryOperator,
+		value:  binaryOperators[rng.Intn(len(binaryOperators))],
+	}
+}
+func generateDelimiter(offset int) token.Token {
+	return genTok{
+		offset: offset,
+		typ:    token.Delimiter,
+	}
+}
+
+func caseShuffle(s string) string {
+
+	var buf bytes.Buffer
+	for _, x := range s {
+		n := rng.Intn(2)
+		r := unicode.ToLower(x)
+		if n == 0 {
+			r = unicode.ToUpper(x)
+		}
+		_, _ = buf.WriteRune(r)
+	}
+
+	return buf.String()
+}
+
+func generateTokenType() token.Type {
+	typ := token.Type(rng.Intn(int(token.Delimiter)))
+	if typ == token.Error || typ == token.Unknown || typ == token.EOF {
+		typ = token.StatementSeparator
+	}
+	return typ
+}
+
+func generateTokenForType(offset int, typ token.Type) token.Token {
+	switch typ {
+	case token.StatementSeparator:
+		return generateStatementSeparator(offset)
+	case token.KeywordAbort:
+		return generateKeywordAbort(offset)
+	case token.KeywordAction:
+		return generateKeywordAction(offset)
+	case token.KeywordAdd:
+		return generateKeywordAdd(offset)
+	case token.KeywordAfter:
+		return generateKeywordAfter(offset)
+	case token.KeywordAll:
+		return generateKeywordAll(offset)
+	case token.KeywordAlter:
+		return generateKeywordAlter(offset)
+	case token.KeywordAnalyze:
+		return generateKeywordAnalyze(offset)
+	case token.KeywordAnd:
+		return generateKeywordAnd(offset)
+	case token.KeywordAs:
+		return generateKeywordAs(offset)
+	case token.KeywordAsc:
+		return generateKeywordAsc(offset)
+	case token.KeywordAttach:
+		return generateKeywordAttach(offset)
+	case token.KeywordAutoincrement:
+		return generateKeywordAutoincrement(offset)
+	case token.KeywordBefore:
+		return generateKeywordBefore(offset)
+	case token.KeywordBegin:
+		return generateKeywordBegin(offset)
+	case token.KeywordBetween:
+		return generateKeywordBetween(offset)
+	case token.KeywordBy:
+		return generateKeywordBy(offset)
+	case token.KeywordCascade:
+		return generateKeywordCascade(offset)
+	case token.KeywordCase:
+		return generateKeywordCase(offset)
+	case token.KeywordCast:
+		return generateKeywordCast(offset)
+	case token.KeywordCheck:
+		return generateKeywordCheck(offset)
+	case token.KeywordCollate:
+		return generateKeywordCollate(offset)
+	case token.KeywordColumn:
+		return generateKeywordColumn(offset)
+	case token.KeywordCommit:
+		return generateKeywordCommit(offset)
+	case token.KeywordConflict:
+		return generateKeywordConflict(offset)
+	case token.KeywordConstraint:
+		return generateKeywordConstraint(offset)
+	case token.KeywordCreate:
+		return generateKeywordCreate(offset)
+	case token.KeywordCross:
+		return generateKeywordCross(offset)
+	case token.KeywordCurrent:
+		return generateKeywordCurrent(offset)
+	case token.KeywordCurrentDate:
+		return generateKeywordCurrentDate(offset)
+	case token.KeywordCurrentTime:
+		return generateKeywordCurrentTime(offset)
+	case token.KeywordCurrentTimestamp:
+		return generateKeywordCurrentTimestamp(offset)
+	case token.KeywordDatabase:
+		return generateKeywordDatabase(offset)
+	case token.KeywordDefault:
+		return generateKeywordDefault(offset)
+	case token.KeywordDeferrable:
+		return generateKeywordDeferrable(offset)
+	case token.KeywordDeferred:
+		return generateKeywordDeferred(offset)
+	case token.KeywordDelete:
+		return generateKeywordDelete(offset)
+	case token.KeywordDesc:
+		return generateKeywordDesc(offset)
+	case token.KeywordDetach:
+		return generateKeywordDetach(offset)
+	case token.KeywordDistinct:
+		return generateKeywordDistinct(offset)
+	case token.KeywordDo:
+		return generateKeywordDo(offset)
+	case token.KeywordDrop:
+		return generateKeywordDrop(offset)
+	case token.KeywordEach:
+		return generateKeywordEach(offset)
+	case token.KeywordElse:
+		return generateKeywordElse(offset)
+	case token.KeywordEnd:
+		return generateKeywordEnd(offset)
+	case token.KeywordEscape:
+		return generateKeywordEscape(offset)
+	case token.KeywordExcept:
+		return generateKeywordExcept(offset)
+	case token.KeywordExclude:
+		return generateKeywordExclude(offset)
+	case token.KeywordExclusive:
+		return generateKeywordExclusive(offset)
+	case token.KeywordExists:
+		return generateKeywordExists(offset)
+	case token.KeywordExplain:
+		return generateKeywordExplain(offset)
+	case token.KeywordFail:
+		return generateKeywordFail(offset)
+	case token.KeywordFilter:
+		return generateKeywordFilter(offset)
+	case token.KeywordFirst:
+		return generateKeywordFirst(offset)
+	case token.KeywordFollowing:
+		return generateKeywordFollowing(offset)
+	case token.KeywordFor:
+		return generateKeywordFor(offset)
+	case token.KeywordForeign:
+		return generateKeywordForeign(offset)
+	case token.KeywordFrom:
+		return generateKeywordFrom(offset)
+	case token.KeywordFull:
+		return generateKeywordFull(offset)
+	case token.KeywordGenerated:
+		return generateKeywordGenerated(offset)
+	case token.KeywordGlob:
+		return generateKeywordGlob(offset)
+	case token.KeywordGroup:
+		return generateKeywordGroup(offset)
+	case token.KeywordGroups:
+		return generateKeywordGroups(offset)
+	case token.KeywordHaving:
+		return generateKeywordHaving(offset)
+	case token.KeywordIf:
+		return generateKeywordIf(offset)
+	case token.KeywordIgnore:
+		return generateKeywordIgnore(offset)
+	case token.KeywordImmediate:
+		return generateKeywordImmediate(offset)
+	case token.KeywordIn:
+		return generateKeywordIn(offset)
+	case token.KeywordIndex:
+		return generateKeywordIndex(offset)
+	case token.KeywordIndexed:
+		return generateKeywordIndexed(offset)
+	case token.KeywordInitially:
+		return generateKeywordInitially(offset)
+	case token.KeywordInner:
+		return generateKeywordInner(offset)
+	case token.KeywordInsert:
+		return generateKeywordInsert(offset)
+	case token.KeywordInstead:
+		return generateKeywordInstead(offset)
+	case token.KeywordIntersect:
+		return generateKeywordIntersect(offset)
+	case token.KeywordInto:
+		return generateKeywordInto(offset)
+	case token.KeywordIs:
+		return generateKeywordIs(offset)
+	case token.KeywordIsnull:
+		return generateKeywordIsnull(offset)
+	case token.KeywordJoin:
+		return generateKeywordJoin(offset)
+	case token.KeywordKey:
+		return generateKeywordKey(offset)
+	case token.KeywordLast:
+		return generateKeywordLast(offset)
+	case token.KeywordLeft:
+		return generateKeywordLeft(offset)
+	case token.KeywordLike:
+		return generateKeywordLike(offset)
+	case token.KeywordLimit:
+		return generateKeywordLimit(offset)
+	case token.KeywordMatch:
+		return generateKeywordMatch(offset)
+	case token.KeywordNatural:
+		return generateKeywordNatural(offset)
+	case token.KeywordNo:
+		return generateKeywordNo(offset)
+	case token.KeywordNot:
+		return generateKeywordNot(offset)
+	case token.KeywordNothing:
+		return generateKeywordNothing(offset)
+	case token.KeywordNotnull:
+		return generateKeywordNotnull(offset)
+	case token.KeywordNull:
+		return generateKeywordNull(offset)
+	case token.KeywordNulls:
+		return generateKeywordNulls(offset)
+	case token.KeywordOf:
+		return generateKeywordOf(offset)
+	case token.KeywordOffset:
+		return generateKeywordOffset(offset)
+	case token.KeywordOn:
+		return generateKeywordOn(offset)
+	case token.KeywordOr:
+		return generateKeywordOr(offset)
+	case token.KeywordOrder:
+		return generateKeywordOrder(offset)
+	case token.KeywordOthers:
+		return generateKeywordOthers(offset)
+	case token.KeywordOuter:
+		return generateKeywordOuter(offset)
+	case token.KeywordOver:
+		return generateKeywordOver(offset)
+	case token.KeywordPartition:
+		return generateKeywordPartition(offset)
+	case token.KeywordPlan:
+		return generateKeywordPlan(offset)
+	case token.KeywordPragma:
+		return generateKeywordPragma(offset)
+	case token.KeywordPreceding:
+		return generateKeywordPreceding(offset)
+	case token.KeywordPrimary:
+		return generateKeywordPrimary(offset)
+	case token.KeywordQuery:
+		return generateKeywordQuery(offset)
+	case token.KeywordRaise:
+		return generateKeywordRaise(offset)
+	case token.KeywordRange:
+		return generateKeywordRange(offset)
+	case token.KeywordRecursive:
+		return generateKeywordRecursive(offset)
+	case token.KeywordReferences:
+		return generateKeywordReferences(offset)
+	case token.KeywordRegexp:
+		return generateKeywordRegexp(offset)
+	case token.KeywordReindex:
+		return generateKeywordReindex(offset)
+	case token.KeywordRelease:
+		return generateKeywordRelease(offset)
+	case token.KeywordRename:
+		return generateKeywordRename(offset)
+	case token.KeywordReplace:
+		return generateKeywordReplace(offset)
+	case token.KeywordRestrict:
+		return generateKeywordRestrict(offset)
+	case token.KeywordRight:
+		return generateKeywordRight(offset)
+	case token.KeywordRollback:
+		return generateKeywordRollback(offset)
+	case token.KeywordRow:
+		return generateKeywordRow(offset)
+	case token.KeywordRows:
+		return generateKeywordRows(offset)
+	case token.KeywordSavepoint:
+		return generateKeywordSavepoint(offset)
+	case token.KeywordSelect:
+		return generateKeywordSelect(offset)
+	case token.KeywordSet:
+		return generateKeywordSet(offset)
+	case token.KeywordTable:
+		return generateKeywordTable(offset)
+	case token.KeywordTemp:
+		return generateKeywordTemp(offset)
+	case token.KeywordTemporary:
+		return generateKeywordTemporary(offset)
+	case token.KeywordThen:
+		return generateKeywordThen(offset)
+	case token.KeywordTies:
+		return generateKeywordTies(offset)
+	case token.KeywordTo:
+		return generateKeywordTo(offset)
+	case token.KeywordTransaction:
+		return generateKeywordTransaction(offset)
+	case token.KeywordTrigger:
+		return generateKeywordTrigger(offset)
+	case token.KeywordUnbounded:
+		return generateKeywordUnbounded(offset)
+	case token.KeywordUnion:
+		return generateKeywordUnion(offset)
+	case token.KeywordUnique:
+		return generateKeywordUnique(offset)
+	case token.KeywordUpdate:
+		return generateKeywordUpdate(offset)
+	case token.KeywordUsing:
+		return generateKeywordUsing(offset)
+	case token.KeywordVacuum:
+		return generateKeywordVacuum(offset)
+	case token.KeywordValues:
+		return generateKeywordValues(offset)
+	case token.KeywordView:
+		return generateKeywordView(offset)
+	case token.KeywordVirtual:
+		return generateKeywordVirtual(offset)
+	case token.KeywordWhen:
+		return generateKeywordWhen(offset)
+	case token.KeywordWhere:
+		return generateKeywordWhere(offset)
+	case token.KeywordWindow:
+		return generateKeywordWindow(offset)
+	case token.KeywordWith:
+		return generateKeywordWith(offset)
+	case token.KeywordWithout:
+		return generateKeywordWithout(offset)
+	case token.Literal:
+		return generateLiteral(offset)
+	case token.UnaryOperator:
+		return generateUnaryOperator(offset)
+	case token.BinaryOperator:
+		return generateBinaryOperator(offset)
+	default:
+	}
+	return generateStatementSeparator(offset)
+}
+
+func generateToken(offset int) token.Token {
+	if rng.Intn(2) == 0 {
+		return generateTokenForType(offset, token.Literal)
+	}
+
+	typ := generateTokenType()
+	return generateTokenForType(offset, typ)
+}
+
+func generateScannerInputAndExpectedOutput() (scannerInput string, scannerOutput []token.Token) {
+
+	var buf bytes.Buffer
+
+	amountOfTokens := rng.Intn(200)
+	currentOffset := 0
+	for i := 0; i < amountOfTokens; i++ {
+		// generate token
+		tok := generateToken(currentOffset)
+		currentOffset += tok.Length() - 1
+
+		// append to results
+		buf.WriteString(tok.Value())
+		scannerOutput = append(scannerOutput, tok)
+
+		// generate whitespace
+		whitespaces := rng.Intn(5) + 1
+		currentOffset += whitespaces
+		for i := 0; i < whitespaces; i++ {
+			_, _ = buf.WriteRune(' ')
+		}
+	}
+
+	// EOF token
+	scannerOutput = append(scannerOutput, generateEOF(currentOffset))
+
+	scannerInput = buf.String()
+	return
+}
diff --git a/internal/parser/scanner/test/gen_test.go b/internal/parser/scanner/test/gen_test.go
new file mode 100644
index 0000000..b7a502e
--- /dev/null
+++ b/internal/parser/scanner/test/gen_test.go
@@ -0,0 +1,16 @@
+package scanner
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func Test_generateScannerInputAndExpectedOutput(t *testing.T) {
+	start := time.Now()
+	scIn, _ := generateScannerInputAndExpectedOutput()
+	fmt.Printf("took %v\n", time.Since(start).Round(time.Millisecond))
+
+	t.Log(scIn)
+	// t.Fail()
+}
diff --git a/internal/parser/simple_parser_test.go b/internal/parser/simple_parser_test.go
deleted file mode 100644
index 0c93d76..0000000
--- a/internal/parser/simple_parser_test.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package parser
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/tomarrell/lbadd/internal/parser/ast"
-	"github.com/tomarrell/lbadd/internal/parser/scanner"
-	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
-)
-
-var _ scanner.Scanner = (*_testScanner)(nil) // ensure that testScanner implements scanner.Scanner
-
-type _testScanner struct {
-	pos    int
-	tokens []token.Token
-}
-
-func scannerOf(tokens ...token.Token) *_testScanner {
-	return &_testScanner{tokens: tokens}
-}
-
-func (s *_testScanner) HasNext() bool {
-	return s.pos < len(s.tokens)
-}
-
-func (s *_testScanner) Next() token.Token {
-	tk := s.Peek()
-	s.pos++
-	return tk
-}
-
-func (s *_testScanner) Peek() token.Token {
-	return s.tokens[s.pos]
-}
-
-func Test_simpleParser_Next(t *testing.T) {
-	tests := []struct {
-		name   string
-		tokens []token.Token
-		stmt   *ast.SQLStmt
-		errs   []error
-		ok     bool
-	}{
-		{
-			"no tokens",
-			[]token.Token{},
-			nil,
-			[]error{},
-			false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			p := &simpleParser{
-				scanner: scannerOf(tt.tokens...),
-			}
-			stmt, errs, ok := p.Next()
-			if !reflect.DeepEqual(stmt, tt.stmt) {
-				t.Errorf("simpleParser.Next() stmt = %v, want %v", stmt, tt.stmt)
-			}
-			if !reflect.DeepEqual(errs, tt.errs) {
-				t.Errorf("simpleParser.Next() errs = %v, want %v", errs, tt.errs)
-			}
-			if ok != tt.ok {
-				t.Errorf("simpleParser.Next() ok = %v, want %v", ok, tt.ok)
-			}
-		})
-	}
-}
diff --git a/parser_test.go b/parser_test.go
deleted file mode 100644
index a01019d..0000000
--- a/parser_test.go
+++ /dev/null
@@ -1,54 +0,0 @@
-package lbadd
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParser(t *testing.T) {
-	cases := []struct {
-		name     string
-		sql      string
-		expected query
-		err      error
-	}{
-		// SELECT
-		{
-			name:     "select single field from table",
-			sql:      "SELECT a FROM z",
-			expected: query{queryType: selectQuery, fields: []string{"a"}, tableName: "z"},
-		},
-		{
-			name:     "select multiple fields from table",
-			sql:      "SELECT a, b, c FROM z",
-			expected: query{queryType: selectQuery, fields: []string{"a", "b", "c"}, tableName: "z"},
-		},
-		{
-			name:     "select with field and trailing comma error",
-			sql:      "SELECT a, b, c, FROM z",
-			expected: query{queryType: selectQuery, fields: []string{"a", "b", "c"}},
-			err:      fmt.Errorf("at SELECT: unexpected FROM after comma"),
-		},
-		{
-			name:     "select all (*) fields from table",
-			sql:      "SELECT * FROM z",
-			expected: query{queryType: selectQuery, fields: []string{"*"}, tableName: "z"},
-		},
-
-		// INSERT
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			actual, err := parse(tc.sql)
-			if tc.err != nil {
-				assert.EqualError(t, err, tc.err.Error())
-			} else {
-				assert.NoError(t, err)
-			}
-			assert.Equal(t, tc.expected, actual)
-		})
-	}
-}