Repository: tomarrell/lbadd
Pull request #31 (2020-01-21 11:47:19)

internal/parser: add SQLite grammar respecting parser
This PR adds an SQLite AST implementation and a parser interface to the master branch. With that code, `ast2ir` can be designed. The parser is just a stub, there is no functionality yet. The functionality will be developed on the branch `sqlite-grammar`.


Patch:
diff --git a/doc/internal/parser/scanner/scanner b/doc/internal/parser/scanner/scanner
new file mode 100644
index 0000000..e69de29
diff --git a/go.mod b/go.mod
index 5ac9bca..8be9d5b 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/stretchr/testify v1.4.0
+	golang.org/x/text v0.3.2
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v2 v2.2.7 // indirect
 )
diff --git a/go.sum b/go.sum
index 6773dc5..e93bcc3 100644
--- a/go.sum
+++ b/go.sum
@@ -12,6 +12,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
diff --git a/internal/parser/ast/ast.go b/internal/parser/ast/ast.go
new file mode 100644
index 0000000..bd41296
--- /dev/null
+++ b/internal/parser/ast/ast.go
@@ -0,0 +1,1 @@
+package ast
diff --git a/internal/parser/ast/doc.go b/internal/parser/ast/doc.go
new file mode 100644
index 0000000..3c4ad59
--- /dev/null
+++ b/internal/parser/ast/doc.go
@@ -0,0 +1,3 @@
+// Package ast implements an abstract syntax tree that represents SQL
+// statements.
+package ast
diff --git a/internal/parser/ast/statement.go b/internal/parser/ast/statement.go
new file mode 100644
index 0000000..95dafa0
--- /dev/null
+++ b/internal/parser/ast/statement.go
@@ -0,0 +1,829 @@
+package ast
+
+import (
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+// All AST nodes.
+type (
+	// SQLStmt as in the SQLite grammar.
+	SQLStmt struct {
+		Explain token.Token
+		Query   token.Token
+		Plan    token.Token
+
+		AlterTableStmt         *AlterTableStmt
+		AnalyzeStmt            *AnalyzeStmt
+		AttachStmt             *AttachStmt
+		BeginStmt              *BeginStmt
+		CommitStmt             *CommitStmt
+		CreateIndexStmt        *CreateIndexStmt
+		CreateTableStmt        *CreateTableStmt
+		CreateTriggerStmt      *CreateTriggerStmt
+		CreateViewStmt         *CreateViewStmt
+		CreateVirtualTableStmt *CreateVirtualTableStmt
+		DeleteStmt             *DeleteStmt
+		DeleteStmtLimited      *DeleteStmtLimited
+		DetachStmt             *DetachStmt
+		DropIndexStmt          *DropIndexStmt
+		DropTableStmt          *DropTableStmt
+		DropTriggerStmt        *DropTriggerStmt
+		DropViewStmt           *DropViewStmt
+		InsertStmt             *InsertStmt
+		ReindexStmt            *ReindexStmt
+		ReleaseStmt            *ReleaseStmt
+		RollbackStmt           *RollbackStmt
+		SavepointStmt          *SavepointStmt
+		SelectStmt             *SelectStmt
+		UpdateStmt             *UpdateStmt
+		UpdateStmtLimited      *UpdateStmtLimited
+		VacuumStmt             *VacuumStmt
+	}
+
+	// AlterTableStmt as in the SQLite grammar.
+	AlterTableStmt struct {
+		Alter         token.Token
+		Table         token.Token
+		SchemaName    token.Token
+		Period        token.Token
+		TableName     token.Token
+		Rename        token.Token
+		To            token.Token
+		NewTableName  token.Token
+		Column        token.Token
+		ColumnName    token.Token
+		NewColumnName token.Token
+		Add           token.Token
+		ColumnDef     *ColumnDef
+	}
+
+	// AnalyzeStmt as in the SQLite grammar.
+	AnalyzeStmt struct {
+		Analyze          token.Token
+		SchemaName       token.Token
+		TableOrIndexName token.Token
+		Period           token.Token
+	}
+
+	// AttachStmt as in the SQLite grammar.
+	AttachStmt struct {
+		Attach     token.Token
+		Database   token.Token
+		Expr       *Expr
+		As         token.Token
+		SchemaName token.Token
+	}
+
+	// BeginStmt as in the SQLite grammar.
+	BeginStmt struct {
+		Begin       token.Token
+		Deferred    token.Token
+		Immediate   token.Token
+		Exclusive   token.Token
+		Transaction token.Token
+	}
+
+	// CommitStmt as in the SQLite grammar.
+	CommitStmt struct {
+		Commit      token.Token
+		End         token.Token
+		Transaction token.Token
+	}
+
+	// CreateIndexStmt as in the SQLite grammar.
+	CreateIndexStmt struct {
+		Create         token.Token
+		Unique         token.Token
+		Index          token.Token
+		If             token.Token
+		Not            token.Token
+		Exists         token.Token
+		SchemaName     token.Token
+		Period         token.Token
+		IndexName      token.Token
+		On             token.Token
+		TableName      token.Token
+		LeftParen      token.Token
+		IndexedColumns []token.Token
+		RightParen     token.Token
+		Where          token.Token
+		Expr           *Expr
+	}
+
+	// CreateTableStmt as in the SQLite grammar.
+	CreateTableStmt struct {
+		Create          token.Token
+		Temp            token.Token
+		Temporary       token.Token
+		Table           token.Token
+		If              token.Token
+		Not             token.Token
+		Exists          token.Token
+		SchemaName      token.Token
+		Period          token.Token
+		TableName       token.Token
+		LeftParen       token.Token
+		ColumnDef       []*ColumnDef
+		TableConstraint []*TableConstraint
+		RightParen      token.Token
+		Without         token.Token
+		Rowid           token.Token
+		As              token.Token
+		SelectStmt      *SelectStmt
+	}
+
+	// CreateTriggerStmt as in the SQLite grammar.
+	CreateTriggerStmt struct {
+		Create      token.Token
+		Temp        token.Token
+		Temporary   token.Token
+		Trigger     token.Token
+		If          token.Token
+		Not         token.Token
+		Exists      token.Token
+		SchemaName  token.Token
+		Period      token.Token
+		TriggerName token.Token
+		Before      token.Token
+		After       token.Token
+		Instead     token.Token
+		Of1         token.Token
+		Delete      token.Token
+		Insert      token.Token
+		Update      token.Token
+		Of2         token.Token
+		ColumnName  []token.Token
+		On          token.Token
+		TableName   token.Token
+		For         token.Token
+		Each        token.Token
+		Row         token.Token
+		When        token.Token
+		Expr        *Expr
+		Begin       token.Token
+		UpdateStmt  []*UpdateStmt
+		InsertStmt  []*InsertStmt
+		DeleteStmt  []*DeleteStmt
+		SelectStmt  []*SelectStmt
+		End         token.Token
+	}
+
+	// CreateViewStmt as in the SQLite grammar.
+	CreateViewStmt struct {
+		Create     token.Token
+		Temp       token.Token
+		Temporary  token.Token
+		View       token.Token
+		If         token.Token
+		Not        token.Token
+		Exists     token.Token
+		SchemaName token.Token
+		Period     token.Token
+		ViewName   token.Token
+		LeftParen  token.Token
+		ColumnName []token.Token
+		RightParen token.Token
+		As         token.Token
+		SelectStmt *SelectStmt
+	}
+
+	// CreateVirtualTableStmt as in the SQLite grammar.
+	CreateVirtualTableStmt struct {
+		Create         token.Token
+		Virtual        token.Token
+		Table          token.Token
+		If             token.Token
+		Not            token.Token
+		Exists         token.Token
+		SchemaName     token.Token
+		Period         token.Token
+		TableName      token.Token
+		Using          token.Token
+		ModuleName     token.Token
+		LeftParen      token.Token
+		ModuleArgument []token.Token
+		RightParen     token.Token
+	}
+
+	// DeleteStmt as in the SQLite grammar.
+	DeleteStmt struct {
+		WithClause         *WithClause
+		Delete             token.Token
+		From               token.Token
+		QualifiedTableName *QualifiedTableName
+		Where              token.Token
+		Expr               *Expr
+	}
+
+	// DeleteStmtLimited as in the SQLite grammar.
+	DeleteStmtLimited struct {
+		*DeleteStmt
+		Order        token.Token
+		By           token.Token
+		OrderingTerm []*OrderingTerm
+		Limit        token.Token
+		Expr2        *Expr
+		Offset       token.Token
+		Comma        token.Token
+		Expr3        *Expr
+	}
+
+	// DetachStmt as in the SQLite grammar.
+	DetachStmt struct {
+		Detach     token.Token
+		Database   token.Token
+		SchemaName token.Token
+	}
+
+	// DropIndexStmt as in the SQLite grammar.
+	DropIndexStmt struct {
+		Drop       token.Token
+		Index      token.Token
+		If         token.Token
+		Exists     token.Token
+		SchemaName token.Token
+		Period     token.Token
+		IndexName  token.Token
+	}
+
+	// DropTableStmt as in the SQLite grammar.
+	DropTableStmt struct {
+		Drop       token.Token
+		Table      token.Token
+		If         token.Token
+		Exists     token.Token
+		SchemaName token.Token
+		Period     token.Token
+		TableName  token.Token
+	}
+
+	// DropTriggerStmt as in the SQLite grammar.
+	DropTriggerStmt struct {
+		Drop        token.Token
+		Trigger     token.Token
+		If          token.Token
+		Exists      token.Token
+		SchemaName  token.Token
+		Period      token.Token
+		TriggerName token.Token
+	}
+
+	// DropViewStmt as in the SQLite grammar.
+	DropViewStmt struct {
+		Drop       token.Token
+		View       token.Token
+		If         token.Token
+		Exists     token.Token
+		SchemaName token.Token
+		Period     token.Token
+		ViewName   token.Token
+	}
+
+	// QualifiedTableName as in the SQLite grammar.
+	QualifiedTableName struct {
+		SchemaName token.Token
+		Period     token.Token
+		TableName  token.Token
+		As         token.Token
+		Alias      token.Token
+		Not        token.Token
+		Indexed    token.Token
+		By         token.Token
+		IndexName  token.Token
+	}
+
+	// InsertStmt as in the SQLite grammar.
+	InsertStmt struct {
+		WithClause               *WithClause
+		Insert                   token.Token
+		Or                       token.Token
+		Replace                  token.Token
+		Rollback                 token.Token
+		Abort                    token.Token
+		Fail                     token.Token
+		Ignore                   token.Token
+		Into                     token.Token
+		SchemaName               token.Token
+		Period                   token.Token
+		TableName                token.Token
+		As                       token.Token
+		Alias                    token.Token
+		LeftParen1               token.Token
+		ColumnName               []token.Token
+		RightParen2              token.Token
+		Default                  token.Token
+		Values                   token.Token
+		SelectStmt               *SelectStmt
+		ParenthesizedExpressions *[]ParenthesizedExpressions
+		UpsertClause             *UpsertClause
+	}
+
+	// ParenthesizedExpressions as in the SQLite grammar.
+	ParenthesizedExpressions struct {
+		LeftParen  token.Token
+		Exprs      []*Expr
+		RightParen token.Token
+	}
+
+	// ReindexStmt as in the SQLite grammar.
+	ReindexStmt struct {
+		Reindex          token.Token
+		CollationName    token.Token
+		SchemaName       token.Token
+		Period           token.Token
+		TableOrIndexName token.Token
+	}
+
+	// SavepointStmt as in the SQLite grammar.
+	SavepointStmt struct {
+		Savepoint     token.Token
+		SavepointName token.Token
+	}
+
+	// ReleaseStmt as in the SQLite grammar.
+	ReleaseStmt struct {
+		Release       token.Token
+		Savepoint     token.Token
+		SavepointName token.Token
+	}
+
+	// RollbackStmt as in the SQLite grammar.
+	RollbackStmt struct {
+		Rollback      token.Token
+		Transaction   token.Token
+		To            token.Token
+		Savepoint     token.Token
+		SavepointName token.Token
+	}
+
+	// SelectStmt as in the SQLite grammar.
+	SelectStmt struct {
+		With                  token.Token
+		Recursive             token.Token
+		CommonTableExpression []*CommonTableExpression
+		SelectCore            []*SelectCore
+		Order                 token.Token
+		By                    token.Token
+		OrderingTerm          []*OrderingTerm
+		Limit                 token.Token
+		Expr1                 *Expr
+		Offset                token.Token
+		Comma                 token.Token
+		Expr2                 *Expr
+	}
+
+	// SelectCore as in the SQLite grammar.
+	SelectCore struct {
+		Select                   token.Token
+		Distinct                 token.Token
+		All                      token.Token
+		ResultColumn             []*ResultColumn
+		From                     token.Token
+		JoinClause               *JoinClause
+		TableOrSubquery          []*TableOrSubquery
+		Where                    token.Token
+		Expr1                    *Expr
+		Group                    token.Token
+		By                       token.Token
+		Expr2                    []*Expr
+		Having                   token.Token
+		Expr3                    *Expr
+		Window                   token.Token
+		NamedWindow              []*NamedWindow
+		Values                   token.Token
+		ParenthesizedExpressions []*ParenthesizedExpressions
+		CompoundOperator         *CompoundOperator
+	}
+
+	// UpdateStmt as in the SQLite grammar.
+	UpdateStmt struct {
+		WithClause         *WithClause
+		Update             token.Token
+		Or                 token.Token
+		Rollback           token.Token
+		Abort              token.Token
+		Replace            token.Token
+		Fail               token.Token
+		Ignore             token.Token
+		QualifiedTableName *QualifiedTableName
+		Set                token.Token
+		UpdateSetter       []*UpdateSetter
+		Where              token.Token
+		Expr               *Expr
+	}
+
+	// UpdateSetter as in the SQLite grammar.
+	UpdateSetter struct {
+		ColumnName     token.Token
+		ColumnNameList *ColumnNameList
+		Assign         token.Token
+		Expr           *Expr
+	}
+
+	// UpdateStmtLimited as in the SQLite grammar.
+	UpdateStmtLimited struct {
+		*UpdateStmt
+		Order        token.Token
+		By           token.Token
+		OrderingTerm []token.Token
+		Limit        token.Token
+		Expr1        *Expr
+		Offset       token.Token
+		Comma        token.Token
+		Expr2        *Expr
+	}
+
+	// UpsertClause as in the SQLite grammar.
+	UpsertClause struct {
+		On            token.Token
+		Conflict      token.Token
+		LeftParen     token.Token
+		IndexedColumn []*IndexedColumn
+		RightParen    token.Token
+		Where1        token.Token
+		Expr1         *Expr
+		Do            token.Token
+		Nothing       token.Token
+		Update        token.Token
+		Set           token.Token
+		UpdateSetter  []*UpdateSetter
+		Where2        token.Token
+		Expr2         *Expr
+	}
+
+	// VacuumStmt as in the SQLite grammar.
+	VacuumStmt struct {
+		Vacuum     token.Token
+		SchemaName token.Token
+		Into       token.Token
+		Filename   token.Token
+	}
+
+	// WithClause as in the SQLite grammar.
+	WithClause struct {
+		With         token.Token
+		Recursive    token.Token
+		RecursiveCte []*RecursiveCte
+	}
+
+	// RecursiveCte as in the SQLite grammar.
+	RecursiveCte struct {
+		CteTableName *CteTableName
+		As           token.Token
+		LeftParen    token.Token
+		SelectStmt   *SelectStmt
+		RightParen   token.Token
+	}
+
+	// CteTableName as in the SQLite grammar.
+	CteTableName struct {
+		TableName  token.Token
+		LeftParen  token.Token
+		ColumnName []token.Token
+		RightParen token.Token
+	}
+
+	// Expr as in the SQLite grammar.
+	Expr struct {
+		LiteralValue   token.Token
+		BindParameter  token.Token
+		SchemaName     token.Token
+		Period1        token.Token
+		TableName      token.Token
+		Period2        token.Token
+		ColumnName     token.Token
+		UnaryOperator  token.Token
+		Expr1          *Expr
+		BinaryOperator token.Token
+		Expr2          *Expr
+		FunctionName   token.Token
+		LeftParen      token.Token
+		Asterisk       token.Token
+		Distinct       token.Token
+		Expr           []*Expr
+		RightParen     token.Token
+		FilterClause   *FilterClause
+		OverClause     *OverClause
+		Cast           token.Token
+		As             token.Token
+		Collate        token.Token
+		CollationName  token.Token
+		Not            token.Token
+		Like           token.Token
+		Glob           token.Token
+		Regexp         token.Token
+		Match          token.Token
+		Escape         token.Token
+		Expr3          *Expr
+		Isnull         token.Token
+		Notnull        token.Token
+		Null           token.Token
+		Is             token.Token
+		Between        token.Token
+		In             token.Token
+		SelectStmt     *SelectStmt
+		TableFunction  token.Token
+		Exists         token.Token
+		Case           token.Token
+		When           token.Token
+		Then           token.Token
+		Else           token.Token
+		Expr4          *Expr
+		End            token.Token
+		RaiseFunction  *RaiseFunction
+	}
+
+	// FilterClause as in the SQLite grammar.
+	FilterClause struct {
+		Filter     token.Token
+		LeftParen  token.Token
+		Where      token.Token
+		Expr       *Expr
+		RightParen token.Token
+	}
+
+	// OverClause as in the SQLite grammar.
+	OverClause struct {
+		Over           token.Token
+		WindowName     token.Token
+		LeftParen      token.Token
+		BaseWindowName token.Token
+		Partition      token.Token
+		By             token.Token
+		Expr           []*Expr
+		Order          token.Token
+		OrderingTerm   []*OrderingTerm
+		FrameSpec      *FrameSpec
+		RightParen     token.Token
+	}
+
+	// RaiseFunction as in the SQLite grammar.
+	RaiseFunction struct {
+		Raise        token.Token
+		LeftParen    token.Token
+		Ignore       token.Token
+		Rollback     token.Token
+		Abort        token.Token
+		Fail         token.Token
+		Comma        token.Token
+		ErrorMessage token.Token
+		RightParen   token.Token
+	}
+
+	// OrderingTerm as in the SQLite grammar.
+	OrderingTerm struct {
+		Expr          *Expr
+		Collate       token.Token
+		CollationName token.Token
+		Asc           token.Token
+		Desc          token.Token
+		Nulls         token.Token
+		First         token.Token
+		Last          token.Token
+	}
+
+	// ResultColumn as in the SQLite grammar.
+	ResultColumn struct {
+		Expr        *Expr
+		As          token.Token
+		ColumnAlias token.Token
+		Asterisk    token.Token
+		TableName   token.Token
+		Period      token.Token
+	}
+
+	// NamedWindow as in the SQLite grammar.
+	NamedWindow struct {
+		WindowName token.Token
+		As         token.Token
+		WindowDefn *WindowDefn
+	}
+
+	// WindowDefn as in the SQLite grammar.
+	WindowDefn struct {
+		LeftParen      token.Token
+		BaseWindowName token.Token
+		Partition      token.Token
+		By             token.Token
+		Expr           []*Expr
+		Order          token.Token
+		OrderingTerm   []*OrderingTerm
+		FrameSpec      *FrameSpec
+		RightParen     token.Token
+	}
+
+	// FrameSpec as in the SQLite grammar.
+	FrameSpec struct {
+		Range      token.Token
+		Rows       token.Token
+		Groups     token.Token
+		Between    token.Token
+		Unbounded1 token.Token
+		Preceding1 token.Token
+		Expr1      *Expr
+		Current1   token.Token
+		Row1       token.Token
+		Following1 token.Token
+		And        token.Token
+		Expr2      *Expr
+		Preceding2 token.Token
+		Current2   token.Token
+		Row2       token.Token
+		Following2 token.Token
+		Unbounded2 token.Token
+		Exclude    token.Token
+		No         token.Token
+		Others     token.Token
+		Current3   token.Token
+		Row3       token.Token
+		Group      token.Token
+		Ties       token.Token
+	}
+
+	// TableConstraint as in the SQLite grammar.
+	TableConstraint struct {
+		Constraint       token.Token
+		Name             token.Token
+		Primary          token.Token
+		Key              token.Token
+		Unique           token.Token
+		LeftParen        token.Token
+		RightParen       token.Token
+		IndexedColumn    []*IndexedColumn
+		ConflictClause   *ConflictClause
+		Check            token.Token
+		Expr             *Expr
+		Foreign          token.Token
+		ColumnName       []token.Token
+		ForeignKeyClause *ForeignKeyClause
+	}
+
+	// ForeignKeyClause as in the SQLite grammar.
+	ForeignKeyClause struct {
+		References   token.Token
+		ForeignTable token.Token
+		LeftParen    token.Token
+		ColumnName   []token.Token
+		RightParen   token.Token
+		On           token.Token
+		Delete       token.Token
+		Update       token.Token
+		Set          token.Token
+		Null         token.Token
+		Default      token.Token
+		Cascade      token.Token
+		Restrict     token.Token
+		No           token.Token
+		Action       token.Token
+		Match        token.Token
+		Name         token.Token
+		Not          token.Token
+		Deferrable   token.Token
+		Initially    token.Token
+		Deferred     token.Token
+		Immediate    token.Token
+	}
+
+	// CommonTableExpression as in the SQLite grammar.
+	CommonTableExpression struct {
+		TableName   token.Token
+		LeftParen1  token.Token
+		ColumnName  []token.Token
+		RightParen1 token.Token
+		As          token.Token
+		LeftParen2  token.Token
+		SelectStmt  *SelectStmt
+		RightParen2 token.Token
+	}
+
+	// CompoundOperator as in the SQLite grammar.
+	CompoundOperator struct {
+		Union     token.Token
+		All       token.Token
+		Intersect token.Token
+		Except    token.Token
+	}
+
+	// TableOrSubquery as in the SQLite grammar.
+	TableOrSubquery struct {
+		SchemaName        token.Token
+		Period            token.Token
+		TableName         token.Token
+		As                token.Token
+		TableAlias        token.Token
+		Not               token.Token
+		Indexed           token.Token
+		By                token.Token
+		IndexName         token.Token
+		TableFunctionName token.Token
+		LeftParen         token.Token
+		Expr              []*Expr
+		RightParen        token.Token
+		JoinClause        *JoinClause
+		TableOrSubquery   []*TableOrSubquery
+		SelectStmt        *SelectStmt
+	}
+
+	// JoinClause as in the SQLite grammar.
+	JoinClause struct {
+		TableOrSubquery *TableOrSubquery
+		JoinClausePart  *JoinClausePart
+	}
+
+	// JoinClausePart as in the SQLite grammar.
+	JoinClausePart struct {
+		JoinOperator    *JoinOperator
+		TableOrSubquery *TableOrSubquery
+		JoinConstraint  *JoinConstraint
+	}
+
+	// JoinConstraint as in the SQLite grammar.
+	JoinConstraint struct {
+		On         token.Token
+		Expr       *Expr
+		Using      token.Token
+		LeftParen  token.Token
+		ColumnName []token.Token
+		RightParen token.Token
+	}
+
+	// JoinOperator as in the SQLite grammar.
+	JoinOperator struct {
+		Comma   token.Token
+		Natural token.Token
+		Left    token.Token
+		Outer   token.Token
+		Inner   token.Token
+		Cross   token.Token
+		Join    token.Token
+	}
+
+	// ColumnDef as in the SQLite grammar.
+	ColumnDef struct {
+		ColumnName       token.Token
+		ColumnConstraint []*ColumnConstraint
+	}
+
+	// ColumnConstraint as in the SQLite grammar.
+	ColumnConstraint struct {
+		Constraint       token.Token
+		Name             token.Token
+		Primary          token.Token
+		Key              token.Token
+		Asc              token.Token
+		Desc             token.Token
+		ConflictClause   *ConflictClause
+		Autoincrement    token.Token
+		Not              token.Token
+		Null             token.Token
+		Unique           token.Token
+		Check            token.Token
+		LeftParen        token.Token
+		Expr             *Expr
+		RightParen       token.Token
+		Default          token.Token
+		SignedNumber     token.Token
+		LiteralValue     token.Token
+		Collate          token.Token
+		CollationName    token.Token
+		ForeignKeyClause *ForeignKeyClause
+	}
+
+	// ColumnNameList as in the SQLite grammar.
+	ColumnNameList struct {
+		LeftParen  token.Token
+		ColumnName []token.Token
+		RightParen token.Token
+	}
+
+	// ConflictClause as in the SQLite grammar.
+	ConflictClause struct {
+		On       token.Token
+		Conflict token.Token
+		Rollback token.Token
+		Abort    token.Token
+		Fail     token.Token
+		Ignore   token.Token
+		Replace  token.Token
+	}
+
+	// TypeName as in the SQLite grammar.
+	TypeName struct {
+		Name          []token.Token
+		LeftParen     token.Token
+		SignedNumber1 token.Token
+		Comma         token.Token
+		SignedNumber2 token.Token
+		RightParen    token.Token
+	}
+
+	// IndexedColumn as in the SQLite grammar.
+	IndexedColumn struct {
+		ColumnName    token.Token
+		Expr          *Expr
+		Collate       token.Token
+		CollationName token.Token
+		Asc           token.Token
+		Desc          token.Token
+	}
+)
diff --git a/internal/parser/doc.go b/internal/parser/doc.go
new file mode 100644
index 0000000..ac99a4b
--- /dev/null
+++ b/internal/parser/doc.go
@@ -0,0 +1,3 @@
+// Package parser implements a SQL parser with respect to the SQLite grammar.
+// https://www.sqlite.org/lang.html
+package parser
diff --git a/internal/parser/parser.go b/internal/parser/parser.go
new file mode 100644
index 0000000..7842165
--- /dev/null
+++ b/internal/parser/parser.go
@@ -0,0 +1,37 @@
+package parser
+
+import (
+	"github.com/tomarrell/lbadd/internal/parser/ast"
+)
+
+type sentinel string
+
+func (s sentinel) Error() string { return string(s) }
+
+// parser errors
+const (
+	ErrIncompleteStatement  = sentinel("incomplete statement")
+	ErrPrematureEOF         = sentinel("unexpectedly reached EOF")
+	ErrUnexpectedToken      = sentinel("unexpected token")
+	ErrUnknownToken         = sentinel("unknown token")
+	ErrUnsupportedConstruct = sentinel("unsupported construct")
+)
+
+// Parser describes a parser that returns (maybe multiple) SQLStatements from a
+// given input.
+type Parser interface {
+	// Next returns stmt=<statement>, errs=nil, ok=true if a statement was
+	// parsed successfully without any parse errors. If there were parse errors,
+	// Next will return stmt=<statement>, errs=([]error), ok=true.
+	//
+	// stmt always is the statement that was parsed. If it could not be parsed
+	// free of errors, the statement might be incomplete or incorrect, but
+	// efforts will be taken to parse as much out of the given input as
+	// possible. ok indicates whether any statement could have been parsed, or
+	// more precisely, if the underlying scanner had any more tokens.
+	//
+	// If ok=false, that means that the parser has reached its EOF and no more
+	// statements can be returned. Subsequent calls to Next will result in
+	// stmt=nil, errs=nil, ok=false.
+	Next() (stmt *ast.SQLStmt, errs []error, ok bool)
+}
diff --git a/internal/parser/scanner/doc.go b/internal/parser/scanner/doc.go
new file mode 100644
index 0000000..04800cf
--- /dev/null
+++ b/internal/parser/scanner/doc.go
@@ -0,0 +1,3 @@
+// Package scanner implements a tokenizer that creates tokens of the SQLite
+// grammar.
+package scanner
diff --git a/internal/parser/scanner/matcher/doc.go b/internal/parser/scanner/matcher/doc.go
new file mode 100644
index 0000000..32e51a5
--- /dev/null
+++ b/internal/parser/scanner/matcher/doc.go
@@ -0,0 +1,3 @@
+// Package matcher implements a matcher component, that can determine whether or
+// not a rune is part of a custom character class.
+package matcher
\ No newline at end of file
diff --git a/internal/parser/scanner/matcher/matcher.go b/internal/parser/scanner/matcher/matcher.go
new file mode 100644
index 0000000..1e3dc76
--- /dev/null
+++ b/internal/parser/scanner/matcher/matcher.go
@@ -0,0 +1,66 @@
+package matcher
+
+import (
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/unicode/rangetable"
+)
+
+// M is the definition of a character class, which can tell whether a rune is
+// part of that character definition or not.
+type M struct {
+	rt   *unicode.RangeTable
+	desc string
+}
+
+// Matches describes, whether the given rune is contained in this matcher's
+// range table.
+func (m M) Matches(r rune) bool { return unicode.Is(m.rt, r) }
+
+// String returns a human readable string description of the runes that this
+// matcher matches.
+func (m M) String() string { return m.desc }
+
+// Matcher constructors
+
+// New creates a new matcher from a given match function.
+func New(desc string, rt *unicode.RangeTable) M {
+	return M{rt, desc}
+}
+
+// String creates a matcher that checks whether a rune is part of the given
+// string.
+func String(s string) M {
+	return M{
+		rt:   rangetable.New([]rune(s)...),
+		desc: "one of '" + s + "'",
+	}
+}
+
+// RuneWithDesc creates a matcher that matches only the given rune. The
+// description is the string representation of this matcher. This is useful when
+// dealing with whitespace characters.
+func RuneWithDesc(desc string, exp rune) M {
+	return M{
+		rt:   rangetable.New(exp),
+		desc: desc,
+	}
+}
+
+// Merge creates a new matcher, that accepts runes that are matched by one or
+// more of the given matchers.
+func Merge(ms ...M) M {
+	var rtms []*unicode.RangeTable
+	descs := make([]string, len(ms))
+
+	for i, m := range ms {
+		descs[i] = m.String()
+		rtms = append(rtms, m.rt)
+	}
+
+	return M{
+		rt:   rangetable.Merge(rtms...),
+		desc: strings.Join(descs, " or "),
+	}
+}
diff --git a/internal/parser/scanner/scanner.go b/internal/parser/scanner/scanner.go
new file mode 100644
index 0000000..bb9cd59
--- /dev/null
+++ b/internal/parser/scanner/scanner.go
@@ -0,0 +1,300 @@
+package scanner
+
+import (
+	"fmt"
+
+	"github.com/tomarrell/lbadd/internal/parser/scanner/matcher"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+// Scanner is the interface that describes a scanner.
+type Scanner interface {
+	HasNext() bool
+	Next() token.Token
+	Peek() token.Token
+}
+
+type scanner struct {
+	input          []rune
+	start          int
+	pos, line, col int
+	closed         bool
+}
+
+// checkpoint represents the state of a scanner at any given point in time. A
+// scanner can be restored to a checkpoint.
+//
+//	var s *Scanner
+//	...
+//	chck := s.checkpoint() // create a checkpoint
+//	...                     // accept(), next(), goback(), whatever
+//	s.restore(chck)        // scanner is in the same state as when the checkpoint was created
+//
+// This is useful when a state should not return an error if something
+// unexpected was read, but for example another grammar production should be
+// tried. To guarantee that both tries start with the same scanner state, a
+// checkpoint can be used.
+type checkpoint struct {
+	start int
+	pos   int
+
+	startLine, startCol int
+	line, lastCol, col  int
+}
+
+// New returns a new scanner
+func New(input []rune) Scanner {
+	return &scanner{
+		input: input,
+		start: 0,
+		pos:   0,
+
+		closed: false,
+	}
+}
+
+// HasNext checks for existance of next token, returns true if exists, false otherwise.
+func (s *scanner) HasNext() bool {
+	if s.done() {
+		return false
+	}
+	return true
+}
+
+// Next reads the next token. This is basically starting from the initial state until a
+// token gets emitted. If an error occurs, simply return an error token.
+func (s *scanner) Next() token.Token {
+	// if s.done() {
+	// 	return token.New(s.line, s.col, s.pos-s.start, s.pos, token.Error, fmt.Sprintf("Scanner closed. Can't read from it."))
+	// }
+	switch s.peek() {
+	case 'A':
+		return scanAKeyword(s)
+	case 'B':
+		return scanBKeyword(s)
+	case 'C':
+		return scanCKeyword(s)
+	case 'D':
+		return scanDKeyword(s)
+	case 'E':
+		return scanEKeyword(s)
+	case 'F':
+		return scanFKeyword(s)
+	case 'G':
+		return scanGKeyword(s)
+	case 'H':
+		return scanHKeyword(s)
+	case 'I':
+		return scanIKeyword(s)
+	case 'J':
+		return scanJKeyword(s)
+	case 'K':
+		return scanKKeyword(s)
+	case 'L':
+		return scanLKeyword(s)
+	case 'M':
+		return scanMKeyword(s)
+	case 'N':
+		return scanNKeyword(s)
+	case 'O':
+		return scanOKeyword(s)
+	case 'P':
+		return scanPKeyword(s)
+	case 'Q':
+		return scanQKeyword(s)
+	case 'R':
+		return scanRKeyword(s)
+	case 'S':
+		return scanSKeyword(s)
+	case 'T':
+		return scanTKeyword(s)
+	case 'U':
+		return scanUKeyword(s)
+	case 'V':
+		return scanVKeyword(s)
+	// case ' ':
+	// 	return scanSpace(s)
+	// case '"':
+	// 	return scanDoubleQuote(s)
+	// case '%':
+	// 	return scanPercent(s)
+	// case '&':
+	// 	return scanAmpersand(s)
+	// case '\'':
+	// 	return scanQuote(s)
+	// case '(':
+	// 	return scanLeftParanthesis(s)
+	// case ')':
+	// 	return scanRightParanthesis(s)
+	// case '*':
+	// 	return scanAsterisk(s)
+	// case '+':
+	// 	return scanPlusSign(s)
+	// case ',':
+	// 	return scanComma(s)
+	// case '-':
+	// 	return scanMinusSign(s)
+	// case '.':
+	// 	return scanPeriod(s)
+	// case '/':
+	// 	return scanSolidus(s)
+	// case '\\':
+	// 	return scanReverseSolidus(s)
+	// case ':':
+	// 	return scanColon(s)
+	// case ';':
+	// 	return scanSemiColon(s)
+	// case '<':
+	// 	return scanLessThanOperator(s)
+	// case '=':
+	// 	return scanEqualsOperator(s)
+	// case '>':
+	// 	return scanGreaterThanOperator(s)
+	// case '?':
+	// 	return scanQuestioMarkOrTrigraphs(s)
+	// case '[':
+	// 	return scanLeftBracket(s)
+	// case ']':
+	// 	return scanRightBracket(s)
+	// case '^':
+	// 	return scanCircumflex(s)
+	// case '_':
+	// 	return scanUnderscore(s)
+	// case '|':
+	// 	return scanVerticalBar(s)
+	// case '{':
+	// 	return scanLeftBrace(s)
+	// case '}':
+	// 	return scanRightBrace(s)
+	// case '$':
+	// 	return scanDollarSign(s)
+	default:
+		fmt.Println("TBI")
+	}
+	return nil
+}
+
+func (s *scanner) Peek() token.Token {
+	panic("implement me")
+}
+
+// Close will cause this scanner to not execute any more states. The execution
+// of the current state cannot be aborted, but the scanner will stop executing
+// states after the current state has finished.
+func (s *scanner) Close() error {
+	s.closed = true
+	return nil
+}
+
+// done determines whether the scanner is done with its work. This is the case,
+// if either the scanner was closed, or the scanner has reached the end of its
+// input.
+func (s *scanner) done() bool {
+	return s.closed ||
+		s.pos >= len(s.input)
+}
+
+// next returns the next rune of the scanners input and advances its pointer by
+// one position. This method also updates the line and col information of the
+// scanner. If the scanner.done()==true and this method is called, it will panic
+// with a syntax error.
+//
+// The process of advancing the pointer and returning the read rune is called
+// "consuming a rune" or "accepting a rune".
+func (s *scanner) next() rune {
+	// get the actual next rune
+	next := s.input[s.pos]
+	if next == '\n' {
+		s.line++
+		s.col = 1
+	}
+	// update current scanner position
+	s.pos++
+
+	return next
+}
+
+// peek returns the next rune of the scanners input without consuming it.
+func (s *scanner) peek() rune {
+	return s.input[s.pos]
+}
+
+// goback decrements the scanner's position by one and updates its line and col
+// information.
+func (s *scanner) goback() {
+	s.pos--
+}
+
+// ignore discards all accepted runes. This is done by simply setting the start
+// position of the last read token to the current scanner position.
+func (s *scanner) ignore() {
+	s.start = s.pos
+}
+
+// accept accepts exactly one rune matched by the given matcher. This means,
+// that: If the next rune is matched by the scanner, it is consumed and ok=true
+// is returned. If the next rune is NOT matched, it is unread and ok=false is
+// returned. This implies, that accept(Alphanumeric) will actually do nothing if
+// the next rune is not Alphanumeric. However, if the next rune is Alphanumeric,
+// it will be accepted.
+func (s *scanner) accept(m matcher.M) bool {
+	if m.Matches(s.next()) {
+		return true
+	}
+	s.goback()
+	return false
+}
+
+// acceptMultiple accepts multiple runes that are matched by the given matcher.
+// See the godoc on (*scanner).accept for more information. The amount of
+// matched runes is returned.
+func (s *scanner) acceptMultiple(m matcher.M) (matched uint) {
+	for s.accept(m) {
+		matched++
+	}
+	return
+}
+
+// acceptString accepts the exact sequence of runes that the given string
+// represents, or does nothing, if the string is not matched.
+//
+//	input := []rune(".hello")
+//	...
+//	s.acceptString("hello") // will do nothing, as the next rune is '.'
+//	s.next()                // advance the position by one (next rune is now 'h')
+//	s.acceptString("hello") // will accept 5 runes, the scanner has reached its EOF now
+func (s *scanner) acceptString(str string) bool {
+	if s.peekString(str) {
+		s.pos += len(str)
+		return true
+	}
+	return false
+}
+
+// peekString works like (*scanner).acceptString, except it doesn't consume any
+// runes. It just peeks, if the given string lays ahead.
+func (s *scanner) peekString(str string) bool {
+	for i, r := range str {
+		if r != s.input[s.pos+i] {
+			return false
+		}
+	}
+	return true
+}
+
+// createToken creates a token with given parameters
+func createToken(line, col, start, pos int, t token.Type, value string, s *scanner) token.Token {
+	token := token.New(line, col, start, pos-start, t, value)
+	s.start = pos
+	return token
+}
+
+// seekNext returns the position of the end of a keyword.
+// It takes the start position of the keyword.
+func (s *scanner) seekNext(start int) int {
+	start++
+	for s.input[start] != ' ' {
+		start++
+	}
+	return start
+}
diff --git a/internal/parser/scanner/states.go b/internal/parser/scanner/states.go
new file mode 100644
index 0000000..78bec70
--- /dev/null
+++ b/internal/parser/scanner/states.go
@@ -0,0 +1,586 @@
+package scanner
+
+import (
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+var keywordsWithA map[string]token.Type = map[string]token.Type{
+	"ABORT":          token.KeywordAbort,
+	"ACTION":         token.KeywordAction,
+	"ADD":            token.KeywordAdd,
+	"AFTER":          token.KeywordAdd,
+	"ALL":            token.KeywordAll,
+	"ALTER":          token.KeywordAlter,
+	"ANALYZE":        token.KeywordAnalyze,
+	"AND":            token.KeywordAnd,
+	"AS":             token.KeywordAnd,
+	"ASC":            token.KeywordAsc,
+	"ATTACH":         token.KeywordAttach,
+	"AUTO INCREMENT": token.KeywordAutoincrement,
+}
+
+var keywordsWithB map[string]token.Type = map[string]token.Type{
+	"BEFORE":  token.KeywordBefore,
+	"BEGIN":   token.KeywordBegin,
+	"BETWEEN": token.KeywordBetween,
+	"BY":      token.KeywordBy,
+}
+
+var keywordsWithC map[string]token.Type = map[string]token.Type{
+	"CASCADE":           token.KeywordCascade,
+	"CASE":              token.KeywordCase,
+	"CAST":              token.KeywordCast,
+	"CHECK":             token.KeywordCheck,
+	"COLLATE":           token.KeywordCollate,
+	"COLUMN":            token.KeywordColumn,
+	"COMMIT":            token.KeywordCommit,
+	"CONFLICT":          token.KeywordConflict,
+	"CONSTRAINT":        token.KeywordConstraint,
+	"CREATE":            token.KeywordCreate,
+	"CROSS":             token.KeywordCross,
+	"CURRENT":           token.KeywordCurrent,
+	"CURRENT_DATE":      token.KeywordCurrentDate,
+	"CURRENT_TIME":      token.KeywordCurrentTime,
+	"CURRENT_TIMESTAMP": token.KeywordCurrentTimestamp,
+}
+
+var keywordsWithD map[string]token.Type = map[string]token.Type{
+	"DATABASE":   token.KeywordDatabase,
+	"DEFAULT":    token.KeywordDefault,
+	"DEFERRABLE": token.KeywordDeferrable,
+	"DEFERRED":   token.KeywordDeferred,
+	"DELETE":     token.KeywordDelete,
+	"DESC":       token.KeywordDesc,
+	"DETACH":     token.KeywordDetach,
+	"DISTINCT":   token.KeywordDistinct,
+	"DO":         token.KeywordDo,
+	"DROP":       token.KeywordDrop,
+}
+
+var keywordsWithE map[string]token.Type = map[string]token.Type{
+	"EACH":      token.KeywordEach,
+	"ELSE":      token.KeywordElse,
+	"END":       token.KeywordEnd,
+	"ESCAPE":    token.KeywordEscape,
+	"EXCEPT":    token.KeywordExcept,
+	"EXCLUDE":   token.KeywordExclude,
+	"EXCLUSIVE": token.KeywordExclusive,
+	"EXISTS":    token.KeywordExists,
+	"EXPLAIN":   token.KeywordExplain,
+}
+
+var keywordsWithF map[string]token.Type = map[string]token.Type{
+	"FAIL":      token.KeywordFail,
+	"FILTER":    token.KeywordFilter,
+	"FIRST":     token.KeywordFirst,
+	"FOLLOWING": token.KeywordFollowing,
+	"FOR":       token.KeywordFor,
+	"FOREIGN":   token.KeywordForeign,
+	"FROM":      token.KeywordFrom,
+	"FULL":      token.KeywordFull,
+}
+
+var keywordsWithG map[string]token.Type = map[string]token.Type{
+	"GLOB":   token.KeywordGlob,
+	"GROUP":  token.KeywordGroup,
+	"GROUPS": token.KeywordGroups,
+}
+
+var keywordsWithH map[string]token.Type = map[string]token.Type{
+	"HAVING": token.KeywordHaving,
+}
+
+var keywordsWithI map[string]token.Type = map[string]token.Type{
+	"IF":        token.KeywordIf,
+	"IGNORE":    token.KeywordIgnore,
+	"IMMEDIATE": token.KeywordImmediate,
+	"IN":        token.KeywordIn,
+	"INDEX":     token.KeywordIndex,
+	"INDEXED":   token.KeywordIndexed,
+	"INITIALLY": token.KeywordInitially,
+	"INNER":     token.KeywordInner,
+	"INSERT":    token.KeywordInsert,
+	"INSTEAD":   token.KeywordInstead,
+	"INTERSECT": token.KeywordIntersect,
+	"INTO":      token.KeywordInto,
+	"IS":        token.KeywordIs,
+	"ISNULL":    token.KeywordIsnull,
+}
+
+var keywordsWithJ map[string]token.Type = map[string]token.Type{
+	"JOIN": token.KeywordJoin,
+}
+
+var keywordsWithK map[string]token.Type = map[string]token.Type{
+	"KEY": token.KeywordKey,
+}
+
+var keywordsWithL map[string]token.Type = map[string]token.Type{
+	"LAST":  token.KeywordLast,
+	"LEFT":  token.KeywordLeft,
+	"LIKE":  token.KeywordLike,
+	"LIMIT": token.KeywordLimit,
+}
+
+var keywordsWithM map[string]token.Type = map[string]token.Type{
+	"MATCH": token.KeywordMatch,
+}
+
+var keywordsWithN map[string]token.Type = map[string]token.Type{
+	"NATURAL": token.KeywordNatural,
+	"NO":      token.KeywordNo,
+	"NOT":     token.KeywordNot,
+	"NOTHING": token.KeywordNothing,
+	"NOTNULL": token.KeywordNotnull,
+	"NULL":    token.KeywordNull,
+}
+
+var keywordsWithO map[string]token.Type = map[string]token.Type{
+	"OF":     token.KeywordOf,
+	"OFFSET": token.KeywordOffset,
+	"ON":     token.KeywordOn,
+	"OR":     token.KeywordOr,
+	"ORDER":  token.KeywordOrder,
+	"OTHERS": token.KeywordOthers,
+	"OUTER":  token.KeywordOuter,
+	"OVER":   token.KeywordOver,
+}
+
+var keywordsWithP map[string]token.Type = map[string]token.Type{
+	"PARTITION": token.KeywordPartition,
+	"PLAN":      token.KeywordPlan,
+	"PRAGMA":    token.KeywordPragma,
+	"PRECEDING": token.KeywordPreceding,
+	"PRIMARY":   token.KeywordPrimary,
+}
+
+var keywordsWithQ map[string]token.Type = map[string]token.Type{
+	"QUERY": token.KeywordQuery,
+}
+
+var keywordsWithR map[string]token.Type = map[string]token.Type{
+	"RAISE":      token.KeywordRaise,
+	"RANGE":      token.KeywordRange,
+	"RECURSIVE":  token.KeywordRecursive,
+	"REFERENCES": token.KeywordReferences,
+	"REGEXP":     token.KeywordRegexp,
+	"REINDEX":    token.KeywordReindex,
+	"RELEASE":    token.KeywordRelease,
+	"RENAME":     token.KeywordRename,
+	"REPLACE":    token.KeywordReplace,
+	"RESTRICT":   token.KeywordRestrict,
+	"RIGHT":      token.KeywordRight,
+	"ROLLBACK":   token.KeywordRollback,
+	"ROW":        token.KeywordRow,
+	"ROWS":       token.KeywordRows,
+}
+
+var keywordsWithS map[string]token.Type = map[string]token.Type{
+	"SAVEPOINT": token.KeywordSavepoint,
+	"SELECT":    token.KeywordSelect,
+	"SET":       token.KeywordSet,
+}
+
+var keywordsWithT map[string]token.Type = map[string]token.Type{
+	"TABLE":       token.KeywordTable,
+	"TEMP":        token.KeywordTemp,
+	"TEMPORARY":   token.KeywordTemporary,
+	"THEN":        token.KeywordThen,
+	"TIES":        token.KeywordTies,
+	"TO":          token.KeywordTo,
+	"TRANSACTION": token.KeywordTransaction,
+	"TRIGGER":     token.KeywordTrigger,
+}
+
+var keywordsWithU map[string]token.Type = map[string]token.Type{
+	"UNBOUNDED": token.KeywordUnbounded,
+	"UNION":     token.KeywordUnion,
+	"UNIQUE":    token.KeywordUnique,
+	"UPDATE":    token.KeywordUpdate,
+	"USING":     token.KeywordUsing,
+}
+
+var keywordsWithV map[string]token.Type = map[string]token.Type{
+	"VACUUM":  token.KeywordVacuum,
+	"VALUES":  token.KeywordValues,
+	"VIEW":    token.KeywordView,
+	"VIRTUAL": token.KeywordVirtual,
+}
+
+var keywordsWithW map[string]token.Type = map[string]token.Type{
+	"WHEN":    token.KeywordWhen,
+	"WHERE":   token.KeywordWhere,
+	"WINDOW":  token.KeywordWindow,
+	"WITH":    token.KeywordWith,
+	"WITHOUT": token.KeywordWithout,
+}
+
+// func scanSpace(s *scanner) token.Token {
+// 	s.accept(matcher.String(" "))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanDoubleQuote(s *scanner) token.Token {
+// 	s.accept(matcher.String("\""))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanPercent(s *scanner) token.Token {
+// 	s.accept(matcher.String("%"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanAmpersand(s *scanner) token.Token {
+// 	s.accept(matcher.String("&"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanQuote(s *scanner) token.Token {
+// 	s.accept(matcher.String("'"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanLeftParanthesis(s *scanner) token.Token {
+// 	s.accept(matcher.String("("))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanRightParanthesis(s *scanner) token.Token {
+// 	s.accept(matcher.String(")"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanAsterisk(s *scanner) token.Token {
+// 	s.accept(matcher.String("*"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanPlusSign(s *scanner) token.Token {
+// 	s.accept(matcher.String("+"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanComma(s *scanner) token.Token {
+// 	s.accept(matcher.String(":true,"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanMinusSign(s *scanner) token.Token {
+// 	s.accept(matcher.String("-"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanPeriod(s *scanner) token.Token {
+// 	s.accept(matcher.String("."))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanSolidus(s *scanner) token.Token {
+// 	s.accept(matcher.String("/"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanReverseSolidus(s *scanner) token.Token {
+// 	s.accept(matcher.String("\\"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanColon(s *scanner) token.Token {
+// 	s.accept(matcher.String(":"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanSemiColon(s *scanner) token.Token {
+// 	s.accept(matcher.String(";"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanLessThanOperator(s *scanner) token.Token {
+// 	s.accept(matcher.String("<"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanEqualsOperator(s *scanner) token.Token {
+// 	s.accept(matcher.String("="))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanGreaterThanOperator(s *scanner) token.Token {
+// 	s.accept(matcher.String("<"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// // needs more work
+// func scanQuestioMarkOrTrigraphs(s *scanner) token.Token {
+// 	s.accept(matcher.String("="))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanLeftBracket(s *scanner) token.Token {
+// 	s.accept(matcher.String("["))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanRightBracket(s *scanner) token.Token {
+// 	s.accept(matcher.String("]"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanCircumflex(s *scanner) token.Token {
+// 	s.accept(matcher.String("^"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanUnderscore(s *scanner) token.Token {
+// 	s.accept(matcher.String("_"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanVerticalBar(s *scanner) token.Token {
+// 	s.accept(matcher.String("|"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanLeftBrace(s *scanner) token.Token {
+// 	s.accept(matcher.String("{"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanRightBrace(s *scanner) token.Token {
+// 	s.accept(matcher.String("}"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+// func scanDollarSign(s *scanner) token.Token {
+// 	s.accept(matcher.String("$"))
+// 	return createToken(s.line, s.col, s.start, s.pos, token.SQLSpecialCharacter, string(s.input[s.start:s.pos]), s)
+// }
+
+func scanAKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithA[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithA[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanBKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithB[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithB[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanCKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithC[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithC[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanDKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithD[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithD[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanEKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithE[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithE[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanFKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithF[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithF[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanGKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithG[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithG[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanHKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithH[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithH[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanIKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithI[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithI[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanJKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithJ[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithJ[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanKKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithK[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithK[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanLKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithL[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithL[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanMKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithM[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithM[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanNKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithN[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithN[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanOKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithO[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithO[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanPKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithP[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithP[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanQKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithQ[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithQ[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanRKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithR[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithR[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanSKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithS[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithS[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanTKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithT[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithT[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanUKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithU[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithU[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+func scanVKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithV[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithV[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
+
+func scanWKeyword(s *scanner) token.Token {
+	input := string(s.input[s.start:s.seekNext(s.start)])
+	if _, ok := keywordsWithW[input]; ok {
+		s.acceptString(input)
+		return createToken(s.line, s.col, s.start, s.pos, keywordsWithW[input], input, s)
+	}
+	// return an error that this doesnt exist
+	return createToken(s.line, s.col, s.start, s.pos, token.KeywordSelect, string(s.input[s.start:s.pos]), s)
+}
diff --git a/internal/parser/scanner/token/doc.go b/internal/parser/scanner/token/doc.go
new file mode 100644
index 0000000..61bd00b
--- /dev/null
+++ b/internal/parser/scanner/token/doc.go
@@ -0,0 +1,2 @@
+// Package token implements tokens that are produced by the scanner.
+package token
diff --git a/internal/parser/scanner/token/token.go b/internal/parser/scanner/token/token.go
new file mode 100644
index 0000000..da10219
--- /dev/null
+++ b/internal/parser/scanner/token/token.go
@@ -0,0 +1,88 @@
+package token
+
+import "fmt"
+
+// Token describes a single token that was produced by a scanner.
+type Token interface {
+	Positioner
+	Lengther
+	Typer
+	Valuer
+}
+
+// Positioner describes something that has a 1-based line and col, and a 0-based
+// offset.
+type Positioner interface {
+	Line() int
+	Col() int
+	Offset() int
+}
+
+// Lengther describes something that has a length.
+type Lengther interface {
+	Length() int
+}
+
+// Typer describes a token that has a token type.
+type Typer interface {
+	Type() Type
+}
+
+// Valuer describes something that has a string value.
+type Valuer interface {
+	Value() string
+}
+
+var _ Token = (*tok)(nil) // ensure that tok implements Token
+
+type tok struct {
+	line, col int
+	offset    int
+	length    int
+	typ       Type
+	value     string
+}
+
+// New creates a new Token implementation, representing the given values.
+func New(line, col, offset, length int, typ Type, value string) Token {
+	return tok{
+		line:   line,
+		col:    col,
+		offset: offset,
+		length: length,
+		typ:    typ,
+		value:  value,
+	}
+}
+
+func (t tok) Line() int {
+	return t.line
+}
+
+func (t tok) Col() int {
+	return t.col
+}
+
+func (t tok) Offset() int {
+	return t.offset
+}
+
+func (t tok) Length() int {
+	return t.length
+}
+
+func (t tok) Type() Type {
+	return t.typ
+}
+
+func (t tok) Value() string {
+	return t.value
+}
+
+func (t tok) IsLiteral() bool {
+	return t.IsLiteral()
+}
+
+func (t tok) String() string {
+	return fmt.Sprintf("%s(%s)", t.typ.String(), t.value)
+}
diff --git a/internal/parser/scanner/token/type.go b/internal/parser/scanner/token/type.go
new file mode 100644
index 0000000..46ae29d
--- /dev/null
+++ b/internal/parser/scanner/token/type.go
@@ -0,0 +1,170 @@
+package token
+
+//go:generate stringer -type=Type
+
+// Type is the type of a token.
+type Type uint16
+
+// All available token types.
+const (
+	Unknown Type = iota
+	// Error indicates that a syntax error has been detected by the lexical
+	// analyzer (scanner) and that the error should be printed. The parser also
+	// should consider resetting its state.
+	Error
+	// EOF indicates that the lexical analyzer (scanner) has reached the end of
+	// its input. After receiving this token, the parser can close the token
+	// stream, as there will not be any more tokens. He also can start building
+	// (if not already done) the AST, as he know knows of all tokens.
+	EOF
+
+	// StatementSeparator is the type of tokens that represent a single
+	// semicolon, as a single semicolon separates multiple sql statements.
+	StatementSeparator
+
+	KeywordAbort
+	KeywordAction
+	KeywordAdd
+	KeywordAfter
+	KeywordAll
+	KeywordAlter
+	KeywordAnalyze
+	KeywordAnd
+	KeywordAs
+	KeywordAsc
+	KeywordAttach
+	KeywordAutoincrement
+	KeywordBefore
+	KeywordBegin
+	KeywordBetween
+	KeywordBy
+	KeywordCascade
+	KeywordCase
+	KeywordCast
+	KeywordCheck
+	KeywordCollate
+	KeywordColumn
+	KeywordCommit
+	KeywordConflict
+	KeywordConstraint
+	KeywordCreate
+	KeywordCross
+	KeywordCurrent
+	KeywordCurrentDate
+	KeywordCurrentTime
+	KeywordCurrentTimestamp
+	KeywordDatabase
+	KeywordDefault
+	KeywordDeferrable
+	KeywordDeferred
+	KeywordDelete
+	KeywordDesc
+	KeywordDetach
+	KeywordDistinct
+	KeywordDo
+	KeywordDrop
+	KeywordEach
+	KeywordElse
+	KeywordEnd
+	KeywordEscape
+	KeywordExcept
+	KeywordExclude
+	KeywordExclusive
+	KeywordExists
+	KeywordExplain
+	KeywordFail
+	KeywordFilter
+	KeywordFirst
+	KeywordFollowing
+	KeywordFor
+	KeywordForeign
+	KeywordFrom
+	KeywordFull
+	KeywordGlob
+	KeywordGroup
+	KeywordGroups
+	KeywordHaving
+	KeywordIf
+	KeywordIgnore
+	KeywordImmediate
+	KeywordIn
+	KeywordIndex
+	KeywordIndexed
+	KeywordInitially
+	KeywordInner
+	KeywordInsert
+	KeywordInstead
+	KeywordIntersect
+	KeywordInto
+	KeywordIs
+	KeywordIsnull
+	KeywordJoin
+	KeywordKey
+	KeywordLast
+	KeywordLeft
+	KeywordLike
+	KeywordLimit
+	KeywordMatch
+	KeywordNatural
+	KeywordNo
+	KeywordNot
+	KeywordNothing
+	KeywordNotnull
+	KeywordNull
+	KeywordNulls
+	KeywordOf
+	KeywordOffset
+	KeywordOn
+	KeywordOr
+	KeywordOrder
+	KeywordOthers
+	KeywordOuter
+	KeywordOver
+	KeywordPartition
+	KeywordPlan
+	KeywordPragma
+	KeywordPreceding
+	KeywordPrimary
+	KeywordQuery
+	KeywordRaise
+	KeywordRange
+	KeywordRecursive
+	KeywordReferences
+	KeywordRegexp
+	KeywordReindex
+	KeywordRelease
+	KeywordRename
+	KeywordReplace
+	KeywordRestrict
+	KeywordRight
+	KeywordRollback
+	KeywordRow
+	KeywordRows
+	KeywordSavepoint
+	KeywordSelect
+	KeywordSet
+	KeywordTable
+	KeywordTemp
+	KeywordTemporary
+	KeywordThen
+	KeywordTies
+	KeywordTo
+	KeywordTransaction
+	KeywordTrigger
+	KeywordUnbounded
+	KeywordUnion
+	KeywordUnique
+	KeywordUpdate
+	KeywordUsing
+	KeywordVacuum
+	KeywordValues
+	KeywordView
+	KeywordVirtual
+	KeywordWhen
+	KeywordWhere
+	KeywordWindow
+	KeywordWith
+	KeywordWithout
+
+	Literal
+)
diff --git a/internal/parser/scanner/token/type_string.go b/internal/parser/scanner/token/type_string.go
new file mode 100644
index 0000000..44f98bb
--- /dev/null
+++ b/internal/parser/scanner/token/type_string.go
@@ -0,0 +1,170 @@
+// Code generated by "stringer -type=Type"; DO NOT EDIT.
+
+package token
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Unknown-0]
+	_ = x[Error-1]
+	_ = x[EOF-2]
+	_ = x[StatementSeparator-3]
+	_ = x[KeywordAbort-4]
+	_ = x[KeywordAction-5]
+	_ = x[KeywordAdd-6]
+	_ = x[KeywordAfter-7]
+	_ = x[KeywordAll-8]
+	_ = x[KeywordAlter-9]
+	_ = x[KeywordAnalyze-10]
+	_ = x[KeywordAnd-11]
+	_ = x[KeywordAs-12]
+	_ = x[KeywordAsc-13]
+	_ = x[KeywordAttach-14]
+	_ = x[KeywordAutoincrement-15]
+	_ = x[KeywordBefore-16]
+	_ = x[KeywordBegin-17]
+	_ = x[KeywordBetween-18]
+	_ = x[KeywordBy-19]
+	_ = x[KeywordCascade-20]
+	_ = x[KeywordCase-21]
+	_ = x[KeywordCast-22]
+	_ = x[KeywordCheck-23]
+	_ = x[KeywordCollate-24]
+	_ = x[KeywordColumn-25]
+	_ = x[KeywordCommit-26]
+	_ = x[KeywordConflict-27]
+	_ = x[KeywordConstraint-28]
+	_ = x[KeywordCreate-29]
+	_ = x[KeywordCross-30]
+	_ = x[KeywordCurrent-31]
+	_ = x[KeywordCurrentDate-32]
+	_ = x[KeywordCurrentTime-33]
+	_ = x[KeywordCurrentTimestamp-34]
+	_ = x[KeywordDatabase-35]
+	_ = x[KeywordDefault-36]
+	_ = x[KeywordDeferrable-37]
+	_ = x[KeywordDeferred-38]
+	_ = x[KeywordDelete-39]
+	_ = x[KeywordDesc-40]
+	_ = x[KeywordDetach-41]
+	_ = x[KeywordDistinct-42]
+	_ = x[KeywordDo-43]
+	_ = x[KeywordDrop-44]
+	_ = x[KeywordEach-45]
+	_ = x[KeywordElse-46]
+	_ = x[KeywordEnd-47]
+	_ = x[KeywordEscape-48]
+	_ = x[KeywordExcept-49]
+	_ = x[KeywordExclude-50]
+	_ = x[KeywordExclusive-51]
+	_ = x[KeywordExists-52]
+	_ = x[KeywordExplain-53]
+	_ = x[KeywordFail-54]
+	_ = x[KeywordFilter-55]
+	_ = x[KeywordFirst-56]
+	_ = x[KeywordFollowing-57]
+	_ = x[KeywordFor-58]
+	_ = x[KeywordForeign-59]
+	_ = x[KeywordFrom-60]
+	_ = x[KeywordFull-61]
+	_ = x[KeywordGlob-62]
+	_ = x[KeywordGroup-63]
+	_ = x[KeywordGroups-64]
+	_ = x[KeywordHaving-65]
+	_ = x[KeywordIf-66]
+	_ = x[KeywordIgnore-67]
+	_ = x[KeywordImmediate-68]
+	_ = x[KeywordIn-69]
+	_ = x[KeywordIndex-70]
+	_ = x[KeywordIndexed-71]
+	_ = x[KeywordInitially-72]
+	_ = x[KeywordInner-73]
+	_ = x[KeywordInsert-74]
+	_ = x[KeywordInstead-75]
+	_ = x[KeywordIntersect-76]
+	_ = x[KeywordInto-77]
+	_ = x[KeywordIs-78]
+	_ = x[KeywordIsnull-79]
+	_ = x[KeywordJoin-80]
+	_ = x[KeywordKey-81]
+	_ = x[KeywordLast-82]
+	_ = x[KeywordLeft-83]
+	_ = x[KeywordLike-84]
+	_ = x[KeywordLimit-85]
+	_ = x[KeywordMatch-86]
+	_ = x[KeywordNatural-87]
+	_ = x[KeywordNo-88]
+	_ = x[KeywordNot-89]
+	_ = x[KeywordNothing-90]
+	_ = x[KeywordNotnull-91]
+	_ = x[KeywordNull-92]
+	_ = x[KeywordNulls-93]
+	_ = x[KeywordOf-94]
+	_ = x[KeywordOffset-95]
+	_ = x[KeywordOn-96]
+	_ = x[KeywordOr-97]
+	_ = x[KeywordOrder-98]
+	_ = x[KeywordOthers-99]
+	_ = x[KeywordOuter-100]
+	_ = x[KeywordOver-101]
+	_ = x[KeywordPartition-102]
+	_ = x[KeywordPlan-103]
+	_ = x[KeywordPragma-104]
+	_ = x[KeywordPreceding-105]
+	_ = x[KeywordPrimary-106]
+	_ = x[KeywordQuery-107]
+	_ = x[KeywordRaise-108]
+	_ = x[KeywordRange-109]
+	_ = x[KeywordRecursive-110]
+	_ = x[KeywordReferences-111]
+	_ = x[KeywordRegexp-112]
+	_ = x[KeywordReindex-113]
+	_ = x[KeywordRelease-114]
+	_ = x[KeywordRename-115]
+	_ = x[KeywordReplace-116]
+	_ = x[KeywordRestrict-117]
+	_ = x[KeywordRight-118]
+	_ = x[KeywordRollback-119]
+	_ = x[KeywordRow-120]
+	_ = x[KeywordRows-121]
+	_ = x[KeywordSavepoint-122]
+	_ = x[KeywordSelect-123]
+	_ = x[KeywordSet-124]
+	_ = x[KeywordTable-125]
+	_ = x[KeywordTemp-126]
+	_ = x[KeywordTemporary-127]
+	_ = x[KeywordThen-128]
+	_ = x[KeywordTies-129]
+	_ = x[KeywordTo-130]
+	_ = x[KeywordTransaction-131]
+	_ = x[KeywordTrigger-132]
+	_ = x[KeywordUnbounded-133]
+	_ = x[KeywordUnion-134]
+	_ = x[KeywordUnique-135]
+	_ = x[KeywordUpdate-136]
+	_ = x[KeywordUsing-137]
+	_ = x[KeywordVacuum-138]
+	_ = x[KeywordValues-139]
+	_ = x[KeywordView-140]
+	_ = x[KeywordVirtual-141]
+	_ = x[KeywordWhen-142]
+	_ = x[KeywordWhere-143]
+	_ = x[KeywordWindow-144]
+	_ = x[KeywordWith-145]
+	_ = x[KeywordWithout-146]
+	_ = x[Literal-147]
+}
+
+const _Type_name = "UnknownErrorEOFStatementSeparatorKeywordAbortKeywordActionKeywordAddKeywordAfterKeywordAllKeywordAlterKeywordAnalyzeKeywordAndKeywordAsKeywordAscKeywordAttachKeywordAutoincrementKeywordBeforeKeywordBeginKeywordBetweenKeywordByKeywordCascadeKeywordCaseKeywordCastKeywordCheckKeywordCollateKeywordColumnKeywordCommitKeywordConflictKeywordConstraintKeywordCreateKeywordCrossKeywordCurrentKeywordCurrentDateKeywordCurrentTimeKeywordCurrentTimestampKeywordDatabaseKeywordDefaultKeywordDeferrableKeywordDeferredKeywordDeleteKeywordDescKeywordDetachKeywordDistinctKeywordDoKeywordDropKeywordEachKeywordElseKeywordEndKeywordEscapeKeywordExceptKeywordExcludeKeywordExclusiveKeywordExistsKeywordExplainKeywordFailKeywordFilterKeywordFirstKeywordFollowingKeywordForKeywordForeignKeywordFromKeywordFullKeywordGlobKeywordGroupKeywordGroupsKeywordHavingKeywordIfKeywordIgnoreKeywordImmediateKeywordInKeywordIndexKeywordIndexedKeywordInitiallyKeywordInnerKeywordInsertKeywordInsteadKeywordIntersectKeywordIntoKeywordIsKeywordIsnullKeywordJoinKeywordKeyKeywordLastKeywordLeftKeywordLikeKeywordLimitKeywordMatchKeywordNaturalKeywordNoKeywordNotKeywordNothingKeywordNotnullKeywordNullKeywordNullsKeywordOfKeywordOffsetKeywordOnKeywordOrKeywordOrderKeywordOthersKeywordOuterKeywordOverKeywordPartitionKeywordPlanKeywordPragmaKeywordPrecedingKeywordPrimaryKeywordQueryKeywordRaiseKeywordRangeKeywordRecursiveKeywordReferencesKeywordRegexpKeywordReindexKeywordReleaseKeywordRenameKeywordReplaceKeywordRestrictKeywordRightKeywordRollbackKeywordRowKeywordRowsKeywordSavepointKeywordSelectKeywordSetKeywordTableKeywordTempKeywordTemporaryKeywordThenKeywordTiesKeywordToKeywordTransactionKeywordTriggerKeywordUnboundedKeywordUnionKeywordUniqueKeywordUpdateKeywordUsingKeywordVacuumKeywordValuesKeywordViewKeywordVirtualKeywordWhenKeywordWhereKeywordWindowKeywordWithKeywordWithoutLiteral"
+
+var _Type_index = [...]uint16{0, 7, 12, 15, 33, 45, 58, 68, 80, 90, 102, 116, 126, 135, 145, 158, 178, 191, 203, 217, 226, 240, 251, 262, 274, 288, 301, 314, 329, 346, 359, 371, 385, 403, 421, 444, 459, 473, 490, 505, 518, 529, 542, 557, 566, 577, 588, 599, 609, 622, 635, 649, 665, 678, 692, 703, 716, 728, 744, 754, 768, 779, 790, 801, 813, 826, 839, 848, 861, 877, 886, 898, 912, 928, 940, 953, 967, 983, 994, 1003, 1016, 1027, 1037, 1048, 1059, 1070, 1082, 1094, 1108, 1117, 1127, 1141, 1155, 1166, 1178, 1187, 1200, 1209, 1218, 1230, 1243, 1255, 1266, 1282, 1293, 1306, 1322, 1336, 1348, 1360, 1372, 1388, 1405, 1418, 1432, 1446, 1459, 1473, 1488, 1500, 1515, 1525, 1536, 1552, 1565, 1575, 1587, 1598, 1614, 1625, 1636, 1645, 1663, 1677, 1693, 1705, 1718, 1731, 1743, 1756, 1769, 1780, 1794, 1805, 1817, 1830, 1841, 1855, 1862}
+
+func (i Type) String() string {
+	if i >= Type(len(_Type_index)-1) {
+		return "Type(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _Type_name[_Type_index[i]:_Type_index[i+1]]
+}
diff --git a/internal/parser/simple_parser.go b/internal/parser/simple_parser.go
new file mode 100644
index 0000000..680afc2
--- /dev/null
+++ b/internal/parser/simple_parser.go
@@ -0,0 +1,200 @@
+package parser
+
+import (
+	"fmt"
+	"github.com/tomarrell/lbadd/internal/parser/ast"
+	"github.com/tomarrell/lbadd/internal/parser/scanner"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+type errorReporter struct {
+	p      *simpleParser
+	errs   []error
+	sealed bool
+}
+
+func (r *errorReporter) incompleteStatement() {
+	next, ok := r.p.lookahead()
+	if !ok {
+		r.errorf("%w: EOF", ErrIncompleteStatement)
+	} else {
+		r.errorf("%w: %s at (%d:%d) offset %d length %d", ErrIncompleteStatement, next.Type().String(), next.Line(), next.Col(), next.Offset(), next.Length())
+	}
+}
+
+func (r *errorReporter) prematureEOF() {
+	r.errorf("%w", ErrPrematureEOF)
+	r.sealed = true
+}
+
+func (r *errorReporter) unexpectedToken(expected ...token.Type) {
+	if r.sealed {
+		return
+	}
+	next, ok := r.p.lookahead()
+	if !ok || next.Type() == token.EOF {
+		// use this instead of r.prematureEOF() because we can add the
+		// information about what tokens were expected
+		r.errorf("%w: expected %s", ErrPrematureEOF, expected)
+		r.sealed = true
+		return
+	}
+
+	r.errorf("%w: got %s but expected one of %s at (%d:%d) offset %d length %d", ErrUnexpectedToken, next, expected, next.Line(), next.Col(), next.Offset(), next.Length())
+}
+
+func (r *errorReporter) unhandledToken(t token.Token) {
+	r.errorf("%w: %s(%s) at (%d:%d) offset %d lenght %d", ErrUnknownToken, t.Type().String(), t.Value(), t.Line(), t.Col(), t.Offset(), t.Length())
+}
+
+func (r *errorReporter) unsupportedConstruct(t token.Token) {
+	r.errorf("%w: %s(%s) at (%d:%d) offset %d lenght %d", ErrUnsupportedConstruct, t.Type().String(), t.Value(), t.Line(), t.Col(), t.Offset(), t.Length())
+}
+
+func (r *errorReporter) errorf(format string, args ...interface{}) {
+	r.errs = append(r.errs, fmt.Errorf(format, args...))
+}
+
+type reporter interface {
+	incompleteStatement()
+	prematureEOF()
+	unexpectedToken(expected ...token.Type)
+	unhandledToken(t token.Token)
+	unsupportedConstruct(t token.Token)
+}
+
+var _ Parser = (*simpleParser)(nil) // ensure that simpleParser implements Parser
+
+type simpleParser struct {
+	scanner scanner.Scanner
+}
+
+// New creates new ready to use parser.
+func New(input string) Parser {
+	return &simpleParser{
+		scanner: scanner.New([]rune(input)),
+	}
+}
+
+func (p *simpleParser) Next() (*ast.SQLStmt, []error, bool) {
+	if !p.scanner.HasNext() {
+		return nil, []error{}, false
+	}
+	errs := &errorReporter{
+		p:    p,
+		errs: []error{},
+	}
+	stmt := p.parseSQLStatement(errs)
+	return stmt, errs.errs, true
+}
+
+// skipUntil skips tokens until a token is of one of the given types. That token
+// will not be consumed, every other token will be consumed and an unexpected
+// token error will be reported.
+func (p *simpleParser) skipUntil(r reporter, types ...token.Type) {
+	for {
+		next, ok := p.lookahead()
+		if !ok || next.Type() == token.EOF || next.Type() == token.StatementSeparator {
+			return
+		}
+		for _, typ := range types {
+			if next.Type() == typ {
+				return
+			}
+		}
+		r.unexpectedToken(types...)
+		p.consumeToken()
+	}
+}
+
+func (p *simpleParser) skipUntilNoError(types ...token.Type) {
+	for {
+		next, ok := p.lookahead()
+		if !ok || next.Type() == token.EOF || next.Type() == token.StatementSeparator {
+			return
+		}
+		for _, typ := range types {
+			if next.Type() == typ {
+				return
+			}
+		}
+		p.consumeToken()
+	}
+}
+
+func (p *simpleParser) lookahead() (next token.Token, hasNext bool) {
+	if !p.scanner.HasNext() {
+		return nil, false
+	}
+
+	return p.scanner.Peek(), true
+}
+
+func (p *simpleParser) lookaheadWithType(typ token.Type) (token.Token, bool) {
+	next, hasNext := p.lookahead()
+	return next, hasNext && next.Type() == typ
+}
+
+func (p *simpleParser) consumeToken() {
+	_ = p.scanner.Next()
+}
+
+func (p *simpleParser) parseSQLStatement(r reporter) (stmt *ast.SQLStmt) {
+	stmt = &ast.SQLStmt{}
+
+	if next, ok := p.lookaheadWithType(token.KeywordExplain); ok {
+		stmt.Explain = next
+		p.consumeToken()
+
+		if next, ok := p.lookaheadWithType(token.KeywordQuery); ok {
+			stmt.Query = next
+			p.consumeToken()
+
+			if next, ok := p.lookaheadWithType(token.KeywordPlan); ok {
+				stmt.Plan = next
+				p.consumeToken()
+			} else {
+				r.unexpectedToken(token.KeywordPlan)
+				// At this point, just assume that 'QUERY' was a mistake. Don't
+				// abort. It's very unlikely that 'PLAN' occurs somewhere, so
+				// assume that the user meant to input 'EXPLAIN <statement>'
+				// instead of 'EXPLAIN QUERY PLAN <statement>'.
+			}
+		}
+	}
+
+	// according to the grammar, these are the tokens that initiate a statement
+	p.skipUntil(r, token.KeywordAlter, token.KeywordAnalyze, token.KeywordAttach, token.KeywordBegin, token.KeywordCommit, token.KeywordCreate, token.KeywordDelete, token.KeywordDetach, token.KeywordDrop, token.KeywordInsert, token.KeywordPragma, token.KeywordReindex, token.KeywordRelease, token.KeywordRollback, token.KeywordSavepoint, token.KeywordSelect, token.KeywordUpdate, token.KeywordVacuum)
+
+	next, ok := p.lookahead()
+	if !ok || next.Type() == token.EOF {
+		r.prematureEOF()
+		return stmt
+	}
+
+	switch next.Type() {
+	case token.KeywordAlter:
+		stmt.AlterTableStmt = p.parseAlterTableStmt(r)
+	case token.StatementSeparator:
+		r.incompleteStatement()
+		p.consumeToken()
+	case token.KeywordPragma:
+		// we don't support pragmas, as we don't need them yet
+		r.unsupportedConstruct(next)
+		p.skipUntilNoError(token.StatementSeparator, token.EOF)
+	default:
+		r.unsupportedConstruct(next)
+		p.skipUntilNoError(token.StatementSeparator, token.EOF)
+	}
+
+	next, ok = p.lookahead()
+	if ok && (next.Type() != token.StatementSeparator || next.Type() != token.EOF) {
+		r.unexpectedToken(token.StatementSeparator, token.EOF)
+	}
+
+	return stmt
+}
+
+func (p *simpleParser) parseAlterTableStmt(r reporter) (stmt *ast.AlterTableStmt) {
+	panic("implement me")
+}


Test patch:
diff --git a/internal/parser/scanner/scanner_test.go b/internal/parser/scanner/scanner_test.go
new file mode 100644
index 0000000..d92e5dc
--- /dev/null
+++ b/internal/parser/scanner/scanner_test.go
@@ -0,0 +1,15 @@
+package scanner
+
+import (
+	"testing"
+)
+
+func Test_hasNext(t *testing.T) {
+	t.SkipNow()
+
+	input := "SELECT "
+	scanner := New([]rune(input))
+	for scanner.HasNext() {
+		scanner.Next()
+	}
+}
diff --git a/internal/parser/simple_parser_test.go b/internal/parser/simple_parser_test.go
new file mode 100644
index 0000000..0c93d76
--- /dev/null
+++ b/internal/parser/simple_parser_test.go
@@ -0,0 +1,70 @@
+package parser
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/tomarrell/lbadd/internal/parser/ast"
+	"github.com/tomarrell/lbadd/internal/parser/scanner"
+	"github.com/tomarrell/lbadd/internal/parser/scanner/token"
+)
+
+var _ scanner.Scanner = (*_testScanner)(nil) // ensure that testScanner implements scanner.Scanner
+
+type _testScanner struct {
+	pos    int
+	tokens []token.Token
+}
+
+func scannerOf(tokens ...token.Token) *_testScanner {
+	return &_testScanner{tokens: tokens}
+}
+
+func (s *_testScanner) HasNext() bool {
+	return s.pos < len(s.tokens)
+}
+
+func (s *_testScanner) Next() token.Token {
+	tk := s.Peek()
+	s.pos++
+	return tk
+}
+
+func (s *_testScanner) Peek() token.Token {
+	return s.tokens[s.pos]
+}
+
+func Test_simpleParser_Next(t *testing.T) {
+	tests := []struct {
+		name   string
+		tokens []token.Token
+		stmt   *ast.SQLStmt
+		errs   []error
+		ok     bool
+	}{
+		{
+			"no tokens",
+			[]token.Token{},
+			nil,
+			[]error{},
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &simpleParser{
+				scanner: scannerOf(tt.tokens...),
+			}
+			stmt, errs, ok := p.Next()
+			if !reflect.DeepEqual(stmt, tt.stmt) {
+				t.Errorf("simpleParser.Next() stmt = %v, want %v", stmt, tt.stmt)
+			}
+			if !reflect.DeepEqual(errs, tt.errs) {
+				t.Errorf("simpleParser.Next() errs = %v, want %v", errs, tt.errs)
+			}
+			if ok != tt.ok {
+				t.Errorf("simpleParser.Next() ok = %v, want %v", ok, tt.ok)
+			}
+		})
+	}
+}