func (s *Scanner) Lex(v *yySymType) int {
	tok, pos, lit := s.scan()
	s.lastScanOffset = pos.Offset
	v.offset = pos.Offset
	v.ident = lit
	if tok == identifier {
		tok = handleIdent(v)
	}
	if tok == identifier {
		if tok1 := s.isTokenIdentifier(lit, pos.Offset); tok1 != 0 {
			tok = tok1
		}
	}
	if s.sqlMode.HasANSIQuotesMode() &&
		tok == stringLit &&
		s.r.s[v.offset] == '"' {
		tok = identifier
	}
	if tok == pipes && !(s.sqlMode.HasPipesAsConcatMode()) {
		return pipesAsOr
	}
	if tok == not && s.sqlMode.HasHighNotPrecedenceMode() {
		return not2
	}
	switch tok {
	case intLit:
		return toInt(s, v, lit)
	case floatLit:
		return toFloat(s, v, lit)
	case decLit:
		return toDecimal(s, v, lit)
	case hexLit:
		return toHex(s, v, lit)
	case bitLit:
		return toBit(s, v, lit)
	case singleAtIdentifier, doubleAtIdentifier, cast, extract:
		v.item = lit
		return tok
	case null:
		v.item = nil
	case quotedIdentifier:
		tok = identifier
	}
	if tok == unicode.ReplacementChar && s.r.eof() {
		return 0
	}
	return tok
}
Lex returns a token and store the token value in v.
Scanner satisfies yyLexer interface.
0 and invalid are special token id this function would return:
return 0 tells parser that scanner meets EOF,
return invalid tells parser that scanner meets illegal character.
package parser
import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
	"github.com/nooncall/shazam/mysql"
)
var _ = yyLexer(&Scanner{})
type Pos struct {
	Line   int
	Col    int
	Offset int
}
type Scanner struct {
	r   reader
	buf bytes.Buffer
	errs         []error
	warns        []error
	stmtStartPos int
	specialComment specialCommentScanner
	sqlMode mysql.SQLMode
	supportWindowFunc bool
	lastScanOffset int
}
type specialCommentScanner interface {
	scan() (tok int, pos Pos, lit string)
}
type mysqlSpecificCodeScanner struct {
	*Scanner
	Pos
}
func (s *mysqlSpecificCodeScanner) scan() (tok int, pos Pos, lit string) {
	tok, pos, lit = s.Scanner.scan()
	pos.Line += s.Pos.Line
	pos.Col += s.Pos.Col
	pos.Offset += s.Pos.Offset
	return
}
type optimizerHintScanner struct {
	*Scanner
	Pos
	end bool
}
func (s *optimizerHintScanner) scan() (tok int, pos Pos, lit string) {
	tok, pos, lit = s.Scanner.scan()
	pos.Line += s.Pos.Line
	pos.Col += s.Pos.Col
	pos.Offset += s.Pos.Offset
	if tok == 0 {
		if !s.end {
			tok = hintEnd
			s.end = true
		}
	}
	return
}
func (s *Scanner) Errors() (warns []error, errs []error) {
	return s.warns, s.errs
}
func (s *Scanner) reset(sql string) {
	s.r = reader{s: sql, p: Pos{Line: 1}}
	s.buf.Reset()
	s.errs = s.errs[:0]
	s.warns = s.warns[:0]
	s.stmtStartPos = 0
	s.specialComment = nil
}
func (s *Scanner) stmtText() string {
	endPos := s.r.pos().Offset
	if s.r.s[endPos-1] == '\n' {
		endPos = endPos - 1  
	}
	if s.r.s[s.stmtStartPos] == '\n' {
		s.stmtStartPos++
	}
	text := s.r.s[s.stmtStartPos:endPos]
	s.stmtStartPos = endPos
	return text
}
func (s *Scanner) Errorf(format string, a ...interface{}) {
	str := fmt.Sprintf(format, a...)
	val := s.r.s[s.lastScanOffset:]
	var lenStr = ""
	if len(val) > 2048 {
		lenStr = "(total length " + strconv.Itoa(len(val)) + ")"
		val = val[:2048]
	}
	err := fmt.Errorf("line %d column %d near \"%s\"%s %s",
		s.r.p.Line, s.r.p.Col, val, str, lenStr)
	s.errs = append(s.errs, err)
}