Repository: tetratelabs/wazero
Pull request #78 (2021-12-21 01:51:29)

Simplifies token parsing API with byte slices
This removes byte position from the token parsing api as passing byte
slices is equally performant. This also renames some variables to be
consistent.


Patch:
diff --git a/wasm/wat/lexer.go b/wasm/wat/lexer.go
index aaeb8c43..bf94f862 100644
--- a/wasm/wat/lexer.go
+++ b/wasm/wat/lexer.go
@@ -6,25 +6,32 @@ import (
 )
 
 // parseToken allows a parser to inspect a token without necessarily allocating strings
-// * source is the underlying byte stream: do not modify this
 // * tokenType is the token type
-// * beginPos is the byte position in the source where the token begins, inclusive
-// * endPos is the byte position in the source where the token ends, exclusive
+// * tokenBytes are the UTF-8 bytes representing the token. Do not modify this.
+// * line is the source line number determined by unescaped '\n' characters.
+// * col is the UTF-8 column number.
 //
 // Returning an error will short-circuit any future invocations.
-type parseToken func(source []byte, tok tokenType, beginLine, beginCol, beginPos, endPos int) error
+type parseToken func(tok tokenType, tokenBytes []byte, line, col int) error
 
-// lex invokes the parser function for each token, the source is exhausted. The fileName is used in error messages.
+var (
+	constantLParen = []byte{'('}
+	constantRParen = []byte{')'}
+)
+
+// lex invokes the parser function for the given source.
 //
-// Errors from the parser or during tokenization exit early, such as dangling block comments or unexpected characters.
-func lex(fileName string, source []byte, parser parseToken) error {
-	length := len(source)
-	// p is the position in the source, and a parameter to the parser.
-	p := 0
-	// line is the line number in the source as determined by unescaped '\n' characters, and a parameter to the parser.
+// This function returns when the source is exhausted or an error occurs. Example errors include an error invoking the
+// parser, dangling block comments or unexpected characters.
+func lex(parser parseToken, fileName string, source []byte) error {
+	// i is the source index to begin reading, inclusive.
+	i := 0
+	// end is the source index to stop reading, exclusive.
+	end := len(source)
+	// line is the source line number determined by unescaped '\n' characters.
 	line := 1
-	// col is the UTF-8 aware column number, and a parameter to the parser
-	col := 0
+	// col is the UTF-8 column number.
+	col := 1
 
 	// Web assembly expressions are grouped by parenthesis, even the minimal example "(module)". We track nesting level
 	// to help report problems instead of bubbling to the parser layer.
@@ -35,8 +42,8 @@ func lex(fileName string, source []byte, parser parseToken) error {
 	// inline processing.
 	blockCommentDepth := 0
 
-	for ; p < length; p = p + 1 {
-		b1 := source[p]
+	for ; i < end; i, col = i+1, col+1 {
+		b1 := source[i]
 
 		// The spec does not consider newlines apart from '\n'. Notably, a bare '\r' is not a newline here.
 		// See https://www.w3.org/TR/wasm-core-1/#text-comment
@@ -46,8 +53,6 @@ func lex(fileName string, source []byte, parser parseToken) error {
 			continue // next line
 		}
 
-		// Otherwise, advance the column regardless of whether we are whitespace or not
-		col = col + 1                              // the current character is at least one byte long
 		if b1 == ' ' || b1 == '\t' || b1 == '\r' { // fast path ASCII whitespace
 			continue // next whitespace
 		}
@@ -55,39 +60,39 @@ func lex(fileName string, source []byte, parser parseToken) error {
 		// Handle parens and comments, noting block comments, ex. "(; look! ;)", can be nested.
 		switch b1 {
 		case '(':
-			peekPos := p + 1
-			if peekPos == length { // invalid regardless of block comment or not. nothing opens at EOF!
+			peek := i + 1
+			if peek == end { // invalid regardless of block comment or not. nothing opens at EOF!
 				return fmt.Errorf("%s:%d:%d: found '(' at end of input", fileName, line, col)
 			}
-			if source[peekPos] == ';' { // next block comment
-				p = peekPos // continue after "(;"
+			if source[peek] == ';' { // next block comment
+				i = peek // continue after "(;"
 				col = col + 1
 				blockCommentDepth = blockCommentDepth + 1
 				continue
-			} else if blockCommentDepth == 0 {
-				if e := parser(source, tokenLParen, line, col, p, p+1); e != nil {
+			} else if blockCommentDepth == 0 { // Fast path left paren token at the expense of code duplication.
+				if e := parser(tokenLParen, constantLParen, line, col); e != nil {
 					return e
 				}
 				parenDepth = parenDepth + 1
 				continue
 			}
 		case ')':
-			if blockCommentDepth == 0 {
+			if blockCommentDepth == 0 { // Fast path right paren token at the expense of code duplication.
 				if parenDepth == 0 {
 					return fmt.Errorf("%s:%d:%d: found ')' before '('", fileName, line, col)
 				}
-				if e := parser(source, tokenRParen, line, col, p, p+1); e != nil {
+				if e := parser(tokenRParen, constantRParen, line, col); e != nil {
 					return e
 				}
 				parenDepth = parenDepth - 1
 				continue
 			}
 		case ';': // possible line comment or block comment end
-			peekPos := p + 1
-			if peekPos < length {
-				b2 := source[peekPos]
+			peek := i + 1
+			if peek < end {
+				b2 := source[peek]
 				if blockCommentDepth > 0 && b2 == ')' {
-					p = peekPos // continue after ";)"
+					i = peek // continue after ";)"
 					col = col + 1
 					blockCommentDepth = blockCommentDepth - 1
 					continue
@@ -95,12 +100,12 @@ func lex(fileName string, source []byte, parser parseToken) error {
 
 				if b2 == ';' { // line comment
 					// Start after ";;" and run until the end. Note UTF-8 (multi-byte) characters are allowed.
-					peekPos = peekPos + 1
+					peek = peek + 1
 					col = col + 1
 
 				LineComment:
-					for peekPos < length {
-						peeked := source[peekPos]
+					for peek < end {
+						peeked := source[peek]
 						if peeked == '\n' {
 							break LineComment // EOL bookkeeping will proceed on the next iteration
 						}
@@ -110,12 +115,12 @@ func lex(fileName string, source []byte, parser parseToken) error {
 						if s == 0 {
 							return fmt.Errorf("%s:%d:%d: found an invalid byte in line comment: 0x%x", fileName, line, col, peeked)
 						}
-						peekPos = peekPos + s
+						peek = peek + s
 					}
 
 					// -1 because for loop will + 1: This optimizes speed of tokenization over line comments.
-					p = peekPos - 1 // at the '\n'
-					continue        // end of line comment
+					i = peek - 1 // at the '\n'
+					continue     // end of line comment
 				}
 			}
 		}
@@ -126,7 +131,7 @@ func lex(fileName string, source []byte, parser parseToken) error {
 			if s == 0 {
 				return fmt.Errorf("%s:%d:%d: found an invalid byte in block comment: 0x%x", fileName, line, col, b1)
 			}
-			p = p + s - 1 // -1 because for loop will + 1: This optimizes speed of tokenization over block comments.
+			i = i + s - 1 // -1 because for loop will + 1: This optimizes speed of tokenization over block comments.
 			continue
 		}
 
@@ -137,28 +142,28 @@ func lex(fileName string, source []byte, parser parseToken) error {
 		//
 		// See https://www.w3.org/TR/wasm-core-1/#characters%E2%91%A0
 
+		tok := firstTokenByte[b1]
 		// Track positions passed to the parser
-		// beginLine == line because no token is allowed to include an unescaped '\n'
-		beginCol := col  // the start column of the token (fixed)
-		beginPos := p    // the start position of the token (fixed)
-		peekPos := p + 1 // when finished scanning, this becomes endPos (the position after the token).
+		b := i        // the start position of the token (fixed)
+		peek := i + 1 // when finished scanning, this becomes end (the position after the token).
+		// line == line because no token is allowed to include an unescaped '\n'
+		c := col // the start column of the token (fixed)
 
-		tok := firstTokenByte[b1]
 		switch tok {
 		case tokenSN: // min 2 bytes for sign and number; ambiguous: could be tokenFN
 			return fmt.Errorf("%s:%d:%d: TODO: signed", fileName, line, col)
 		case tokenUN: // min 1 byte; ambiguous when >=3 bytes as could be tokenFN
-			if peekPos < length {
-				peeked := source[peekPos]
+			if peek < end {
+				peeked := source[peek]
 				if peeked == 'x' {
 					return fmt.Errorf("%s:%d:%d: TODO: hex", fileName, line, col)
 				}
 			Number:
 				// Start after the number and run until the end. Note all allowed characters are single byte.
-				for ; peekPos < length; peekPos = peekPos + 1 {
-					switch source[peekPos] {
+				for ; peek < end; peek = peek + 1 {
+					switch source[peek] {
 					case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '_':
-						p = peekPos
+						i = peek
 						col = col + 1
 					default:
 						break Number // end of this token (or malformed, which the next loop will notice)
@@ -169,8 +174,8 @@ func lex(fileName string, source []byte, parser parseToken) error {
 			hitQuote := false
 			// Start at the second character and run until the end. Note UTF-8 (multi-byte) characters are allowed.
 		String:
-			for peekPos < length {
-				peeked := source[peekPos]
+			for peek < end {
+				peeked := source[peek]
 				if peeked == '"' { // TODO: escaping and banning disallowed characters like newlines.
 					hitQuote = true
 					break String
@@ -181,27 +186,27 @@ func lex(fileName string, source []byte, parser parseToken) error {
 				if s == 0 {
 					return fmt.Errorf("%s:%d:%d: found an invalid byte in string token: 0x%x", fileName, line, col, peeked)
 				}
-				peekPos = peekPos + s
+				peek = peek + s
 			}
 
 			if !hitQuote {
 				return fmt.Errorf("%s:%d:%d: expected end quote", fileName, line, col)
 			}
 
-			p = peekPos
+			i = peek
 			// set the position to after the quote
-			peekPos = peekPos + 1
+			peek = peek + 1
 			col = col + 1
 		case tokenKeyword, tokenID, tokenReserved: // min 1 byte; end with zero or more idChar
 			// Start after the first character and run until the end. Note all allowed characters are single byte.
 		IdChars:
-			for ; peekPos < length; peekPos = peekPos + 1 {
-				if !idChar[source[peekPos]] {
+			for ; peek < end; peek = peek + 1 {
+				if !idChar[source[peek]] {
 					break IdChars // end of this token (or malformed, which the next loop will notice)
 				}
 				col = col + 1
 			}
-			p = peekPos - 1
+			i = peek - 1
 		default:
 			if b1 > 0x7F { // non-ASCII
 				r, _ := utf8.DecodeRune(source[line:])
@@ -216,16 +221,16 @@ func lex(fileName string, source []byte, parser parseToken) error {
 		//
 		// TODO: Ex. inf nan nan:0xfffffffffffff or nan:0x400000
 
-		if e := parser(source, tok, line, beginCol, beginPos, peekPos); e != nil {
+		if e := parser(tok, source[b:peek], line, c); e != nil {
 			return e
 		}
 	}
 
 	if blockCommentDepth > 0 {
-		return fmt.Errorf("%s:%d:%d: expected block comment end ';)', but reached end of input", fileName, line, col+1)
+		return fmt.Errorf("%s:%d:%d: expected block comment end ';)', but reached end of input", fileName, line, col)
 	}
 	if parenDepth > 0 {
-		return fmt.Errorf("%s:%d:%d: expected ')', but reached end of input", fileName, line, col+1)
+		return fmt.Errorf("%s:%d:%d: expected ')', but reached end of input", fileName, line, col)
 	}
 	return nil // EOF
 }


Test patch:
diff --git a/wasm/wat/lexer_test.go b/wasm/wat/lexer_test.go
index 867a5acd..bde8ae15 100644
--- a/wasm/wat/lexer_test.go
+++ b/wasm/wat/lexer_test.go
@@ -39,150 +39,150 @@ func TestLex_Example(t *testing.T) {
 	tokens, e := lexTokens(exampleWat)
 	require.NoError(t, e)
 	require.Equal(t, []*token{
-		{tokenLParen, 1, 1, 0, "("},
-		{tokenKeyword, 1, 2, 1, "module"},
-		{tokenLParen, 3, 3, 118, "("},
-		{tokenKeyword, 3, 4, 119, "memory"},
-		{tokenUN, 3, 11, 126, "1"},
-		{tokenRParen, 3, 12, 127, ")"},
-		{tokenLParen, 4, 3, 131, "("},
-		{tokenKeyword, 4, 4, 132, "func"},
-		{tokenID, 4, 9, 137, "$main"},
-		{tokenLParen, 4, 15, 143, "("},
-		{tokenKeyword, 4, 16, 144, "local"},
-		{tokenKeyword, 4, 22, 150, "i32"},
-		{tokenKeyword, 4, 26, 154, "i32"},
-		{tokenKeyword, 4, 30, 158, "i32"},
-		{tokenKeyword, 4, 34, 162, "i32"},
-		{tokenRParen, 4, 37, 165, ")"},
-		{tokenLParen, 5, 5, 171, "("},
-		{tokenKeyword, 5, 6, 172, "set_local"},
-		{tokenUN, 5, 16, 182, "0"},
-		{tokenLParen, 5, 18, 184, "("},
-		{tokenKeyword, 5, 19, 185, "i32.const"},
-		{tokenUN, 5, 29, 195, "0"},
-		{tokenRParen, 5, 30, 196, ")"},
-		{tokenRParen, 5, 31, 197, ")"},
-		{tokenLParen, 6, 5, 203, "("},
-		{tokenKeyword, 6, 6, 204, "set_local"},
-		{tokenUN, 6, 16, 214, "1"},
-		{tokenLParen, 6, 18, 216, "("},
-		{tokenKeyword, 6, 19, 217, "i32.const"},
-		{tokenUN, 6, 29, 227, "1"},
-		{tokenRParen, 6, 30, 228, ")"},
-		{tokenRParen, 6, 31, 229, ")"},
-		{tokenLParen, 7, 5, 235, "("},
-		{tokenKeyword, 7, 6, 236, "set_local"},
-		{tokenUN, 7, 16, 246, "2"},
-		{tokenLParen, 7, 18, 248, "("},
-		{tokenKeyword, 7, 19, 249, "i32.const"},
-		{tokenUN, 7, 29, 259, "1"},
-		{tokenRParen, 7, 30, 260, ")"},
-		{tokenRParen, 7, 31, 261, ")"},
-		{tokenLParen, 8, 5, 267, "("},
-		{tokenKeyword, 8, 6, 268, "set_local"},
-		{tokenUN, 8, 16, 278, "3"},
-		{tokenLParen, 8, 18, 280, "("},
-		{tokenKeyword, 8, 19, 281, "i32.const"},
-		{tokenUN, 8, 29, 291, "0"},
-		{tokenRParen, 8, 30, 292, ")"},
-		{tokenRParen, 8, 31, 293, ")"},
-		{tokenLParen, 9, 5, 299, "("},
-		{tokenKeyword, 9, 6, 300, "block"},
-		{tokenLParen, 10, 5, 310, "("},
-		{tokenKeyword, 10, 6, 311, "loop"},
-		{tokenLParen, 11, 9, 324, "("},
-		{tokenKeyword, 11, 10, 325, "br_if"},
-		{tokenUN, 11, 16, 331, "1"},
-		{tokenLParen, 11, 18, 333, "("},
-		{tokenKeyword, 11, 19, 334, "i32.gt_s"},
-		{tokenLParen, 11, 28, 343, "("},
-		{tokenKeyword, 11, 29, 344, "get_local"},
-		{tokenUN, 11, 39, 354, "0"},
-		{tokenRParen, 11, 40, 355, ")"},
-		{tokenLParen, 11, 42, 357, "("},
-		{tokenKeyword, 11, 43, 358, "i32.const"},
-		{tokenUN, 11, 53, 368, "5"},
-		{tokenRParen, 11, 54, 369, ")"},
-		{tokenRParen, 11, 55, 370, ")"},
-		{tokenRParen, 11, 56, 371, ")"},
-		{tokenLParen, 12, 9, 381, "("},
-		{tokenKeyword, 12, 10, 382, "set_local"},
-		{tokenUN, 12, 20, 392, "3"},
-		{tokenLParen, 12, 22, 394, "("},
-		{tokenKeyword, 12, 23, 395, "get_local"},
-		{tokenUN, 12, 33, 405, "2"},
-		{tokenRParen, 12, 34, 406, ")"},
-		{tokenRParen, 12, 35, 407, ")"},
-		{tokenLParen, 13, 9, 417, "("},
-		{tokenKeyword, 13, 10, 418, "set_local"},
-		{tokenUN, 13, 20, 428, "2"},
-		{tokenLParen, 13, 22, 430, "("},
-		{tokenKeyword, 13, 23, 431, "i32.add"},
-		{tokenLParen, 13, 31, 439, "("},
-		{tokenKeyword, 13, 32, 440, "get_local"},
-		{tokenUN, 13, 42, 450, "2"},
-		{tokenRParen, 13, 43, 451, ")"},
-		{tokenLParen, 13, 45, 453, "("},
-		{tokenKeyword, 13, 46, 454, "get_local"},
-		{tokenUN, 13, 56, 464, "1"},
-		{tokenRParen, 13, 57, 465, ")"},
-		{tokenRParen, 13, 58, 466, ")"},
-		{tokenRParen, 13, 59, 467, ")"},
-		{tokenLParen, 14, 9, 477, "("},
-		{tokenKeyword, 14, 10, 478, "set_local"},
-		{tokenUN, 14, 20, 488, "1"},
-		{tokenLParen, 14, 22, 490, "("},
-		{tokenKeyword, 14, 23, 491, "get_local"},
-		{tokenUN, 14, 33, 501, "3"},
-		{tokenRParen, 14, 34, 502, ")"},
-		{tokenRParen, 14, 35, 503, ")"},
-		{tokenLParen, 15, 9, 513, "("},
-		{tokenKeyword, 15, 10, 514, "set_local"},
-		{tokenUN, 15, 20, 524, "0"},
-		{tokenLParen, 15, 22, 526, "("},
-		{tokenKeyword, 15, 23, 527, "i32.add"},
-		{tokenLParen, 15, 31, 535, "("},
-		{tokenKeyword, 15, 32, 536, "get_local"},
-		{tokenUN, 15, 42, 546, "0"},
-		{tokenRParen, 15, 43, 547, ")"},
-		{tokenLParen, 15, 45, 549, "("},
-		{tokenKeyword, 15, 46, 550, "i32.const"},
-		{tokenUN, 15, 56, 560, "1"},
-		{tokenRParen, 15, 57, 561, ")"},
-		{tokenRParen, 15, 58, 562, ")"},
-		{tokenRParen, 15, 59, 563, ")"},
-		{tokenLParen, 16, 9, 573, "("},
-		{tokenKeyword, 16, 10, 574, "br"},
-		{tokenUN, 16, 13, 577, "0"},
-		{tokenRParen, 16, 14, 578, ")"},
-		{tokenRParen, 17, 5, 584, ")"},
-		{tokenRParen, 18, 5, 590, ")"},
-		{tokenLParen, 19, 5, 596, "("},
-		{tokenKeyword, 19, 6, 597, "i32.store"},
-		{tokenLParen, 19, 16, 607, "("},
-		{tokenKeyword, 19, 17, 608, "i32.const"},
-		{tokenUN, 19, 27, 618, "0"},
-		{tokenRParen, 19, 28, 619, ")"},
-		{tokenLParen, 19, 30, 621, "("},
-		{tokenKeyword, 19, 31, 622, "get_local"},
-		{tokenUN, 19, 41, 632, "2"},
-		{tokenRParen, 19, 42, 633, ")"},
-		{tokenRParen, 19, 43, 634, ")"},
-		{tokenRParen, 20, 3, 638, ")"},
-		{tokenLParen, 21, 3, 642, "("},
-		{tokenKeyword, 21, 4, 643, "start"},
-		{tokenID, 21, 10, 649, "$main"},
-		{tokenRParen, 21, 15, 654, ")"},
-		{tokenLParen, 22, 3, 658, "("},
-		{tokenKeyword, 22, 4, 659, "data"},
-		{tokenLParen, 22, 9, 664, "("},
-		{tokenKeyword, 22, 10, 665, "i32.const"},
-		{tokenUN, 22, 20, 675, "0"},
-		{tokenRParen, 22, 21, 676, ")"},
-		{tokenString, 22, 23, 678, "\"0000\""},
-		{tokenRParen, 22, 29, 684, ")"},
-		{tokenRParen, 23, 1, 686, ")"},
+		{tokenLParen, 1, 1, "("},
+		{tokenKeyword, 1, 2, "module"},
+		{tokenLParen, 3, 3, "("},
+		{tokenKeyword, 3, 4, "memory"},
+		{tokenUN, 3, 11, "1"},
+		{tokenRParen, 3, 12, ")"},
+		{tokenLParen, 4, 3, "("},
+		{tokenKeyword, 4, 4, "func"},
+		{tokenID, 4, 9, "$main"},
+		{tokenLParen, 4, 15, "("},
+		{tokenKeyword, 4, 16, "local"},
+		{tokenKeyword, 4, 22, "i32"},
+		{tokenKeyword, 4, 26, "i32"},
+		{tokenKeyword, 4, 30, "i32"},
+		{tokenKeyword, 4, 34, "i32"},
+		{tokenRParen, 4, 37, ")"},
+		{tokenLParen, 5, 5, "("},
+		{tokenKeyword, 5, 6, "set_local"},
+		{tokenUN, 5, 16, "0"},
+		{tokenLParen, 5, 18, "("},
+		{tokenKeyword, 5, 19, "i32.const"},
+		{tokenUN, 5, 29, "0"},
+		{tokenRParen, 5, 30, ")"},
+		{tokenRParen, 5, 31, ")"},
+		{tokenLParen, 6, 5, "("},
+		{tokenKeyword, 6, 6, "set_local"},
+		{tokenUN, 6, 16, "1"},
+		{tokenLParen, 6, 18, "("},
+		{tokenKeyword, 6, 19, "i32.const"},
+		{tokenUN, 6, 29, "1"},
+		{tokenRParen, 6, 30, ")"},
+		{tokenRParen, 6, 31, ")"},
+		{tokenLParen, 7, 5, "("},
+		{tokenKeyword, 7, 6, "set_local"},
+		{tokenUN, 7, 16, "2"},
+		{tokenLParen, 7, 18, "("},
+		{tokenKeyword, 7, 19, "i32.const"},
+		{tokenUN, 7, 29, "1"},
+		{tokenRParen, 7, 30, ")"},
+		{tokenRParen, 7, 31, ")"},
+		{tokenLParen, 8, 5, "("},
+		{tokenKeyword, 8, 6, "set_local"},
+		{tokenUN, 8, 16, "3"},
+		{tokenLParen, 8, 18, "("},
+		{tokenKeyword, 8, 19, "i32.const"},
+		{tokenUN, 8, 29, "0"},
+		{tokenRParen, 8, 30, ")"},
+		{tokenRParen, 8, 31, ")"},
+		{tokenLParen, 9, 5, "("},
+		{tokenKeyword, 9, 6, "block"},
+		{tokenLParen, 10, 5, "("},
+		{tokenKeyword, 10, 6, "loop"},
+		{tokenLParen, 11, 9, "("},
+		{tokenKeyword, 11, 10, "br_if"},
+		{tokenUN, 11, 16, "1"},
+		{tokenLParen, 11, 18, "("},
+		{tokenKeyword, 11, 19, "i32.gt_s"},
+		{tokenLParen, 11, 28, "("},
+		{tokenKeyword, 11, 29, "get_local"},
+		{tokenUN, 11, 39, "0"},
+		{tokenRParen, 11, 40, ")"},
+		{tokenLParen, 11, 42, "("},
+		{tokenKeyword, 11, 43, "i32.const"},
+		{tokenUN, 11, 53, "5"},
+		{tokenRParen, 11, 54, ")"},
+		{tokenRParen, 11, 55, ")"},
+		{tokenRParen, 11, 56, ")"},
+		{tokenLParen, 12, 9, "("},
+		{tokenKeyword, 12, 10, "set_local"},
+		{tokenUN, 12, 20, "3"},
+		{tokenLParen, 12, 22, "("},
+		{tokenKeyword, 12, 23, "get_local"},
+		{tokenUN, 12, 33, "2"},
+		{tokenRParen, 12, 34, ")"},
+		{tokenRParen, 12, 35, ")"},
+		{tokenLParen, 13, 9, "("},
+		{tokenKeyword, 13, 10, "set_local"},
+		{tokenUN, 13, 20, "2"},
+		{tokenLParen, 13, 22, "("},
+		{tokenKeyword, 13, 23, "i32.add"},
+		{tokenLParen, 13, 31, "("},
+		{tokenKeyword, 13, 32, "get_local"},
+		{tokenUN, 13, 42, "2"},
+		{tokenRParen, 13, 43, ")"},
+		{tokenLParen, 13, 45, "("},
+		{tokenKeyword, 13, 46, "get_local"},
+		{tokenUN, 13, 56, "1"},
+		{tokenRParen, 13, 57, ")"},
+		{tokenRParen, 13, 58, ")"},
+		{tokenRParen, 13, 59, ")"},
+		{tokenLParen, 14, 9, "("},
+		{tokenKeyword, 14, 10, "set_local"},
+		{tokenUN, 14, 20, "1"},
+		{tokenLParen, 14, 22, "("},
+		{tokenKeyword, 14, 23, "get_local"},
+		{tokenUN, 14, 33, "3"},
+		{tokenRParen, 14, 34, ")"},
+		{tokenRParen, 14, 35, ")"},
+		{tokenLParen, 15, 9, "("},
+		{tokenKeyword, 15, 10, "set_local"},
+		{tokenUN, 15, 20, "0"},
+		{tokenLParen, 15, 22, "("},
+		{tokenKeyword, 15, 23, "i32.add"},
+		{tokenLParen, 15, 31, "("},
+		{tokenKeyword, 15, 32, "get_local"},
+		{tokenUN, 15, 42, "0"},
+		{tokenRParen, 15, 43, ")"},
+		{tokenLParen, 15, 45, "("},
+		{tokenKeyword, 15, 46, "i32.const"},
+		{tokenUN, 15, 56, "1"},
+		{tokenRParen, 15, 57, ")"},
+		{tokenRParen, 15, 58, ")"},
+		{tokenRParen, 15, 59, ")"},
+		{tokenLParen, 16, 9, "("},
+		{tokenKeyword, 16, 10, "br"},
+		{tokenUN, 16, 13, "0"},
+		{tokenRParen, 16, 14, ")"},
+		{tokenRParen, 17, 5, ")"},
+		{tokenRParen, 18, 5, ")"},
+		{tokenLParen, 19, 5, "("},
+		{tokenKeyword, 19, 6, "i32.store"},
+		{tokenLParen, 19, 16, "("},
+		{tokenKeyword, 19, 17, "i32.const"},
+		{tokenUN, 19, 27, "0"},
+		{tokenRParen, 19, 28, ")"},
+		{tokenLParen, 19, 30, "("},
+		{tokenKeyword, 19, 31, "get_local"},
+		{tokenUN, 19, 41, "2"},
+		{tokenRParen, 19, 42, ")"},
+		{tokenRParen, 19, 43, ")"},
+		{tokenRParen, 20, 3, ")"},
+		{tokenLParen, 21, 3, "("},
+		{tokenKeyword, 21, 4, "start"},
+		{tokenID, 21, 10, "$main"},
+		{tokenRParen, 21, 15, ")"},
+		{tokenLParen, 22, 3, "("},
+		{tokenKeyword, 22, 4, "data"},
+		{tokenLParen, 22, 9, "("},
+		{tokenKeyword, 22, 10, "i32.const"},
+		{tokenUN, 22, 20, "0"},
+		{tokenRParen, 22, 21, ")"},
+		{tokenString, 22, 23, "\"0000\""},
+		{tokenRParen, 22, 29, ")"},
+		{tokenRParen, 23, 1, ")"},
 	}, tokens)
 }
 
@@ -199,22 +199,22 @@ func TestLex(t *testing.T) {
 		{
 			name:     "only parens",
 			input:    "()",
-			expected: []*token{{tokenLParen, 1, 1, 0, "("}, {tokenRParen, 1, 2, 1, ")"}},
+			expected: []*token{{tokenLParen, 1, 1, "("}, {tokenRParen, 1, 2, ")"}},
 		},
 		{
 			name:     "shortest keywords",
 			input:    "a z",
-			expected: []*token{{tokenKeyword, 1, 1, 0, "a"}, {tokenKeyword, 1, 3, 2, "z"}},
+			expected: []*token{{tokenKeyword, 1, 1, "a"}, {tokenKeyword, 1, 3, "z"}},
 		},
 		{
 			name:     "shortest tokens - EOL",
 			input:    "(a)\n",
-			expected: []*token{{tokenLParen, 1, 1, 0, "("}, {tokenKeyword, 1, 2, 1, "a"}, {tokenRParen, 1, 3, 2, ")"}},
+			expected: []*token{{tokenLParen, 1, 1, "("}, {tokenKeyword, 1, 2, "a"}, {tokenRParen, 1, 3, ")"}},
 		},
 		{
 			name:     "only tokens",
 			input:    "(module)",
-			expected: []*token{{tokenLParen, 1, 1, 0, "("}, {tokenKeyword, 1, 2, 1, "module"}, {tokenRParen, 1, 8, 7, ")"}},
+			expected: []*token{{tokenLParen, 1, 1, "("}, {tokenKeyword, 1, 2, "module"}, {tokenRParen, 1, 8, ")"}},
 		},
 		{
 			name:  "only white space characters",
@@ -223,12 +223,12 @@ func TestLex(t *testing.T) {
 		{
 			name:     "after white space characters - EOL",
 			input:    " \t\na",
-			expected: []*token{{tokenKeyword, 2, 1, 3, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:     "after white space characters - Windows EOL",
 			input:    " \t\r\na",
-			expected: []*token{{tokenKeyword, 2, 1, 4, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:  "only line comment - EOL before EOF",
@@ -245,42 +245,42 @@ func TestLex(t *testing.T) {
 		{
 			name:     "after line comment",
 			input:    ";; TODO\na",
-			expected: []*token{{tokenKeyword, 2, 1, 8, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:     "double line comment",
 			input:    ";; TODO\n;; YOLO\na",
-			expected: []*token{{tokenKeyword, 3, 1, 16, "a"}},
+			expected: []*token{{tokenKeyword, 3, 1, "a"}},
 		},
 		{
 			name:     "after unicode line comment",
 			input:    ";; брэд-ЛГТМ\na",
-			expected: []*token{{tokenKeyword, 2, 1, 21, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:     "after line comment - Windows EOL",
 			input:    ";; TODO\r\na",
-			expected: []*token{{tokenKeyword, 2, 1, 9, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:     "after redundant line comment",
 			input:    ";;;; TODO\na",
-			expected: []*token{{tokenKeyword, 2, 1, 10, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:     "after line commenting out block comment",
 			input:    ";; TODO (; ;)\na",
-			expected: []*token{{tokenKeyword, 2, 1, 14, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:     "after line commenting out open block comment",
 			input:    ";; TODO (;\na",
-			expected: []*token{{tokenKeyword, 2, 1, 11, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:     "after line commenting out close block comment",
 			input:    ";; TODO ;)\na",
-			expected: []*token{{tokenKeyword, 2, 1, 11, "a"}},
+			expected: []*token{{tokenKeyword, 2, 1, "a"}},
 		},
 		{
 			name:  "only block comment - EOL before EOF",
@@ -297,17 +297,17 @@ func TestLex(t *testing.T) {
 		{
 			name:     "double block comment",
 			input:    "(; TODO ;)(; YOLO ;)a",
-			expected: []*token{{tokenKeyword, 1, 21, 20, "a"}},
+			expected: []*token{{tokenKeyword, 1, 21, "a"}},
 		},
 		{
 			name:     "double block comment - EOL",
 			input:    "(; TODO ;)\n(; YOLO ;)\na",
-			expected: []*token{{tokenKeyword, 3, 1, 22, "a"}},
+			expected: []*token{{tokenKeyword, 3, 1, "a"}},
 		},
 		{
 			name:     "after block comment",
 			input:    "(; TODO ;)a",
-			expected: []*token{{tokenKeyword, 1, 11, 10, "a"}},
+			expected: []*token{{tokenKeyword, 1, 11, "a"}},
 		},
 		{
 			name:  "only nested block comment - EOL before EOF",
@@ -324,92 +324,92 @@ func TestLex(t *testing.T) {
 		{
 			name:     "after nested block comment",
 			input:    "(; TODO (; (YOLO) ;) ;)a",
-			expected: []*token{{tokenKeyword, 1, 24, 23, "a"}},
+			expected: []*token{{tokenKeyword, 1, 24, "a"}},
 		},
 		{
 			name:     "after nested block comment - EOL",
 			input:    "(; TODO (; (YOLO) ;) ;)\n a",
-			expected: []*token{{tokenKeyword, 2, 2, 25, "a"}},
+			expected: []*token{{tokenKeyword, 2, 2, "a"}},
 		},
 		{
 			name:     "after nested block comment - Windows EOL",
 			input:    "(; TODO (; (YOLO) ;) ;)\r\n a",
-			expected: []*token{{tokenKeyword, 2, 2, 26, "a"}},
+			expected: []*token{{tokenKeyword, 2, 2, "a"}},
 		},
 		{
 			name:     "white space between parens",
 			input:    "( )",
-			expected: []*token{{tokenLParen, 1, 1, 0, "("}, {tokenRParen, 1, 3, 2, ")"}},
+			expected: []*token{{tokenLParen, 1, 1, "("}, {tokenRParen, 1, 3, ")"}},
 		},
 		{
 			name:  "nested parens",
 			input: "(())",
 			expected: []*token{
-				{tokenLParen, 1, 1, 0, "("},
-				{tokenLParen, 1, 2, 1, "("},
-				{tokenRParen, 1, 3, 2, ")"},
-				{tokenRParen, 1, 4, 3, ")"},
+				{tokenLParen, 1, 1, "("},
+				{tokenLParen, 1, 2, "("},
+				{tokenRParen, 1, 3, ")"},
+				{tokenRParen, 1, 4, ")"},
 			},
 		},
 		{
 			name:     "empty string",
 			input:    `""`,
-			expected: []*token{{tokenString, 1, 1, 0, `""`}},
+			expected: []*token{{tokenString, 1, 1, `""`}},
 		},
 		{
 			name:     "unicode string",
 			input:    "\"брэд-ЛГТМ\"",
-			expected: []*token{{tokenString, 1, 1, 0, "\"брэд-ЛГТМ\""}},
+			expected: []*token{{tokenString, 1, 1, "\"брэд-ЛГТМ\""}},
 		},
 		{
 			name:     "string inside tokens with newline",
 			input:    "(\"\n\")", // TODO newline char isn't actually allowed unless escaped!
-			expected: []*token{{tokenLParen, 1, 1, 0, "("}, {tokenString, 1, 2, 1, "\"\n\""}, {tokenRParen, 1, 5, 4, ")"}},
+			expected: []*token{{tokenLParen, 1, 1, "("}, {tokenString, 1, 2, "\"\n\""}, {tokenRParen, 1, 5, ")"}},
 		},
 		{
 			name:     "unsigned shortest - EOL",
 			input:    "1\n",
-			expected: []*token{{tokenUN, 1, 1, 0, "1"}},
+			expected: []*token{{tokenUN, 1, 1, "1"}},
 		},
 		{
 			name:     "unsigned shortest - EOF",
 			input:    "1",
-			expected: []*token{{tokenUN, 1, 1, 0, "1"}},
+			expected: []*token{{tokenUN, 1, 1, "1"}},
 		},
 		{
 			name:     "unsigned shortest inside tokens",
 			input:    "(1)",
-			expected: []*token{{tokenLParen, 1, 1, 0, "("}, {tokenUN, 1, 2, 1, "1"}, {tokenRParen, 1, 3, 2, ")"}},
+			expected: []*token{{tokenLParen, 1, 1, "("}, {tokenUN, 1, 2, "1"}, {tokenRParen, 1, 3, ")"}},
 		},
 		{
 			name:     "unsigned shortest then string",
 			input:    `1"1"`,
-			expected: []*token{{tokenUN, 1, 1, 0, "1"}, {tokenString, 1, 2, 1, `"1"`}},
+			expected: []*token{{tokenUN, 1, 1, "1"}, {tokenString, 1, 2, `"1"`}},
 		},
 		{
 			name:     "unsigned - EOL",
 			input:    "123\n",
-			expected: []*token{{tokenUN, 1, 1, 0, "123"}},
+			expected: []*token{{tokenUN, 1, 1, "123"}},
 		},
 		{
 			name:     "unsigned - EOF",
 			input:    "123",
-			expected: []*token{{tokenUN, 1, 1, 0, "123"}},
+			expected: []*token{{tokenUN, 1, 1, "123"}},
 		},
 		{
 			name:     "unsigned inside tokens",
 			input:    "(123)",
-			expected: []*token{{tokenLParen, 1, 1, 0, "("}, {tokenUN, 1, 2, 1, "123"}, {tokenRParen, 1, 5, 4, ")"}},
+			expected: []*token{{tokenLParen, 1, 1, "("}, {tokenUN, 1, 2, "123"}, {tokenRParen, 1, 5, ")"}},
 		},
 		{
 			name:     "unsigned then string",
 			input:    `123"123"`,
-			expected: []*token{{tokenUN, 1, 1, 0, "123"}, {tokenString, 1, 4, 3, `"123"`}},
+			expected: []*token{{tokenUN, 1, 1, "123"}, {tokenString, 1, 4, `"123"`}},
 		},
 		{
 			name:     "unsigned then keyword",
 			input:    "1a", // whitespace is optional between tokens, and a keyword can be single-character!
-			expected: []*token{{tokenUN, 1, 1, 0, "1"}, {tokenKeyword, 1, 2, 1, "a"}},
+			expected: []*token{{tokenUN, 1, 1, "1"}, {tokenKeyword, 1, 2, "a"}},
 		},
 		{
 			name:  "0x80 in block comment",
@@ -426,7 +426,7 @@ func TestLex(t *testing.T) {
 		{
 			name:     "0x80 in string",
 			input:    "\" \000\"",
-			expected: []*token{{tokenString, 1, 1, 0, "\" \000\""}},
+			expected: []*token{{tokenString, 1, 1, "\" \000\""}},
 		},
 	}
 
@@ -521,21 +521,21 @@ func TestLex_Errors(t *testing.T) {
 	for _, tt := range tests {
 		tc := tt
 		t.Run(tc.name, func(t *testing.T) {
-			require.EqualError(t, lex("test.wasm", tc.input, noopParseToken), tc.expectedErr)
+			require.EqualError(t, lex(noopParseToken, "test.wasm", tc.input), tc.expectedErr)
 		})
 	}
 }
 
 func lexTokens(input string) ([]*token, error) {
 	var tokens []*token
-	e := lex("test.wasm", []byte(input), func(source []byte, tok tokenType, line, col, beginPos, endPos int) (err error) {
-		tokens = append(tokens, &token{tok, line, col, beginPos, string(source[beginPos:endPos])})
+	e := lex(func(tok tokenType, tokenUTF8 []byte, line, col int) (err error) {
+		tokens = append(tokens, &token{tok, line, col, string(tokenUTF8)})
 		return
-	})
+	}, "test.wasm", []byte(input))
 	return tokens, e
 }
 
-var noopParseToken parseToken = func(source []byte, tok tokenType, beginLine, beginCol, beginPos, endPos int) error {
+var noopParseToken parseToken = func(tok tokenType, tokenUTF8 []byte, line, col int) error {
 	return nil
 }
 
@@ -571,7 +571,7 @@ func BenchmarkLex(b *testing.B) {
 		b.Run(bm.name, func(b *testing.B) {
 			b.ReportAllocs()
 			for i := 0; i < b.N; i++ {
-				err := lex("test.wasm", bm.data, noopParseToken)
+				err := lex(noopParseToken, "test.wasm", bm.data)
 				if err != nil {
 					panic(err)
 				}
@@ -582,11 +582,11 @@ func BenchmarkLex(b *testing.B) {
 
 type token struct {
 	tokenType
-	line, col, pos int
-	value          string
+	line, col int
+	token     string
 }
 
 // String helps format to allow copy/pasting of expected values
 func (t *token) String() string {
-	return fmt.Sprintf("{%s, %d, %d, %d, %q}", t.tokenType, t.line, t.col, t.pos, t.value)
+	return fmt.Sprintf("{%s, %d, %d, %q}", t.tokenType, t.line, t.col, t.token)
 }