Repository: d5/tengo (go)

Issue:
All ints are parsed in base 10
Octal and hexadecimal numbers are partially supported (by scanner), but always parsed in base 10.

#### Expected:

```
fmt := import("fmt")

fmt.println(0123)  // 83
fmt.println(123)   // 123
fmt.println(0x123) // 291
```

#### Actual:

```
fmt := import("fmt")

fmt.println(0123)  // 123
fmt.println(123)   // 123
fmt.println(0x123) // 0
```

Pull request description:
Fix parsing oct and hex numbers
Fixes #363 

Patch:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f760294..01aca2d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -14,9 +14,9 @@ jobs:
       - name: set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.16
       - name: run goreleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v3
         with:
           version: latest
           args: release --rm-dist
diff --git a/compiler.go b/compiler.go
index e4e0430..6591603 100644
--- a/compiler.go
+++ b/compiler.go
@@ -1220,14 +1220,14 @@ func (c *Compiler) optimizeFunc(node parser.Node) {
 	iterateInstructions(c.scopes[c.scopeIndex].Instructions,
 		func(pos int, opcode parser.Opcode, operands []int) bool {
 			switch {
+			case dsts[pos]:
+				dstIdx++
+				deadCode = false
 			case opcode == parser.OpReturn:
 				if deadCode {
 					return true
 				}
 				deadCode = true
-			case dsts[pos]:
-				dstIdx++
-				deadCode = false
 			case deadCode:
 				return true
 			}
@@ -1242,6 +1242,7 @@ func (c *Compiler) optimizeFunc(node parser.Node) {
 	var appendReturn bool
 	endPos := len(c.scopes[c.scopeIndex].Instructions)
 	newEndPost := len(newInsts)
+
 	iterateInstructions(newInsts,
 		func(pos int, opcode parser.Opcode, operands []int) bool {
 			switch opcode {
diff --git a/docs/stdlib-base64.md b/docs/stdlib-base64.md
index 93548e0..64a5ab3 100644
--- a/docs/stdlib-base64.md
+++ b/docs/stdlib-base64.md
@@ -1,7 +1,7 @@
 # Module - "base64"
 
 ```golang
-fmt := import("base64")
+base64 := import("base64")
 ```
 
 ## Functions
diff --git a/docs/stdlib-hex.md b/docs/stdlib-hex.md
index fc2eb58..68bbf45 100644
--- a/docs/stdlib-hex.md
+++ b/docs/stdlib-hex.md
@@ -1,7 +1,7 @@
 # Module - "hex"
 
 ```golang
-fmt := import("hex")
+hex := import("hex")
 ```
 
 ## Functions
diff --git a/parser/parser.go b/parser/parser.go
index fd20423..eaa9dc9 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -375,7 +375,12 @@ func (p *Parser) parseOperand() Expr {
 	case token.Ident:
 		return p.parseIdent()
 	case token.Int:
-		v, _ := strconv.ParseInt(p.tokenLit, 10, 64)
+		v, err := strconv.ParseInt(p.tokenLit, 0, 64)
+		if err == strconv.ErrRange {
+			p.error(p.pos, "number out of range")
+		} else if err != nil {
+			p.error(p.pos, "invalid integer")
+		}
 		x := &IntLit{
 			Value:    v,
 			ValuePos: p.pos,
@@ -383,8 +388,14 @@ func (p *Parser) parseOperand() Expr {
 		}
 		p.next()
 		return x
+
 	case token.Float:
-		v, _ := strconv.ParseFloat(p.tokenLit, 64)
+		v, err := strconv.ParseFloat(p.tokenLit, 64)
+		if err == strconv.ErrRange {
+			p.error(p.pos, "number out of range")
+		} else if err != nil {
+			p.error(p.pos, "invalid float")
+		}
 		x := &FloatLit{
 			Value:    v,
 			ValuePos: p.pos,
@@ -447,10 +458,11 @@ func (p *Parser) parseOperand() Expr {
 		return p.parseErrorExpr()
 	case token.Immutable: // immutable expression
 		return p.parseImmutableExpr()
+	default:
+		p.errorExpected(p.pos, "operand")
 	}
 
 	pos := p.pos
-	p.errorExpected(pos, "operand")
 	p.advance(stmtStart)
 	return &BadExpr{From: pos, To: p.pos}
 }
diff --git a/parser/scanner.go b/parser/scanner.go
index f1d820a..5f79770 100644
--- a/parser/scanner.go
+++ b/parser/scanner.go
@@ -93,9 +93,9 @@ func (s *Scanner) Scan() (
 			token.Export, token.True, token.False, token.Undefined:
 			insertSemi = true
 		}
-	case '0' <= ch && ch <= '9':
+	case ('0' <= ch && ch <= '9') || (ch == '.' && '0' <= s.peek() && s.peek() <= '9'):
 		insertSemi = true
-		tok, literal = s.scanNumber(false)
+		tok, literal = s.scanNumber()
 	default:
 		s.next() // always make progress
 
@@ -125,16 +125,11 @@ func (s *Scanner) Scan() (
 		case ':':
 			tok = s.switch2(token.Colon, token.Define)
 		case '.':
-			if '0' <= s.ch && s.ch <= '9' {
-				insertSemi = true
-				tok, literal = s.scanNumber(true)
-			} else {
-				tok = token.Period
-				if s.ch == '.' && s.peek() == '.' {
-					s.next()
-					s.next() // consume last '.'
-					tok = token.Ellipsis
-				}
+			tok = token.Period
+			if s.ch == '.' && s.peek() == '.' {
+				s.next()
+				s.next() // consume last '.'
+				tok = token.Ellipsis
 			}
 		case ',':
 			tok = token.Comma
@@ -379,86 +374,58 @@ func (s *Scanner) scanIdentifier() string {
 	return string(s.src[offs:s.offset])
 }
 
-func (s *Scanner) scanMantissa(base int) {
-	for digitVal(s.ch) < base {
+func (s *Scanner) scanDigits(base int) {
+	for s.ch == '_' || digitVal(s.ch) < base {
 		s.next()
 	}
 }
 
-func (s *Scanner) scanNumber(
-	seenDecimalPoint bool,
-) (tok token.Token, lit string) {
-	// digitVal(s.ch) < 10
+func (s *Scanner) scanNumber() (token.Token, string) {
 	offs := s.offset
-	tok = token.Int
-
-	defer func() {
-		lit = string(s.src[offs:s.offset])
-	}()
-
-	if seenDecimalPoint {
-		offs--
-		tok = token.Float
-		s.scanMantissa(10)
-		goto exponent
-	}
+	tok := token.Int
+	base := 10
 
-	if s.ch == '0' {
-		// int or float
-		offs := s.offset
+	// Determine base
+	switch {
+	case s.ch == '0' && lower(s.peek()) == 'b':
+		base = 2
+		s.next()
+		s.next()
+	case s.ch == '0' && lower(s.peek()) == 'o':
+		base = 8
+		s.next()
+		s.next()
+	case s.ch == '0' && lower(s.peek()) == 'x':
+		base = 16
+		s.next()
 		s.next()
-		if s.ch == 'x' || s.ch == 'X' {
-			// hexadecimal int
-			s.next()
-			s.scanMantissa(16)
-			if s.offset-offs <= 2 {
-				// only scanned "0x" or "0X"
-				s.error(offs, "illegal hexadecimal number")
-			}
-		} else {
-			// octal int or float
-			seenDecimalDigit := false
-			s.scanMantissa(8)
-			if s.ch == '8' || s.ch == '9' {
-				// illegal octal int or float
-				seenDecimalDigit = true
-				s.scanMantissa(10)
-			}
-			if s.ch == '.' || s.ch == 'e' || s.ch == 'E' || s.ch == 'i' {
-				goto fraction
-			}
-			// octal int
-			if seenDecimalDigit {
-				s.error(offs, "illegal octal number")
-			}
-		}
-		return
 	}
 
-	// decimal int or float
-	s.scanMantissa(10)
+	// Scan whole number
+	s.scanDigits(base)
 
-fraction:
-	if s.ch == '.' {
+	// Scan fractional part
+	if s.ch == '.' && (base == 10 || base == 16) {
 		tok = token.Float
 		s.next()
-		s.scanMantissa(10)
+		s.scanDigits(base)
 	}
 
-exponent:
-	if s.ch == 'e' || s.ch == 'E' {
+	// Scan exponent
+	if s.ch == 'e' || s.ch == 'E' || s.ch == 'p' || s.ch == 'P' {
 		tok = token.Float
 		s.next()
 		if s.ch == '-' || s.ch == '+' {
 			s.next()
 		}
-		if digitVal(s.ch) < 10 {
-			s.scanMantissa(10)
-		} else {
-			s.error(offs, "illegal floating-point exponent")
+		offs := s.offset
+		s.scanDigits(10)
+		if offs == s.offset {
+			s.error(offs, "exponent has no digits")
 		}
 	}
-	return
+
+	return tok, string(s.src[offs:s.offset])
 }
 
 func (s *Scanner) scanEscape(quote rune) bool {
@@ -687,3 +654,7 @@ func digitVal(ch rune) int {
 	}
 	return 16 // larger than any legal digit val
 }
+
+func lower(c byte) byte {
+	return c | ('x' - 'X')
+}


Test patch:
diff --git a/compiler_test.go b/compiler_test.go
index 21d1f02..fd1d422 100644
--- a/compiler_test.go
+++ b/compiler_test.go
@@ -1159,6 +1159,30 @@ func() {
 				tengo.MakeInstruction(parser.OpReturn, 1),
 				tengo.MakeInstruction(parser.OpConstant, 1),
 				tengo.MakeInstruction(parser.OpReturn, 1)))))
+
+	expectCompile(t, `
+func() {
+	if true {
+		return
+	}
+
+    return
+
+    return 123
+}`, bytecode(
+		concatInsts(
+			tengo.MakeInstruction(parser.OpConstant, 1),
+			tengo.MakeInstruction(parser.OpPop),
+			tengo.MakeInstruction(parser.OpSuspend)),
+		objectsArray(
+			intObject(123),
+			compiledFunction(0, 0,
+				tengo.MakeInstruction(parser.OpTrue),
+				tengo.MakeInstruction(parser.OpJumpFalsy, 6),
+				tengo.MakeInstruction(parser.OpReturn, 0),
+				tengo.MakeInstruction(parser.OpReturn, 0),
+				tengo.MakeInstruction(parser.OpConstant, 0),
+				tengo.MakeInstruction(parser.OpReturn, 1)))))
 }
 
 func TestCompilerScopes(t *testing.T) {
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 4ee6ad7..8c2c530 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -41,8 +42,8 @@ func TestParseArray(t *testing.T) {
 
 	expectParse(t, `
 [
-	1, 
-	2, 
+	1,
+	2,
 	3
 ]`, func(p pfn) []Stmt {
 		return stmts(
@@ -54,8 +55,8 @@ func TestParseArray(t *testing.T) {
 	})
 	expectParse(t, `
 [
-	1, 
-	2, 
+	1,
+	2,
 	3
 
 ]`, func(p pfn) []Stmt {
@@ -113,14 +114,14 @@ func TestParseArray(t *testing.T) {
 	expectParseError(t, `[1, 2, 3,]`)
 	expectParseError(t, `
 [
-	1, 
-	2, 
+	1,
+	2,
 	3,
 ]`)
 	expectParseError(t, `
 [
-	1, 
-	2, 
+	1,
+	2,
 	3,
 
 ]`)
@@ -497,8 +498,8 @@ c`, func(p pfn) []Stmt {
 		"x := (a ? (b ? c : d) : e)")
 
 	// ? : should be at the end of each line if it's multi-line
-	expectParseError(t, `a 
-? b 
+	expectParseError(t, `a
+? b
 : c`)
 	expectParseError(t, `a ? (b : e)`)
 	expectParseError(t, `(a ? b) : e`)
@@ -892,13 +893,13 @@ func TestParseIf(t *testing.T) {
 		})
 
 	expectParse(t, `
-if a == 5 { 
-	b = 3 
+if a == 5 {
+	b = 3
 	c = 1
-} else if d == 3 { 
+} else if d == 3 {
 	e = 8
 	f = 3
-} else { 
+} else {
 	g = 2
 	h = 4
 }`, func(p pfn) []Stmt {
@@ -1505,6 +1506,153 @@ func TestParseString(t *testing.T) {
 	})
 }
 
+func TestParseInt(t *testing.T) {
+	testCases := []string{
+		// All valid digits
+		"1234567890",
+		"0b10",
+		"0o12345670",
+		"0x123456789abcdef0",
+		"0x123456789ABCDEF0",
+
+		// Alternative base prefixes
+		"010",
+		"0B10",
+		"0O10",
+		"0X10",
+
+		// Invalid digits
+		"0b2",
+		"08",
+		"0o8",
+		"1a",
+		"0xg",
+
+		// Range errors
+		"9223372036854775807",
+		"9223372036854775808", // invalid: range error
+
+		// Examples from specification (https://go.dev/ref/spec#Integer_literals)
+		"42",
+		"4_2",
+		"0600",
+		"0_600",
+		"0o600",
+		"0O600", // second character is capital letter 'O'
+		"0xBadFace",
+		"0xBad_Face",
+		"0x_67_7a_2f_cc_40_c6",
+		"170141183460469231731687303715884105727",
+		"170_141183_460469_231731_687303_715884_105727",
+		"42_",        // invalid: _ must separate successive digits
+		"4__2",       // invalid: only one _ at a time
+		"0_xBadFace", // invalid: _ must separate successive digits
+	}
+
+	for _, num := range testCases {
+		t.Run(num, func(t *testing.T) {
+			expected, err := strconv.ParseInt(num, 0, 64)
+			if err == nil {
+				expectParse(t, num, func(p pfn) []Stmt {
+					return stmts(exprStmt(intLit(expected, p(1, 1))))
+				})
+			} else {
+				expectParseError(t, num)
+			}
+		})
+	}
+}
+
+func TestParseFloat(t *testing.T) {
+	testCases := []string{
+		// Different placements of decimal point
+		".0",
+		"0.",
+		"0.0",
+		"00.0",
+		"00.00",
+		"0.0.0",
+		"0..0",
+
+		// Ignoring leading zeros
+		"010.0",
+		"00010.0",
+		"08.0",
+		"0a.0", // ivalid: hex character
+
+		// Exponents
+		"1e1",
+		"1E1",
+		"1e1.1",
+		"1e+1",
+		"1e-1",
+		"1e+-1",
+		"0x1p1",
+		"0x10p1",
+
+		// Examples from language specifcation (https://go.dev/ref/spec#Floating-point_literals)
+		"0.",
+		"72.40",
+		"072.40", // == 72.40
+		"2.71828",
+		"1.e+0",
+		"6.67428e-11",
+		"1E6",
+		".25",
+		".12345E+5",
+		"1_5.",        // == 15.0
+		"0.15e+0_2",   // == 15.0
+		"0x1p-2",      // == 0.25
+		"0x2.p10",     // == 2048.0
+		"0x1.Fp+0",    // == 1.9375
+		"0X.8p-0",     // == 0.5
+		"0X_1FFFP-16", // == 0.1249847412109375
+		"0x.p1",       // invalid: mantissa has no digits
+		"1p-2",        // invalid: p exponent requires hexadecimal mantissa
+		"0x1.5e-2",    // invalid: hexadecimal mantissa requires p exponent
+		"1_.5",        // invalid: _ must separate successive digits
+		"1._5",        // invalid: _ must separate successive digits
+		"1.5_e1",      // invalid: _ must separate successive digits
+		"1.5e_1",      // invalid: _ must separate successive digits
+		"1.5e1_",      // invalid: _ must separate successive digits
+	}
+
+	for _, num := range testCases {
+		t.Run(num, func(t *testing.T) {
+			expected, err := strconv.ParseFloat(num, 64)
+			if err == nil {
+				expectParse(t, num, func(p pfn) []Stmt {
+					return stmts(exprStmt(floatLit(expected, p(1, 1))))
+				})
+			} else {
+				expectParseError(t, num)
+			}
+		})
+	}
+}
+
+func TestParseNumberExpressions(t *testing.T) {
+	expectParse(t, `0x15e+2`, func(p pfn) []Stmt {
+		return stmts(
+			exprStmt(
+				binaryExpr(
+					intLit(0x15e, p(1, 1)),
+					intLit(2, p(1, 7)),
+					token.Add,
+					p(1, 6))))
+	})
+
+	expectParse(t, `0-_42`, func(p pfn) []Stmt {
+		return stmts(
+			exprStmt(
+				binaryExpr(
+					intLit(0, p(1, 1)),
+					ident("_42", p(1, 3)),
+					token.Sub,
+					p(1, 2))))
+	})
+}
+
 type pfn func(int, int) Pos          // position conversion function
 type expectedFn func(pos pfn) []Stmt // callback function to return expected results