Repository: jhump/protoreflect
Pull request #436 (2021-10-30 21:11:25)

protoparse: int and float scanning match protoc
I was looking into writing a formal specification for the protobuf language, and in so doing was reviewing the tokenizer here in the `protoparse` package as well as `protoc`'s tokenizer in [`google/protobuf/io/tokenizer.cc`](https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/io/tokenizer.cc).

During this review, I noticed some unintentional differences in behavior between `protoparse` and `protoc`. This change reconciles that difference, making `protoparse` more perfectly match `protoc`.


Patch:
diff --git a/desc/protoparse/lexer.go b/desc/protoparse/lexer.go
index 7c6d6a2..87bc803 100644
--- a/desc/protoparse/lexer.go
+++ b/desc/protoparse/lexer.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -237,7 +238,7 @@ func (l *protoLex) Lex(lval *protoSymType) int {
 			if cn >= '0' && cn <= '9' {
 				l.adjustPos(cn)
 				token := l.readNumber(c, cn)
-				f, err := strconv.ParseFloat(token, 64)
+				f, err := parseFloat(token)
 				if err != nil {
 					l.setError(lval, numError(err, "float", token))
 					return _ERROR
@@ -278,7 +279,7 @@ func (l *protoLex) Lex(lval *protoSymType) int {
 			}
 			if strings.Contains(token, ".") || strings.Contains(token, "e") || strings.Contains(token, "E") {
 				// floating point!
-				f, err := strconv.ParseFloat(token, 64)
+				f, err := parseFloat(token)
 				if err != nil {
 					l.setError(lval, numError(err, "float", token))
 					return _ERROR
@@ -287,14 +288,21 @@ func (l *protoLex) Lex(lval *protoSymType) int {
 				return _FLOAT_LIT
 			}
 			// integer! (decimal or octal)
-			ui, err := strconv.ParseUint(token, 0, 64)
+			base := 10
+			if token[0] == '0' && len(token) > 1 && token[1] >= '0' && token[1] <= '7' {
+				base = 8
+			}
+			ui, err := strconv.ParseUint(token, base, 64)
 			if err != nil {
 				kind := "integer"
+				if base == 8 {
+					kind = "octal integer"
+				}
 				if numErr, ok := err.(*strconv.NumError); ok && numErr.Err == strconv.ErrRange {
 					// if it's too big to be an int, parse it as a float
 					var f float64
 					kind = "float"
-					f, err = strconv.ParseFloat(token, 64)
+					f, err = parseFloat(token)
 					if err == nil {
 						l.setFloat(lval, f)
 						return _FLOAT_LIT
@@ -361,6 +369,19 @@ func (l *protoLex) Lex(lval *protoSymType) int {
 	}
 }
 
+func parseFloat(token string) (float64, error) {
+	f, err := strconv.ParseFloat(token, 64)
+	if err == nil {
+		return f, nil
+	}
+	if numErr, ok := err.(*strconv.NumError); ok && numErr.Err == strconv.ErrRange && math.IsInf(f, 1) {
+		// protoc doesn't complain about float overflow and instead just uses "infinity"
+		// so we mirror that behavior by just returning infinity and ignoring the error
+		return f, nil
+	}
+	return f, err
+}
+
 func (l *protoLex) posRange() ast.PosRange {
 	return ast.PosRange{
 		Start: SourcePos{


Test patch:
diff --git a/desc/protoparse/lexer_test.go b/desc/protoparse/lexer_test.go
index e48f37b..ca39a31 100644
--- a/desc/protoparse/lexer_test.go
+++ b/desc/protoparse/lexer_test.go
@@ -2,6 +2,7 @@ package protoparse
 
 import (
 	"io"
+	"math"
 	"strings"
 	"testing"
 
@@ -56,7 +57,7 @@ foo
 	1.543 g12 /* trailing line comment */
 	000.000
 	0.1234 .5678 .
-	12e12
+	12e12 1.2345e123412341234
 
 	Random_identifier_with_numbers_0123456789_and_letters...
 	// this is a trailing comment
@@ -151,6 +152,7 @@ foo
 		{t: _FLOAT_LIT, line: 46, col: 16, span: 5, v: 0.5678},
 		{t: '.', line: 46, col: 22, span: 1, v: nil},
 		{t: _FLOAT_LIT, line: 47, col: 9, span: 5, v: 12e12},
+		{t: _FLOAT_LIT, line: 47, col: 15, span: 19, v: math.Inf(1)},
 		{t: _NAME, line: 49, col: 9, span: 53, v: "Random_identifier_with_numbers_0123456789_and_letters"},
 		{t: '.', line: 49, col: 62, span: 1, v: nil},
 		{t: '.', line: 49, col: 63, span: 1, v: nil},
@@ -242,6 +244,9 @@ func TestLexerErrors(t *testing.T) {
 		{str: `0.987e34e-20`, errMsg: "invalid syntax"},
 		{str: `0.987e-345e20`, errMsg: "invalid syntax"},
 		{str: `.987to123`, errMsg: "invalid syntax"},
+		{str: `0b0111`, errMsg: "invalid syntax"},
+		{str: `0o765432`, errMsg: "invalid syntax"},
+		{str: `1_000_000`, errMsg: "invalid syntax"},
 		{str: `/* foobar`, errMsg: "unexpected EOF"},
 	}
 	for i, tc := range testCases {