Repository: goplus/gop
Pull request #2232 (2025-03-31 05:37:29)

scanner: scanNumber fix (IMAG, RAT, UNIT)


Patch:
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 8a439329..c004b706 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -54,8 +54,8 @@ type Scanner struct {
 	rdOffset   int  // reading offset (position after current character)
 	lineOffset int  // current line offset
 	nParen     int
+	unitVal    string
 	insertSemi bool // insert a semicolon before next newline
-	needUnit   bool
 
 	// public state - ok to modify
 	ErrorCount int // number of errors encountered
@@ -503,18 +503,19 @@ func (s *Scanner) scanNumber() (token.Token, string) {
 		s.error(s.offset, "hexadecimal mantissa requires a 'p' exponent")
 	}
 
-	// suffix 'i'
-	if s.ch == 'i' {
-		tok = token.IMAG
-		s.next()
-	} else if s.ch == 'r' {
-		tok = token.RAT
-		s.next()
-	} else if isLetter(s.ch) {
-		s.needUnit = true
+	if isLetter(s.ch) {
+		id := s.scanIdentifier()
+		switch id {
+		case "i":
+			tok = token.IMAG
+		case "r":
+			tok = token.RAT
+		default:
+			s.unitVal = id
+		}
 	}
 
-	lit := string(s.src[offs:s.offset])
+	lit := string(s.src[offs : s.offset-len(s.unitVal)])
 	if tok == token.INT && invalid >= 0 {
 		s.errorf(invalid, "invalid digit %q in %s", lit[invalid-offs], litname(prefix))
 	}
@@ -831,10 +832,11 @@ scanAgain:
 
 	// determine token value
 	insertSemi := false
-	if s.needUnit { // number with unit
+	if s.unitVal != "" { // number with unit
 		insertSemi = true
-		tok, lit = token.UNIT, s.scanIdentifier()
-		s.needUnit = false
+		pos -= token.Pos(len(s.unitVal))
+		tok, lit = token.UNIT, s.unitVal
+		s.unitVal = ""
 		goto done
 	}
 	switch ch := s.ch; {
diff --git a/tpl/scanner/scanner.go b/tpl/scanner/scanner.go
index 1d2a714e..32adb0e2 100644
--- a/tpl/scanner/scanner.go
+++ b/tpl/scanner/scanner.go
@@ -50,8 +50,8 @@ type Scanner struct {
 	offset     int  // character offset
 	rdOffset   int  // reading offset (position after current character)
 	lineOffset int  // current line offset
+	unitVal    string
 	insertSemi bool // insert a semicolon before next newline
-	needUnit   bool
 
 	// public state - ok to modify
 	ErrorCount int // number of errors encountered
@@ -93,6 +93,15 @@ func (s *Scanner) next() {
 	}
 }
 
+// peek returns the byte following the most recently read character without
+// advancing the scanner. If the scanner is at EOF, peek returns 0.
+func (s *Scanner) peek() byte {
+	if s.rdOffset < len(s.src) {
+		return s.src[s.rdOffset]
+	}
+	return 0
+}
+
 // A Mode value is a set of flags (or 0).
 // They control scanner behavior.
 type Mode uint
@@ -160,6 +169,10 @@ func (s *Scanner) error(offs int, msg string) {
 	s.ErrorCount++
 }
 
+func (s *Scanner) errorf(offs int, format string, args ...any) {
+	s.error(offs, fmt.Sprintf(format, args...))
+}
+
 var prefix = []byte("//line ")
 
 func (s *Scanner) interpretLineComment(text []byte) {
@@ -294,96 +307,191 @@ func digitVal(ch rune) int {
 	switch {
 	case '0' <= ch && ch <= '9':
 		return int(ch - '0')
-	case 'a' <= ch && ch <= 'f':
-		return int(ch - 'a' + 10)
-	case 'A' <= ch && ch <= 'F':
-		return int(ch - 'A' + 10)
+	case 'a' <= lower(ch) && lower(ch) <= 'f':
+		return int(lower(ch) - 'a' + 10)
 	}
 	return 16 // larger than any legal digit val
 }
 
-func (s *Scanner) scanMantissa(base int) {
-	for digitVal(s.ch) < base {
-		s.next()
+func lower(ch rune) rune     { return ('a' - 'A') | ch } // returns lower-case ch iff ch is ASCII letter
+func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' }
+func isHex(ch rune) bool     { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' }
+
+// digits accepts the sequence { digit | '_' }.
+// If base <= 10, digits accepts any decimal digit but records
+// the offset (relative to the source start) of a digit >= base
+// in *invalid, if *invalid < 0.
+// digits returns a bitset describing whether the sequence contained
+// digits (bit 0 is set), or separators '_' (bit 1 is set).
+func (s *Scanner) digits(base int, invalid *int) (digsep int) {
+	if base <= 10 {
+		max := rune('0' + base)
+		for isDecimal(s.ch) || s.ch == '_' {
+			ds := 1
+			if s.ch == '_' {
+				ds = 2
+			} else if s.ch >= max && *invalid < 0 {
+				*invalid = int(s.offset) // record invalid rune offset
+			}
+			digsep |= ds
+			s.next()
+		}
+	} else {
+		for isHex(s.ch) || s.ch == '_' {
+			ds := 1
+			if s.ch == '_' {
+				ds = 2
+			}
+			digsep |= ds
+			s.next()
+		}
 	}
+	return
 }
 
-func (s *Scanner) scanNumber(seenDecimalPoint bool) (token.Token, string) {
-	// digitVal(s.ch) < 10
+func (s *Scanner) scanNumber() (token.Token, string) {
 	offs := s.offset
-	tok := token.INT
+	tok := token.ILLEGAL
 
-	if seenDecimalPoint {
-		offs--
-		tok = token.FLOAT
-		s.scanMantissa(10)
-		goto exponent
-	}
+	base := 10        // number base
+	prefix := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b'
+	digsep := 0       // bit 0: digit present, bit 1: '_' present
+	invalid := -1     // index of invalid digit in literal, or < 0
 
-	if s.ch == '0' {
-		// int or float
-		offs := s.offset
-		s.next()
-		if s.ch == 'x' || s.ch == 'X' {
-			// hexadecimal int
+	// integer part
+	if s.ch != '.' {
+		tok = token.INT
+		if s.ch == '0' {
 			s.next()
-			s.scanMantissa(16)
-			if s.offset-offs <= 2 {
-				// only scanned "0x" or "0X"
-				s.error(offs, "illegal hexadecimal number")
-			}
-		} else {
-			// octal int or float
-			seenDecimalDigit := false
-			s.scanMantissa(8)
-			if s.ch == '8' || s.ch == '9' {
-				// illegal octal int or float
-				seenDecimalDigit = true
-				s.scanMantissa(10)
-			}
-			if s.ch == '.' || s.ch == 'e' || s.ch == 'E' || s.ch == 'i' {
-				goto fraction
-			}
-			// octal int
-			if seenDecimalDigit {
-				s.error(offs, "illegal octal number")
+			switch lower(s.ch) {
+			case 'x':
+				s.next()
+				base, prefix = 16, 'x'
+			case 'o':
+				s.next()
+				base, prefix = 8, 'o'
+			case 'b':
+				s.next()
+				base, prefix = 2, 'b'
+			default:
+				base, prefix = 8, '0'
+				digsep = 1 // leading 0
 			}
 		}
-		goto exit
+		digsep |= s.digits(base, &invalid)
 	}
 
-	// decimal int or float
-	s.scanMantissa(10)
-
-fraction:
+	// fractional part
 	if s.ch == '.' {
 		tok = token.FLOAT
+		if prefix == 'o' || prefix == 'b' {
+			s.error(s.offset, "invalid radix point in "+litname(prefix))
+		}
 		s.next()
-		s.scanMantissa(10)
+		digsep |= s.digits(base, &invalid)
 	}
 
-exponent:
-	if s.ch == 'e' || s.ch == 'E' {
-		tok = token.FLOAT
+	if digsep&1 == 0 {
+		s.error(s.offset, litname(prefix)+" has no digits")
+	}
+
+	// exponent
+	if e := lower(s.ch); e == 'e' || e == 'p' {
+		switch {
+		case e == 'e' && prefix != 0 && prefix != '0':
+			s.errorf(s.offset, "%q exponent requires decimal mantissa", s.ch)
+		case e == 'p' && prefix != 'x':
+			s.errorf(s.offset, "%q exponent requires hexadecimal mantissa", s.ch)
+		}
 		s.next()
-		if s.ch == '-' || s.ch == '+' {
+		tok = token.FLOAT
+		if s.ch == '+' || s.ch == '-' {
 			s.next()
 		}
-		s.scanMantissa(10)
+		ds := s.digits(10, nil)
+		digsep |= ds
+		if ds&1 == 0 {
+			s.error(s.offset, "exponent has no digits")
+		}
+	} else if prefix == 'x' && tok == token.FLOAT {
+		s.error(s.offset, "hexadecimal mantissa requires a 'p' exponent")
 	}
 
-	if s.ch == 'i' {
-		tok = token.IMAG
-		s.next()
-	} else if s.ch == 'r' {
-		tok = token.RAT
-		s.next()
-	} else if isLetter(s.ch) {
-		s.needUnit = true
+	if isLetter(s.ch) {
+		id := s.scanIdentifier()
+		switch id {
+		case "i":
+			tok = token.IMAG
+		case "r":
+			tok = token.RAT
+		default:
+			s.unitVal = id
+		}
 	}
 
-exit:
-	return tok, string(s.src[offs:s.offset])
+	lit := string(s.src[offs : s.offset-len(s.unitVal)])
+	if tok == token.INT && invalid >= 0 {
+		s.errorf(invalid, "invalid digit %q in %s", lit[invalid-offs], litname(prefix))
+	}
+	if digsep&2 != 0 {
+		if i := invalidSep(lit); i >= 0 {
+			s.error(offs+i, "'_' must separate successive digits")
+		}
+	}
+
+	return tok, lit
+}
+
+func litname(prefix rune) string {
+	switch prefix {
+	case 'x':
+		return "hexadecimal literal"
+	case 'o', '0':
+		return "octal literal"
+	case 'b':
+		return "binary literal"
+	}
+	return "decimal literal"
+}
+
+// invalidSep returns the index of the first invalid separator in x, or -1.
+func invalidSep(x string) int {
+	x1 := ' ' // prefix char, we only care if it's 'x'
+	d := '.'  // digit, one of '_', '0' (a digit), or '.' (anything else)
+	i := 0
+
+	// a prefix counts as a digit
+	if len(x) >= 2 && x[0] == '0' {
+		x1 = lower(rune(x[1]))
+		if x1 == 'x' || x1 == 'o' || x1 == 'b' {
+			d = '0'
+			i = 2
+		}
+	}
+
+	// mantissa and exponent
+	for ; i < len(x); i++ {
+		p := d // previous digit
+		d = rune(x[i])
+		switch {
+		case d == '_':
+			if p != '0' {
+				return i
+			}
+		case isDecimal(d) || x1 == 'x' && isHex(d):
+			d = '0'
+		default:
+			if p == '_' {
+				return i - 1
+			}
+			d = '.'
+		}
+	}
+	if d == '_' {
+		return len(x) - 1
+	}
+
+	return -1
 }
 
 // scanEscape parses an escape sequence where rune is the accepted
@@ -640,20 +748,21 @@ scanAgain:
 
 	// determine token value
 	insertSemi := false
-	if s.needUnit { // number with unit
+	if s.unitVal != "" { // number with unit
 		insertSemi = true
-		t.Tok, t.Lit = token.UNIT, s.scanIdentifier()
-		s.needUnit = false
+		t.Pos -= token.Pos(len(s.unitVal))
+		t.Tok, t.Lit = token.UNIT, s.unitVal
+		s.unitVal = ""
 		goto done
 	}
 	switch ch := s.ch; {
 	case isLetter(ch):
+		insertSemi = true
 		t.Lit = s.scanIdentifier()
 		t.Tok = token.IDENT
+	case isDecimal(ch) || ch == '.' && isDecimal(rune(s.peek())):
 		insertSemi = true
-	case '0' <= ch && ch <= '9':
-		t.Tok, t.Lit = s.scanNumber(false)
-		insertSemi = true
+		t.Tok, t.Lit = s.scanNumber()
 	default:
 		s.next() // always make progress
 		switch ch {
@@ -686,15 +795,11 @@ scanAgain:
 		case ':':
 			t.Tok = s.switch2(token.COLON, token.DEFINE)
 		case '.':
-			if '0' <= s.ch && s.ch <= '9' {
-				insertSemi = true
-				t.Tok, t.Lit = s.scanNumber(true)
-			} else if s.ch == '.' {
+			// fractions starting with a '.' are handled by outer switch
+			if s.ch == '.' && s.peek() == '.' {
 				s.next()
-				if s.ch == '.' {
-					s.next()
-					t.Tok = token.ELLIPSIS
-				}
+				s.next() // consume last '.'
+				t.Tok = token.ELLIPSIS
 			} else {
 				t.Tok = token.PERIOD
 			}


Test patch:
diff --git a/tpl/scanner/_testdata/num/go.expect b/tpl/scanner/_testdata/num/go.expect
new file mode 100644
index 00000000..6877f962
--- /dev/null
+++ b/tpl/scanner/_testdata/num/go.expect
@@ -0,0 +1,15 @@
+1 IDENT echo
+6 INT 1_002
+11 ; 
+
+12 IDENT echo
+17 INT 2
+18 + 
+19 IMAG 3i
+21 ; 
+
+22 IDENT echo
+27 FLOAT 2.
+29 IDENT string
+35 ; 
+
diff --git a/tpl/scanner/_testdata/num/gop.expect b/tpl/scanner/_testdata/num/gop.expect
new file mode 100644
index 00000000..e97b7983
--- /dev/null
+++ b/tpl/scanner/_testdata/num/gop.expect
@@ -0,0 +1,15 @@
+1 IDENT echo
+6 INT 1_002
+11 ; 
+
+12 IDENT echo
+17 INT 2
+18 + 
+19 IMAG 3i
+21 ; 
+
+22 IDENT echo
+27 FLOAT 2.
+29 UNIT string
+35 ; 
+
diff --git a/tpl/scanner/_testdata/num/in.gop b/tpl/scanner/_testdata/num/in.gop
new file mode 100644
index 00000000..73da6cfa
--- /dev/null
+++ b/tpl/scanner/_testdata/num/in.gop
@@ -0,0 +1,3 @@
+echo 1_002
+echo 2+3i
+echo 2.string
diff --git a/tpl/scanner/_testdata/num/tpl.expect b/tpl/scanner/_testdata/num/tpl.expect
new file mode 100644
index 00000000..e97b7983
--- /dev/null
+++ b/tpl/scanner/_testdata/num/tpl.expect
@@ -0,0 +1,15 @@
+1 IDENT echo
+6 INT 1_002
+11 ; 
+
+12 IDENT echo
+17 INT 2
+18 + 
+19 IMAG 3i
+21 ; 
+
+22 IDENT echo
+27 FLOAT 2.
+29 UNIT string
+35 ; 
+
diff --git a/tpl/scanner/_testdata/unit/go.expect b/tpl/scanner/_testdata/unit/go.expect
index 833b113d..713110bb 100644
--- a/tpl/scanner/_testdata/unit/go.expect
+++ b/tpl/scanner/_testdata/unit/go.expect
@@ -2,32 +2,57 @@
 8 STRING "time"
 14 ; 
 
-16 IDENT echo
+16 IDENT x
+18 := 
 21 STRING "Let's wait for a second"
 46 ; 
 
-47 IDENT time
-51 . 
-52 IDENT sleep
-58 INT 1
-59 IDENT s
-60 ; 
-
-61 IDENT time
-65 . 
-66 IDENT sleep
-72 FLOAT 1.
-74 IDENT s
-75 ; 
-
-76 IDENT time
-80 . 
-81 IDENT sleep
-87 FLOAT 1.2
-90 IDENT s
-91 ; 
-
-92 IDENT echo
-97 STRING "Hello, world"
-111 ; 
+47 IDENT echo
+52 IDENT x
+53 ; 
 
+55 IDENT wait
+60 INT 0x10
+64 IDENT s
+65 ; 
+
+66 IDENT wait
+71 FLOAT .1
+73 IDENT s
+74 ; 
+
+75 IDENT wait
+80 FLOAT 1.
+82 IDENT s
+83 ; 
+
+84 IDENT wait
+89 FLOAT 1.e7
+93 IDENT s
+94 ; 
+
+95 IDENT wait
+100 FLOAT 1.2
+103 IDENT s
+104 ; 
+
+106 IDENT time
+110 . 
+111 IDENT sleep
+117 IMAG 100i
+121 IDENT m
+122 ; 
+
+124 IDENT spend
+130 FLOAT 10.0
+134 IDENT rmb
+137 ; 
+
+138 IDENT spend
+144 FLOAT 10.
+147 IDENT rmb
+150 ; 
+
+152 IDENT echo
+157 STRING "Hello, world"
+171 ... 
diff --git a/tpl/scanner/_testdata/unit/gop.expect b/tpl/scanner/_testdata/unit/gop.expect
index 17ac9eb1..1368f765 100644
--- a/tpl/scanner/_testdata/unit/gop.expect
+++ b/tpl/scanner/_testdata/unit/gop.expect
@@ -2,32 +2,59 @@
 8 STRING "time"
 14 ; 
 
-16 IDENT echo
+16 IDENT x
+18 := 
 21 STRING "Let's wait for a second"
 46 ; 
 
-47 IDENT time
-51 . 
-52 IDENT sleep
-58 INT 1
-59 UNIT s
-60 ; 
-
-61 IDENT time
-65 . 
-66 IDENT sleep
-72 FLOAT 1.
-74 UNIT s
-75 ; 
-
-76 IDENT time
-80 . 
-81 IDENT sleep
-87 FLOAT 1.2
-90 UNIT s
-91 ; 
-
-92 IDENT echo
-97 STRING "Hello, world"
-111 ; 
+47 IDENT echo
+52 IDENT x
+53 ; 
+
+55 IDENT wait
+60 INT 0x10
+64 UNIT s
+65 ; 
+
+66 IDENT wait
+71 FLOAT .1
+73 UNIT s
+74 ; 
+
+75 IDENT wait
+80 FLOAT 1.
+82 UNIT s
+83 ; 
+
+84 IDENT wait
+89 FLOAT 1.e7
+93 UNIT s
+94 ; 
+
+95 IDENT wait
+100 FLOAT 1.2
+103 UNIT s
+104 ; 
+
+106 IDENT time
+110 . 
+111 IDENT sleep
+117 INT 100
+120 UNIT im
+122 ; 
+
+124 IDENT spend
+130 FLOAT 10.0
+134 UNIT rmb
+137 ; 
+
+138 IDENT spend
+144 FLOAT 10.
+147 UNIT rmb
+150 ; 
+
+152 IDENT echo
+157 STRING "Hello, world"
+171 ... 
+174 ; 
 
diff --git a/tpl/scanner/_testdata/unit/in.gop b/tpl/scanner/_testdata/unit/in.gop
index 23fa95de..38e5568e 100644
--- a/tpl/scanner/_testdata/unit/in.gop
+++ b/tpl/scanner/_testdata/unit/in.gop
@@ -1,7 +1,17 @@
 import "time"
 
-echo "Let's wait for a second"
-time.sleep 1s
-time.sleep 1.s
-time.sleep 1.2s
-echo "Hello, world"
+x := "Let's wait for a second"
+echo x
+
+wait 0x10s
+wait .1s
+wait 1.s
+wait 1.e7s
+wait 1.2s
+
+time.sleep 100im
+
+spend 10.0rmb
+spend 10.rmb
+
+echo "Hello, world"...
diff --git a/tpl/scanner/_testdata/unit/tpl.expect b/tpl/scanner/_testdata/unit/tpl.expect
index 70503895..3688b4d5 100644
--- a/tpl/scanner/_testdata/unit/tpl.expect
+++ b/tpl/scanner/_testdata/unit/tpl.expect
@@ -2,32 +2,57 @@
 8 STRING "time"
 14 ; 
 
-16 IDENT echo
+16 IDENT x
+18 := 
 21 STRING "Let's wait for a second"
 46 ; 
 
-47 IDENT time
-51 . 
-52 IDENT sleep
-58 INT 1
-59 UNIT s
-60 ; 
-
-61 IDENT time
-65 . 
-66 IDENT sleep
-72 FLOAT 1.
-74 UNIT s
-75 ; 
-
-76 IDENT time
-80 . 
-81 IDENT sleep
-87 FLOAT 1.2
-90 UNIT s
-91 ; 
-
-92 IDENT echo
-97 STRING "Hello, world"
-111 ; 
+47 IDENT echo
+52 IDENT x
+53 ; 
 
+55 IDENT wait
+60 INT 0x10
+64 UNIT s
+65 ; 
+
+66 IDENT wait
+71 FLOAT .1
+73 UNIT s
+74 ; 
+
+75 IDENT wait
+80 FLOAT 1.
+82 UNIT s
+83 ; 
+
+84 IDENT wait
+89 FLOAT 1.e7
+93 UNIT s
+94 ; 
+
+95 IDENT wait
+100 FLOAT 1.2
+103 UNIT s
+104 ; 
+
+106 IDENT time
+110 . 
+111 IDENT sleep
+117 INT 100
+120 UNIT im
+122 ; 
+
+124 IDENT spend
+130 FLOAT 10.0
+134 UNIT rmb
+137 ; 
+
+138 IDENT spend
+144 FLOAT 10.
+147 UNIT rmb
+150 ; 
+
+152 IDENT echo
+157 STRING "Hello, world"
+171 ...