Repository: google/pytype

Issue:
pytype can't parse hex literals
The following code causes the error `ParseError: syntax error, unexpected NAME, expecting ']'`:

```py
from typing import Literal

a: Literal[0x00ff]
```
Add lexing for base 2, 8, and 16 literals
Alters lexer.lex and adds a test for the new numeric types, Closes #494 


Discussion:
I think we need to expand our definition of NUMBER here: https://github.com/google/pytype/blob/ce086c5c6ef55364dac96a3c1e97a4a21b1aefeb/pytype/pyi/lexer.lex#L117.
Is [py3_test_grammar.py](https://github.com/python/cpython/blob/master/Lib/lib2to3/tests/data/py3_test_grammar.py) part of your test suite? It has hex numbers.


Resolving patch:
diff --git a/pytype/tools/traces/source.py b/pytype/tools/traces/source.py
--- a/pytype/tools/traces/source.py
+++ b/pytype/tools/traces/source.py
@@ -4,6 +4,9 @@
 from __future__ import print_function
 import collections
 
+from pytype import compat
+
+
 Location = collections.namedtuple("Location", ("line", "column"))
 
 
@@ -50,7 +53,9 @@ def _init_byte_offsets(self):
     offset = 0
     for line in self._lines:
       self._offsets.append(offset)
-      offset += len(line) + 1  # account for the \n
+      # convert line to bytes
+      bytes_ = compat.bytestring(line)
+      offset += len(bytes_) + 1  # account for the \n
 
   def get_offset(self, location):
     """Gets the utf-8 byte offset of a source.Location from start of source."""


Test patch:
diff --git a/pytype/pyi/lexer_test.py b/pytype/pyi/lexer_test.py
--- a/pytype/pyi/lexer_test.py
+++ b/pytype/pyi/lexer_test.py
@@ -6,6 +6,9 @@
 
 import unittest
 
+# We use '\' to make test code more readable:
+# pylint: disable=g-backslash-continuation
+
 # Map from token code to name.
 TOKEN_NAMES = {code: name for name, code in parser_ext.TOKENS.items()}
 
@@ -130,6 +133,23 @@ def test_number(self):
     self.check([0.5], "+.5")
     self.check([-0.5], "-.5")
 
+  def test_number_base(self):
+    self.check([0], "0b0")
+    self.check([1], "0b1")
+    self.check([42], "0b101010")
+    self.check([-8], "-0b1000")
+
+    self.check([1], "0o1")
+    self.check([8], "0o10")
+    self.check([42], "0o52")
+    self.check([-7], "-0o7")
+
+    self.check([1], "0x1")
+    self.check([240], "0xF0")
+    self.check([-240], "-0xF0")
+    self.check([15], "0x0f")
+    self.check([-15], "-0x0f")
+
   def test_line_numbers(self):
     self.check([("NAME", "a", 1), ("NAME", "b", 2)], "a\nb")
 
diff --git a/pytype/tools/traces/source_test.py b/pytype/tools/traces/source_test.py
--- a/pytype/tools/traces/source_test.py
+++ b/pytype/tools/traces/source_test.py
@@ -1,3 +1,4 @@
+# coding=utf-8
 # Lint as: python2, python3
 """Tests for traces.source."""
 
@@ -52,6 +53,23 @@ def test_get_offset(self):
     src = source.Code("line1\nline2", [], _FakeTrace, "")
     self.assertEqual(src.get_offset(source.Location(2, 3)), 9)
 
+  def test_get_offset_multibyte(self):
+    # With single-byte characters
+    src = source.Code("""\
+      # coding=utf-8
+      line1 # a
+      line2
+    """, [], _FakeTrace, "")
+    self.assertEqual(src.get_offset(source.Location(3, 3)), 40)
+
+    # With a multibyte character the byte offset should change
+    src = source.Code("""\
+      # coding=utf-8
+      line1 # ツ
+      line2
+    """, [], _FakeTrace, "")
+    self.assertEqual(src.get_offset(source.Location(3, 3)), 42)
+
   def test_line(self):
     src = source.Code("line1\nline2", [], _FakeTrace, "")
     self.assertEqual(src.line(2), "line2")