Repository: python/mypy

Issue:
Crash in lexer when source contains funny quotes
I had copied-pasted some code from a website that used Word-style matching quotes. I got a crash in lex.py:

```
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/runpy.py", line 184, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/local/lib/python3.5/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/guido/src/mypy/mypy/__main__.py", line 5, in <module>
    main(None)
  File "/Users/guido/src/mypy/mypy/main.py", line 53, in main
    type_check_only(sources, bin_dir, options)
  File "/Users/guido/src/mypy/mypy/main.py", line 96, in type_check_only
    python_path=options.python_path)
  File "/Users/guido/src/mypy/mypy/build.py", line 203, in build
    result = manager.process(initial_states)
  File "/Users/guido/src/mypy/mypy/build.py", line 386, in process
    next.process()
  File "/Users/guido/src/mypy/mypy/build.py", line 710, in process
    tree = self.parse(self.program_text, self.path)
  File "/Users/guido/src/mypy/mypy/build.py", line 807, in parse
    custom_typing_module=self.manager.custom_typing_module)
  File "/Users/guido/src/mypy/mypy/parse.py", line 83, in parse
    tree = parser.parse(source)
  File "/Users/guido/src/mypy/mypy/parse.py", line 130, in parse
    is_stub_file=self.is_stub_file)
  File "/Users/guido/src/mypy/mypy/lex.py", line 170, in lex
    l.lex(string, first_line)
  File "/Users/guido/src/mypy/mypy/lex.py", line 377, in lex
    map[c]()
IndexError: list index out of range
```



Discussion:

Resolving patch:
diff --git a/mypy/lex.py b/mypy/lex.py
--- a/mypy/lex.py
+++ b/mypy/lex.py
@@ -284,7 +284,7 @@ class Lexer:
 
     # Table from byte character value to lexer method. E.g. entry at ord('0')
     # contains the method lex_number().
-    map = None  # type: List[Callable[[], None]]
+    map = None  # type: Dict[str, Callable[[], None]]
 
     # Indent levels of currently open blocks, in spaces.
     indents = None  # type: List[int]
@@ -300,7 +300,7 @@ class Lexer:
 
     def __init__(self, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
                  is_stub_file: bool = False) -> None:
-        self.map = [self.unknown_character] * 256
+        self.map = {}
         self.tok = []
         self.indents = [0]
         self.open_brackets = []
@@ -325,7 +325,7 @@ def __init__(self, pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
                             (')]}', self.lex_close_bracket),
                             ('-+*/<>%&|^~=!,@' + extra_misc, self.lex_misc)]:
             for c in seq:
-                self.map[ord(c)] = method
+                self.map[c] = method
         if pyversion[0] == 2:
             self.keywords = keywords_common | keywords2
             # Decimal/hex/octal/binary literal or integer complex literal
@@ -364,17 +364,18 @@ def lex(self, text: Union[str, bytes], first_line: int) -> None:
         # an error.
         self.lex_indent()
 
-        # Make a local copy of map as a simple optimization.
+        # Use some local variables as a simple optimization.
         map = self.map
+        default = self.unknown_character
 
         # Lex the file. Repeatedly call the lexer method for the current char.
         while self.i < len(text):
             # Get the character code of the next character to lex.
-            c = ord(text[self.i])
+            c = text[self.i]
             # Dispatch to the relevant lexer method. This will consume some
             # characters in the text, add a token to self.tok and increment
             # self.i.
-            map[c]()
+            map.get(c, default)()
 
         # Append a break if there is no statement/block terminator at the end
         # of input.


Test patch:
diff --git a/mypy/test/data/parse-errors.test b/mypy/test/data/parse-errors.test
--- a/mypy/test/data/parse-errors.test
+++ b/mypy/test/data/parse-errors.test
@@ -472,3 +472,8 @@ file:1: error: Tuples in argument lists only supported in Python 2 mode
 `1 + 2`
 [out]
 file:1: error: Unrecognized character `
+
+[case testSmartQuotes]
+foo = ‘bar’
+[out]
+file:1: error: Unrecognized character
diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py
--- a/mypy/test/testparse.py
+++ b/mypy/test/testparse.py
@@ -64,8 +64,8 @@ def cases(self):
 
 def test_parse_error(testcase):
     try:
-        # Compile temporary file.
-        parse(bytes('\n'.join(testcase.input), 'ascii'), INPUT_FILE_NAME)
+        # Compile temporary file. The test file contains non-ASCII characters.
+        parse(bytes('\n'.join(testcase.input), 'utf-8'), INPUT_FILE_NAME)
         raise AssertionFailure('No errors reported')
     except CompileError as e:
         # Verify that there was a compile error and that the error messages