Repository: PyCQA/pyflakes
Pull request #413 (2019-01-19 19:53:34)

Use file_tokens to better match flake8 api
noticed while integrating with `flake8`.

`flake8` passes `tokens` per line and `file_tokens` per file.  since pyflakes is a per-file checker this is more appropriate


Patch:
diff --git a/pyflakes/api.py b/pyflakes/api.py
index 72d6b6c..6226e5d 100644
--- a/pyflakes/api.py
+++ b/pyflakes/api.py
@@ -70,8 +70,8 @@ def check(codeString, filename, reporter=None):
         reporter.unexpectedError(filename, 'problem decoding source')
         return 1
     # Okay, it's syntactically valid.  Now check it.
-    tokens = checker.make_tokens(codeString)
-    w = checker.Checker(tree, tokens=tokens, filename=filename)
+    file_tokens = checker.make_tokens(codeString)
+    w = checker.Checker(tree, file_tokens=file_tokens, filename=filename)
     w.messages.sort(key=lambda m: m.lineno)
     for warning in w.messages:
         reporter.flake(warning)
diff --git a/pyflakes/checker.py b/pyflakes/checker.py
index 7b68b21..b8d7be7 100644
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -632,11 +632,11 @@ class Checker(object):
         builtIns.update(_customBuiltIns.split(','))
     del _customBuiltIns
 
-    # TODO: tokens= is required to perform checks on type comments, eventually
-    #       make this a required positional argument.  For now it is defaulted
-    #       to `()` for api compatibility.
+    # TODO: file_tokens= is required to perform checks on type comments,
+    #       eventually make this a required positional argument.  For now it
+    #       is defaulted to `()` for api compatibility.
     def __init__(self, tree, filename='(none)', builtins=None,
-                 withDoctest='PYFLAKES_DOCTEST' in os.environ, tokens=()):
+                 withDoctest='PYFLAKES_DOCTEST' in os.environ, file_tokens=()):
         self._nodeHandlers = {}
         self._deferredFunctions = []
         self._deferredAssignments = []
@@ -652,7 +652,7 @@ class Checker(object):
             raise RuntimeError('No scope implemented for the node %r' % tree)
         self.exceptHandlers = [()]
         self.root = tree
-        self._type_comments = _collect_type_comments(tree, tokens)
+        self._type_comments = _collect_type_comments(tree, file_tokens)
         for builtin in self.builtIns:
             self.addBinding(None, Builtin(builtin))
         self.handleChildren(tree)
@@ -1037,7 +1037,10 @@ class Checker(object):
             comment = comment.split(':', 1)[1].strip()
             func_match = TYPE_FUNC_RE.match(comment)
             if func_match:
-                parts = (func_match.group(1), func_match.group(2).strip())
+                parts = (
+                    func_match.group(1).replace('*', ''),
+                    func_match.group(2).strip(),
+                )
             else:
                 parts = (comment,)
 


Test patch:
diff --git a/pyflakes/test/harness.py b/pyflakes/test/harness.py
index d375ea3..b20ac79 100644
--- a/pyflakes/test/harness.py
+++ b/pyflakes/test/harness.py
@@ -16,11 +16,13 @@ class TestCase(unittest.TestCase):
 
     def flakes(self, input, *expectedOutputs, **kw):
         tree = ast.parse(textwrap.dedent(input))
-        tokens = checker.make_tokens(textwrap.dedent(input))
+        file_tokens = checker.make_tokens(textwrap.dedent(input))
         if kw.get('is_segment'):
             tree = tree.body[0]
             kw.pop('is_segment')
-        w = checker.Checker(tree, tokens=tokens, withDoctest=self.withDoctest, **kw)
+        w = checker.Checker(
+            tree, file_tokens=file_tokens, withDoctest=self.withDoctest, **kw
+        )
         outputs = [type(o) for o in w.messages]
         expectedOutputs = list(expectedOutputs)
         outputs.sort(key=lambda t: t.__name__)
diff --git a/pyflakes/test/test_type_annotations.py b/pyflakes/test/test_type_annotations.py
index 1ed676b..40397e8 100644
--- a/pyflakes/test/test_type_annotations.py
+++ b/pyflakes/test/test_type_annotations.py
@@ -231,6 +231,14 @@ class TestTypeAnnotations(TestCase):
             return a + b
         """)
 
+    def test_typeCommentsStarArgs(self):
+        self.flakes("""
+        from mod import A, B, C, D
+        def f(a, *b, **c):
+            # type: (A, *B, **C) -> D
+            return a + b
+        """)
+
     def test_typeCommentsFullSignatureWithDocstring(self):
         self.flakes('''
         from mod import A, B, C, D
diff --git a/pyflakes/test/test_undefined_names.py b/pyflakes/test/test_undefined_names.py
index 25e28dd..c952cbb 100644
--- a/pyflakes/test/test_undefined_names.py
+++ b/pyflakes/test/test_undefined_names.py
@@ -848,7 +848,7 @@ class NameTests(TestCase):
         raised.
         """
         tree = ast.parse("x = 10")
-        tokens = checker.make_tokens("x = 10")
+        file_tokens = checker.make_tokens("x = 10")
         # Make it into something unrecognizable.
         tree.body[0].targets[0].ctx = object()
-        self.assertRaises(RuntimeError, checker.Checker, tree, tokens=tokens)
+        self.assertRaises(RuntimeError, checker.Checker, tree, file_tokens=file_tokens)