Repository: PyCQA/pyflakes

Issue:
Support Python 3.7
*I'm moving https://github.com/PyCQA/pyflakes/pull/90#issuecomment-301317699 here since we have GitHub issue support now.*

It looks like the [CPython 3.7 failure](https://travis-ci.org/PyCQA/pyflakes/jobs/231571260) is related to a change in the AST. The docstring is no longer in `node.body`.

https://github.com/python/cpython/pull/46

We can mostly replace our `getDocstring(node.body[0])` call with `node.docstring`. But I'm not sure how to get the line number of the docstring anymore. `node.lineno` is close, but not quite the line number of the docstring.

https://github.com/PyCQA/pyflakes/blob/074c21da3174bc88b2ec02fea767a436e2db643d/pyflakes/checker.py#L873

```diff
diff --git a/pyflakes/checker.py b/pyflakes/checker.py
index 382574e..2c19533 100644
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -870,7 +870,11 @@ class Checker(object):

     def handleDoctests(self, node):
         try:
-            (docstring, node_lineno) = self.getDocstring(node.body[0])
+            if hasattr(node, 'docstring'):
+                docstring = node.docstring
+                node_lineno = ???  # TODO
+            else:
+                (docstring, node_lineno) = self.getDocstring(node.body[0])
             examples = docstring and self._getDoctestExamples(docstring)
         except (ValueError, IndexError):
             # e.g. line 6 of the docstring for <string> has inconsistent
```

I've reported this to the [CPython issue tracker](http://bugs.python.org/issue30497).

Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 1ea3e20..2d614cd 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,9 +11,6 @@ python:
   - pypy-5.3
   - pypy3
   - pypy3.3-5.2-alpha1
-matrix:
-  allow_failures:
-    - python: nightly
 install:
   - pip install flake8==2.1.0 pep8==1.5.6
   - python setup.py install
diff --git a/README.rst b/README.rst
index e84d334..aeb15f9 100644
--- a/README.rst
+++ b/README.rst
@@ -72,7 +72,7 @@ rebase your commits for you.
 
 All changes should include tests and pass flake8_.
 
-.. image:: https://api.travis-ci.org/PyCQA/pyflakes.svg
+.. image:: https://api.travis-ci.org/PyCQA/pyflakes.svg?branch=master
    :target: https://travis-ci.org/PyCQA/pyflakes
    :alt: Build status
 
diff --git a/pyflakes/api.py b/pyflakes/api.py
index a535bff..49ee38d 100644
--- a/pyflakes/api.py
+++ b/pyflakes/api.py
@@ -5,6 +5,7 @@ from __future__ import with_statement
 
 import sys
 import os
+import re
 import _ast
 
 from pyflakes import checker, __version__
@@ -13,6 +14,9 @@ from pyflakes import reporter as modReporter
 __all__ = ['check', 'checkPath', 'checkRecursive', 'iterSourceCode', 'main']
 
 
+PYTHON_SHEBANG_REGEX = re.compile(br'^#!.*\bpython[23w]?\b\s*$')
+
+
 def check(codeString, filename, reporter=None):
     """
     Check the Python source given by C{codeString} for flakes.
@@ -108,6 +112,25 @@ def checkPath(filename, reporter=None):
     return check(codestr, filename, reporter)
 
 
+def isPythonFile(filename):
+    """Return True if filename points to a Python file."""
+    if filename.endswith('.py'):
+        return True
+
+    max_bytes = 128
+
+    try:
+        with open(filename, 'rb') as f:
+            text = f.read(max_bytes)
+            if not text:
+                return False
+    except IOError:
+        return False
+
+    first_line = text.splitlines()[0]
+    return PYTHON_SHEBANG_REGEX.match(first_line)
+
+
 def iterSourceCode(paths):
     """
     Iterate over all Python source files in C{paths}.
@@ -120,8 +143,9 @@ def iterSourceCode(paths):
         if os.path.isdir(path):
             for dirpath, dirnames, filenames in os.walk(path):
                 for filename in filenames:
-                    if filename.endswith('.py'):
-                        yield os.path.join(dirpath, filename)
+                    full_path = os.path.join(dirpath, filename)
+                    if isPythonFile(full_path):
+                        yield full_path
         else:
             yield path
 
diff --git a/pyflakes/checker.py b/pyflakes/checker.py
index 382574e..75abdc0 100644
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -870,7 +870,19 @@ class Checker(object):
 
     def handleDoctests(self, node):
         try:
-            (docstring, node_lineno) = self.getDocstring(node.body[0])
+            if hasattr(node, 'docstring'):
+                docstring = node.docstring
+
+                # This is just a reasonable guess. In Python 3.7, docstrings no
+                # longer have line numbers associated with them. This will be
+                # incorrect if there are empty lines between the beginning
+                # of the function and the docstring.
+                node_lineno = node.lineno
+                if hasattr(node, 'args'):
+                    node_lineno = max([node_lineno] +
+                                      [arg.lineno for arg in node.args.args])
+            else:
+                (docstring, node_lineno) = self.getDocstring(node.body[0])
             examples = docstring and self._getDoctestExamples(docstring)
         except (ValueError, IndexError):
             # e.g. line 6 of the docstring for <string> has inconsistent
diff --git a/setup.cfg b/setup.cfg
index 5e40900..2a9acf1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,2 @@
-[wheel]
+[bdist_wheel]
 universal = 1


Test patch:
diff --git a/pyflakes/test/test_api.py b/pyflakes/test/test_api.py
index 51b0027..3f54ca4 100644
--- a/pyflakes/test/test_api.py
+++ b/pyflakes/test/test_api.py
@@ -187,6 +187,36 @@ class TestIterSourceCode(TestCase):
             sorted(iterSourceCode([self.tempdir])),
             sorted([apath, bpath, cpath]))
 
+    def test_shebang(self):
+        """
+        Find Python files that don't end with `.py`, but contain a Python
+        shebang.
+        """
+        python = os.path.join(self.tempdir, 'a')
+        with open(python, 'w') as fd:
+            fd.write('#!/usr/bin/env python\n')
+
+        self.makeEmptyFile('b')
+
+        with open(os.path.join(self.tempdir, 'c'), 'w') as fd:
+            fd.write('hello\nworld\n')
+
+        python2 = os.path.join(self.tempdir, 'd')
+        with open(python2, 'w') as fd:
+            fd.write('#!/usr/bin/env python2\n')
+
+        python3 = os.path.join(self.tempdir, 'e')
+        with open(python3, 'w') as fd:
+            fd.write('#!/usr/bin/env python3\n')
+
+        pythonw = os.path.join(self.tempdir, 'f')
+        with open(pythonw, 'w') as fd:
+            fd.write('#!/usr/bin/env pythonw\n')
+
+        self.assertEqual(
+            sorted(iterSourceCode([self.tempdir])),
+            sorted([python, python2, python3, pythonw]))
+
     def test_multipleDirectories(self):
         """
         L{iterSourceCode} can be given multiple directories.  It will recurse