Repository: ambv/black (python)

Issue:
Can't parse `return (i * 2 async for i in arange(n))`
Title says all.

Pull request description:
Support parsing of async generators in non-async functions
This is a new syntax added in python3.7, so black can't verify that reformatting will not change the ast unless black itself is run with 3.7. We'll need to change the error message black gives in this case. @ambv any ideas?

Fixes #125.

Patch:
diff --git a/README.md b/README.md
index 96a24a1..9cb4f18 100644
--- a/README.md
+++ b/README.md
@@ -371,6 +371,12 @@ human-readable strings"](https://stackoverflow.com/a/56190)), you can
 pass `--skip-string-normalization` on the command line.  This is meant as
 an adoption helper, avoid using this for new projects.
 
+### Numeric literals
+
+*Black* standardizes all numeric literals to use lowercase letters: `0xab`
+instead of `0XAB` and `1e10` instead of `1E10`. In Python 3.6+, *Black*
+adds underscores to long numeric literals to aid readability: `100000000`
+becomes `100_000_000`.
 
 ### Line breaks & binary operators
 
@@ -843,6 +849,8 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ### 18.8b0
 
+* code with `_` in numeric literals is recognized as Python 3.6+ (#461)
+
 * numeric literals are now normalized to include `_` separators on Python 3.6+ code
   (#452)
 
diff --git a/black.py b/black.py
index d688748..b65693e 100644
--- a/black.py
+++ b/black.py
@@ -170,7 +170,7 @@ def read_pyproject_toml(
     "--line-length",
     type=int,
     default=DEFAULT_LINE_LENGTH,
-    help="How many character per line to allow.",
+    help="How many characters per line to allow.",
     show_default=True,
 )
 @click.option(
@@ -1164,7 +1164,7 @@ def maybe_remove_trailing_comma(self, closing: Leaf) -> bool:
             self.remove_trailing_comma()
             return True
 
-        # Otheriwsse, if the trailing one is the only one, we might mistakenly
+        # Otherwise, if the trailing one is the only one, we might mistakenly
         # change a tuple into a different type by removing the comma.
         depth = closing.bracket_depth + 1
         commas = 0
@@ -1377,7 +1377,7 @@ def _maybe_empty_lines_for_class_or_def(
                 newlines = 1
             elif current_line.is_class or self.previous_line.is_class:
                 if current_line.is_stub_class and self.previous_line.is_stub_class:
-                    # No blank line between classes with an emty body
+                    # No blank line between classes with an empty body
                     newlines = 0
                 else:
                     newlines = 1
@@ -2891,7 +2891,8 @@ def is_python36(node: Node) -> bool:
     """Return True if the current file is using Python 3.6+ features.
 
     Currently looking for:
-    - f-strings; and
+    - f-strings;
+    - underscores in numeric literals; and
     - trailing commas after * or ** in function signatures and calls.
     """
     for n in node.pre_order():
@@ -2900,6 +2901,10 @@ def is_python36(node: Node) -> bool:
             if value_head in {'f"', 'F"', "f'", "F'", "rf", "fr", "RF", "FR"}:
                 return True
 
+        elif n.type == token.NUMBER:
+            if "_" in n.value:  # type: ignore
+                return True
+
         elif (
             n.type in {syms.typedargslist, syms.arglist}
             and n.children
@@ -3133,7 +3138,7 @@ def return_code(self) -> int:
         - otherwise return 0.
         """
         # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with
-        # 126 we have special returncodes reserved by the shell.
+        # 126 we have special return codes reserved by the shell.
         if self.failure_count:
             return 123
 
diff --git a/blib2to3/pgen2/tokenize.py b/blib2to3/pgen2/tokenize.py
index 9a7664b..1f51ff0 100644
--- a/blib2to3/pgen2/tokenize.py
+++ b/blib2to3/pgen2/tokenize.py
@@ -516,13 +516,14 @@ def generate_tokens(readline):
                         stashed = tok
                         continue
 
-                    if token == 'def':
+                    if token in ('def', 'for'):
                         if (stashed
                                 and stashed[0] == NAME
                                 and stashed[1] == 'async'):
 
-                            async_def = True
-                            async_def_indent = indents[-1]
+                            if token == 'def':
+                                async_def = True
+                                async_def_indent = indents[-1]
 
                             yield (ASYNC, stashed[1],
                                    stashed[2], stashed[3],


Test patch:
diff --git a/tests/data/python37.py b/tests/data/python37.py
new file mode 100644
index 0000000..ae20ade
--- /dev/null
+++ b/tests/data/python37.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env python3.7
+
+def f():
+    return (i*2 async for i in arange(42))
+
+# output
+
+
+#!/usr/bin/env python3.7
+
+
+def f():
+    return (i * 2 async for i in arange(42))
diff --git a/tests/test_black.py b/tests/test_black.py
index 884d046..9c798ca 100644
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -411,6 +411,16 @@ def test_stub(self) -> None:
         self.assertFormatEqual(expected, actual)
         black.assert_stable(source, actual, line_length=ll, mode=mode)
 
+    @patch("black.dump_to_file", dump_to_stderr)
+    def test_python37(self) -> None:
+        source, expected = read_data("python37")
+        actual = fs(source)
+        self.assertFormatEqual(expected, actual)
+        major, minor = sys.version_info[:2]
+        if major > 3 or (major == 3 and minor >= 7):
+            black.assert_equivalent(source, actual)
+        black.assert_stable(source, actual, line_length=ll)
+
     @patch("black.dump_to_file", dump_to_stderr)
     def test_fmtonoff(self) -> None:
         source, expected = read_data("fmtonoff")
@@ -722,6 +732,10 @@ def test_is_python36(self) -> None:
         self.assertTrue(black.is_python36(node))
         node = black.lib2to3_parse("def f(*, arg): f'string'\n")
         self.assertTrue(black.is_python36(node))
+        node = black.lib2to3_parse("123_456\n")
+        self.assertTrue(black.is_python36(node))
+        node = black.lib2to3_parse("123456\n")
+        self.assertFalse(black.is_python36(node))
         source, expected = read_data("function")
         node = black.lib2to3_parse(source)
         self.assertTrue(black.is_python36(node))