Repository: ambv/black (python)

Issue:
black adds trailing comma despite using --target-version py35
Hi,

black 19.3b0 adds trailing commas for function signatures containing `*` despite setting target version 3.5. This creates invalid code for python 3.5 which will raise a SyntaxError. Setting
target version to py34 or using the auto detection doesn't add a trailing comma in this case.

```python3
def foo_func(bar, *, lorem=None, ipsum=None, dolor=None, sit=None, amet=None, consectetur=None):
    print('lorem ipsum dolor sit amet')
```
running
```sh
> black --target-version py35 --line-length 80 foo.py
```
creates

```python3
def foo_func(
    bar,
    *,
    lorem=None,
    ipsum=None,
    dolor=None,
    sit=None,
    amet=None,
    consectetur=None,
):
    print("lorem ipsum dolor sit amet")
``` 

Operating system: Ubuntu 18.04
Python version: Python 3.5/3.6
*Black* version: 
```sh
> black --version
black, version 19.3b0
```
Does also happen on master: yes


Pull request description:
Split the TRAILING_COMMA feature
It turns out that `f(*x,)` became legal in 3.5, but `def f(*x,):` only became legal in 3.6. Therefore, use two separate Features.

Fixes #759.

Patch:
diff --git a/.appveyor.yml b/.appveyor.yml
index 6dadfb3..6389f45 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -20,7 +20,7 @@ deploy:
   provider: GitHub
   description: ''
   auth_token:
-    secure: caPP8BcGffPuTopDfdI4ZVKZkpD9l9UAcjO7rwm5/fFSMbsqh1v4S38nmMn4uSuD
+    secure: uplI9CJ2dTGcEBCbZuIn+Qb4rC38hOoRSH9lcwuGCr5g9fSnhK1MZdNT6Cjf/mFL
   on:
-    appveyor_repo_tag: true
+    APPVEYOR_REPO_TAG: true
 
diff --git a/README.md b/README.md
index 2574c55..540960e 100644
--- a/README.md
+++ b/README.md
@@ -577,7 +577,7 @@ to denote a significant space character.
 ```toml
 [tool.black]
 line-length = 88
-target_version = ['py37']
+target-version = ['py37']
 include = '\.pyi?$'
 exclude = '''
 
@@ -951,6 +951,11 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Change Log
 
+### 19.3b1
+
+* fix bug that led *Black* format some code with a line length target of 1
+  (#762)
+
 ### 19.3b0
 
 * new option `--target-version` to control which Python versions
diff --git a/black.py b/black.py
index 680b1f4..0ea9b9a 100644
--- a/black.py
+++ b/black.py
@@ -68,7 +68,7 @@
 Priority = int
 Index = int
 LN = Union[Leaf, Node]
-SplitFunc = Callable[["Line", bool], Iterator["Line"]]
+SplitFunc = Callable[["Line", Collection["Feature"]], Iterator["Line"]]
 Timestamp = float
 FileSize = int
 CacheInfo = Tuple[Timestamp, FileSize]
@@ -133,31 +133,35 @@ class Feature(Enum):
     UNICODE_LITERALS = 1
     F_STRINGS = 2
     NUMERIC_UNDERSCORES = 3
-    TRAILING_COMMA = 4
+    TRAILING_COMMA_IN_CALL = 4
+    TRAILING_COMMA_IN_DEF = 5
 
 
 VERSION_TO_FEATURES: Dict[TargetVersion, Set[Feature]] = {
     TargetVersion.PY27: set(),
     TargetVersion.PY33: {Feature.UNICODE_LITERALS},
     TargetVersion.PY34: {Feature.UNICODE_LITERALS},
-    TargetVersion.PY35: {Feature.UNICODE_LITERALS, Feature.TRAILING_COMMA},
+    TargetVersion.PY35: {Feature.UNICODE_LITERALS, Feature.TRAILING_COMMA_IN_CALL},
     TargetVersion.PY36: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,
-        Feature.TRAILING_COMMA,
+        Feature.TRAILING_COMMA_IN_CALL,
+        Feature.TRAILING_COMMA_IN_DEF,
     },
     TargetVersion.PY37: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,
-        Feature.TRAILING_COMMA,
+        Feature.TRAILING_COMMA_IN_CALL,
+        Feature.TRAILING_COMMA_IN_DEF,
     },
     TargetVersion.PY38: {
         Feature.UNICODE_LITERALS,
         Feature.F_STRINGS,
         Feature.NUMERIC_UNDERSCORES,
-        Feature.TRAILING_COMMA,
+        Feature.TRAILING_COMMA_IN_CALL,
+        Feature.TRAILING_COMMA_IN_DEF,
     },
 }
 
@@ -683,6 +687,11 @@ def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
     elt = EmptyLineTracker(is_pyi=mode.is_pyi)
     empty_line = Line()
     after = 0
+    split_line_features = {
+        feature
+        for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}
+        if supports_feature(versions, feature)
+    }
     for current_line in lines.visit(src_node):
         for _ in range(after):
             dst_contents += str(empty_line)
@@ -690,9 +699,7 @@ def format_str(src_contents: str, *, mode: FileMode) -> FileContent:
         for _ in range(before):
             dst_contents += str(empty_line)
         for line in split_line(
-            current_line,
-            line_length=mode.line_length,
-            supports_trailing_commas=supports_feature(versions, Feature.TRAILING_COMMA),
+            current_line, line_length=mode.line_length, features=split_line_features
         ):
             dst_contents += str(line)
     return dst_contents
@@ -715,24 +722,20 @@ def decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:
         return tiow.read(), encoding, newline
 
 
-GRAMMARS = [
-    pygram.python_grammar_no_print_statement_no_exec_statement,
-    pygram.python_grammar_no_print_statement,
-    pygram.python_grammar,
-]
-
-
 def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
     if not target_versions:
-        return GRAMMARS
-    elif all(not version.is_python2() for version in target_versions):
-        # Python 3-compatible code, so don't try Python 2 grammar
+        # No target_version specified, so try all grammars.
         return [
             pygram.python_grammar_no_print_statement_no_exec_statement,
             pygram.python_grammar_no_print_statement,
+            pygram.python_grammar,
         ]
-    else:
+    elif all(version.is_python2() for version in target_versions):
+        # Python 2-only code, so try Python 2 grammars.
         return [pygram.python_grammar_no_print_statement, pygram.python_grammar]
+    else:
+        # Python 3-compatible code, so only try Python 3 grammar.
+        return [pygram.python_grammar_no_print_statement_no_exec_statement]
 
 
 def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
@@ -2162,7 +2165,7 @@ def split_line(
     line: Line,
     line_length: int,
     inner: bool = False,
-    supports_trailing_commas: bool = False,
+    features: Collection[Feature] = (),
 ) -> Iterator[Line]:
     """Split a `line` into potentially many lines.
 
@@ -2171,7 +2174,7 @@ def split_line(
     current `line`, possibly transitively. This means we can fallback to splitting
     by delimiters if the LHS/RHS don't yield any results.
 
-    If `supports_trailing_commas` is True, splitting may use the TRAILING_COMMA feature.
+    `features` are syntactical features that may be used in the output.
     """
     if line.is_comment:
         yield line
@@ -2192,13 +2195,9 @@ def split_line(
         split_funcs = [left_hand_split]
     else:
 
-        def rhs(line: Line, supports_trailing_commas: bool = False) -> Iterator[Line]:
+        def rhs(line: Line, features: Collection[Feature]) -> Iterator[Line]:
             for omit in generate_trailers_to_omit(line, line_length):
-                lines = list(
-                    right_hand_split(
-                        line, line_length, supports_trailing_commas, omit=omit
-                    )
-                )
+                lines = list(right_hand_split(line, line_length, features, omit=omit))
                 if is_line_short_enough(lines[0], line_length=line_length):
                     yield from lines
                     return
@@ -2206,7 +2205,7 @@ def rhs(line: Line, supports_trailing_commas: bool = False) -> Iterator[Line]:
             # All splits failed, best effort split with no omits.
             # This mostly happens to multiline strings that are by definition
             # reported as not fitting a single line.
-            yield from right_hand_split(line, line_length, supports_trailing_commas)
+            yield from right_hand_split(line, line_length, features=features)
 
         if line.inside_brackets:
             split_funcs = [delimiter_split, standalone_comment_split, rhs]
@@ -2218,16 +2217,13 @@ def rhs(line: Line, supports_trailing_commas: bool = False) -> Iterator[Line]:
         # split altogether.
         result: List[Line] = []
         try:
-            for l in split_func(line, supports_trailing_commas):
+            for l in split_func(line, features):
                 if str(l).strip("\n") == line_str:
                     raise CannotSplit("Split function returned an unchanged result")
 
                 result.extend(
                     split_line(
-                        l,
-                        line_length=line_length,
-                        inner=True,
-                        supports_trailing_commas=supports_trailing_commas,
+                        l, line_length=line_length, inner=True, features=features
                     )
                 )
         except CannotSplit:
@@ -2241,9 +2237,7 @@ def rhs(line: Line, supports_trailing_commas: bool = False) -> Iterator[Line]:
         yield line
 
 
-def left_hand_split(
-    line: Line, supports_trailing_commas: bool = False
-) -> Iterator[Line]:
+def left_hand_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
     """Split line into many lines, starting with the first matching bracket pair.
 
     Note: this usually looks weird, only use this for function definitions.
@@ -2282,7 +2276,7 @@ def left_hand_split(
 def right_hand_split(
     line: Line,
     line_length: int,
-    supports_trailing_commas: bool = False,
+    features: Collection[Feature] = (),
     omit: Collection[LeafID] = (),
 ) -> Iterator[Line]:
     """Split line into many lines, starting with the last matching bracket pair.
@@ -2341,12 +2335,7 @@ def right_hand_split(
     ):
         omit = {id(closing_bracket), *omit}
         try:
-            yield from right_hand_split(
-                line,
-                line_length,
-                supports_trailing_commas=supports_trailing_commas,
-                omit=omit,
-            )
+            yield from right_hand_split(line, line_length, features=features, omit=omit)
             return
 
         except CannotSplit:
@@ -2435,10 +2424,8 @@ def dont_increase_indentation(split_func: SplitFunc) -> SplitFunc:
     """
 
     @wraps(split_func)
-    def split_wrapper(
-        line: Line, supports_trailing_commas: bool = False
-    ) -> Iterator[Line]:
-        for l in split_func(line, supports_trailing_commas):
+    def split_wrapper(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
+        for l in split_func(line, features):
             normalize_prefix(l.leaves[0], inside_brackets=True)
             yield l
 
@@ -2446,13 +2433,11 @@ def split_wrapper(
 
 
 @dont_increase_indentation
-def delimiter_split(
-    line: Line, supports_trailing_commas: bool = False
-) -> Iterator[Line]:
+def delimiter_split(line: Line, features: Collection[Feature] = ()) -> Iterator[Line]:
     """Split according to delimiters of the highest priority.
 
-    If `supports_trailing_commas` is True, the split will add trailing commas
-    also in function signatures that contain `*` and `**`.
+    If the appropriate Features are given, the split will add trailing commas
+    also in function signatures and calls that contain `*` and `**`.
     """
     try:
         last_leaf = line.leaves[-1]
@@ -2491,10 +2476,16 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]:
             yield from append_to_line(comment_after)
 
         lowest_depth = min(lowest_depth, leaf.bracket_depth)
-        if leaf.bracket_depth == lowest_depth and is_vararg(
-            leaf, within=VARARGS_PARENTS
-        ):
-            trailing_comma_safe = trailing_comma_safe and supports_trailing_commas
+        if leaf.bracket_depth == lowest_depth:
+            if is_vararg(leaf, within={syms.typedargslist}):
+                trailing_comma_safe = (
+                    trailing_comma_safe and Feature.TRAILING_COMMA_IN_DEF in features
+                )
+            elif is_vararg(leaf, within={syms.arglist, syms.argument}):
+                trailing_comma_safe = (
+                    trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features
+                )
+
         leaf_priority = bt.delimiters.get(id(leaf))
         if leaf_priority == delimiter_priority:
             yield current_line
@@ -2513,7 +2504,7 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]:
 
 @dont_increase_indentation
 def standalone_comment_split(
-    line: Line, supports_trailing_commas: bool = False
+    line: Line, features: Collection[Feature] = ()
 ) -> Iterator[Line]:
     """Split standalone comments from the rest of the line."""
     if not line.contains_standalone_comments(0):
@@ -3063,14 +3054,19 @@ def get_features_used(node: Node) -> Set[Feature]:
             and n.children
             and n.children[-1].type == token.COMMA
         ):
+            if n.type == syms.typedargslist:
+                feature = Feature.TRAILING_COMMA_IN_DEF
+            else:
+                feature = Feature.TRAILING_COMMA_IN_CALL
+
             for ch in n.children:
                 if ch.type in STARS:
-                    features.add(Feature.TRAILING_COMMA)
+                    features.add(feature)
 
                 if ch.type == syms.argument:
                     for argch in ch.children:
                         if argch.type in STARS:
-                            features.add(Feature.TRAILING_COMMA)
+                            features.add(feature)
 
     return features
 
diff --git a/plugin/black.vim b/plugin/black.vim
index f0b55b2..36825cf 100644
--- a/plugin/black.vim
+++ b/plugin/black.vim
@@ -98,15 +98,14 @@ if _initialize_black_env():
 def Black():
   start = time.time()
   fast = bool(int(vim.eval("g:black_fast")))
-  line_length = int(vim.eval("g:black_linelength"))
-  mode = black.FileMode.AUTO_DETECT
-  if bool(int(vim.eval("g:black_skip_string_normalization"))):
-    mode |= black.FileMode.NO_STRING_NORMALIZATION
-  if vim.current.buffer.name.endswith('.pyi'):
-    mode |= black.FileMode.PYI
+  mode = black.FileMode(
+    line_length=int(vim.eval("g:black_linelength")),
+    string_normalization=not bool(int(vim.eval("g:black_skip_string_normalization"))),
+    is_pyi=vim.current.buffer.name.endswith('.pyi'),
+  )
   buffer_str = '\n'.join(vim.current.buffer) + '\n'
   try:
-    new_buffer_str = black.format_file_contents(buffer_str, line_length=line_length, fast=fast, mode=mode)
+    new_buffer_str = black.format_file_contents(buffer_str, fast=fast, mode=mode)
   except black.NothingChanged:
     print(f'Already well formatted, good job. (took {time.time() - start:.4f}s)')
   except Exception as exc:
diff --git a/pyproject.toml b/pyproject.toml
index 3b192bf..d88fabe 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@
 
 [tool.black]
 line-length = 88
-target_version = ['py36', 'py37', 'py38']
+target-version = ['py36', 'py37', 'py38']
 include = '\.pyi?$'
 exclude = '''
 /(


Test patch:
diff --git a/tests/test_black.py b/tests/test_black.py
index 645eec7..3e0b8a1 100644
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -28,7 +28,7 @@
 from click.testing import CliRunner
 
 import black
-from black import Feature
+from black import Feature, TargetVersion
 
 try:
     import blackd
@@ -464,7 +464,7 @@ def test_python2(self) -> None:
     @patch("black.dump_to_file", dump_to_stderr)
     def test_python2_print_function(self) -> None:
         source, expected = read_data("python2_print_function")
-        mode = black.FileMode(target_versions={black.TargetVersion.PY27})
+        mode = black.FileMode(target_versions={TargetVersion.PY27})
         actual = fs(source, mode=mode)
         self.assertFormatEqual(expected, actual)
         black.assert_stable(source, actual, mode)
@@ -828,11 +828,40 @@ def err(msg: str, **kwargs: Any) -> None:
                 "2 files would fail to reformat.",
             )
 
+    def test_lib2to3_parse(self) -> None:
+        with self.assertRaises(black.InvalidInput):
+            black.lib2to3_parse("invalid syntax")
+
+        straddling = "x + y"
+        black.lib2to3_parse(straddling)
+        black.lib2to3_parse(straddling, {TargetVersion.PY27})
+        black.lib2to3_parse(straddling, {TargetVersion.PY36})
+        black.lib2to3_parse(straddling, {TargetVersion.PY27, TargetVersion.PY36})
+
+        py2_only = "print x"
+        black.lib2to3_parse(py2_only)
+        black.lib2to3_parse(py2_only, {TargetVersion.PY27})
+        with self.assertRaises(black.InvalidInput):
+            black.lib2to3_parse(py2_only, {TargetVersion.PY36})
+        with self.assertRaises(black.InvalidInput):
+            black.lib2to3_parse(py2_only, {TargetVersion.PY27, TargetVersion.PY36})
+
+        py3_only = "exec(x, end=y)"
+        black.lib2to3_parse(py3_only)
+        with self.assertRaises(black.InvalidInput):
+            black.lib2to3_parse(py3_only, {TargetVersion.PY27})
+        black.lib2to3_parse(py3_only, {TargetVersion.PY36})
+        black.lib2to3_parse(py3_only, {TargetVersion.PY27, TargetVersion.PY36})
+
     def test_get_features_used(self) -> None:
         node = black.lib2to3_parse("def f(*, arg): ...\n")
         self.assertEqual(black.get_features_used(node), set())
         node = black.lib2to3_parse("def f(*, arg,): ...\n")
-        self.assertEqual(black.get_features_used(node), {Feature.TRAILING_COMMA})
+        self.assertEqual(black.get_features_used(node), {Feature.TRAILING_COMMA_IN_DEF})
+        node = black.lib2to3_parse("f(*arg,)\n")
+        self.assertEqual(
+            black.get_features_used(node), {Feature.TRAILING_COMMA_IN_CALL}
+        )
         node = black.lib2to3_parse("def f(*, arg): f'string'\n")
         self.assertEqual(black.get_features_used(node), {Feature.F_STRINGS})
         node = black.lib2to3_parse("123_456\n")
@@ -841,13 +870,14 @@ def test_get_features_used(self) -> None:
         self.assertEqual(black.get_features_used(node), set())
         source, expected = read_data("function")
         node = black.lib2to3_parse(source)
-        self.assertEqual(
-            black.get_features_used(node), {Feature.TRAILING_COMMA, Feature.F_STRINGS}
-        )
+        expected_features = {
+            Feature.TRAILING_COMMA_IN_CALL,
+            Feature.TRAILING_COMMA_IN_DEF,
+            Feature.F_STRINGS,
+        }
+        self.assertEqual(black.get_features_used(node), expected_features)
         node = black.lib2to3_parse(expected)
-        self.assertEqual(
-            black.get_features_used(node), {Feature.TRAILING_COMMA, Feature.F_STRINGS}
-        )
+        self.assertEqual(black.get_features_used(node), expected_features)
         source, expected = read_data("expression")
         node = black.lib2to3_parse(source)
         self.assertEqual(black.get_features_used(node), set())
@@ -1499,8 +1529,8 @@ async def check(header_value: str, expected_status: int) -> None:
 
             await check("3.6", 200)
             await check("py3.6", 200)
-            await check("3.5,3.7", 200)
-            await check("3.5,py3.7", 200)
+            await check("3.6,3.7", 200)
+            await check("3.6,py3.7", 200)
 
             await check("2", 204)
             await check("2.7", 204)