Repository: Instagram/LibCST

Issue:
parse error for comprehensions in f-strings
```
❯ cat test.py                                                                                                                                                    
x = [1, 2]               
print(f"List of: {y for y in x}")           
❯ python3 test.py                                                                                                                                                
List of: <generator object <genexpr> at 0x7facdf7d8258>
❯ libcst print test.py        
Traceback (most recent call last):                                                                                                                               
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/_parser/base_parser.py", line 152, in _add_token
    plan = stack[-1].dfa.transitions[transition]                                                                                                                 
KeyError: ReservedString(for)
                                                                                                                                                                 
During handling of the above exception, another exception occurred:
                                                                                                                                                                 
Traceback (most recent call last):
  File "/opt/instagram/virtualenv/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)                                                       
  File "/opt/instagram/virtualenv/lib/python3.7/runpy.py", line 85, in _run_code 
    exec(code, run_globals)      
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/tool.py", line 832, in <module>
    main(os.environ.get("LIBCST_TOOL_COMMAND_NAME", "libcst.tool"), sys.argv[1:])
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/tool.py", line 827, in main
    return lookup.get(args.action or None, _invalid_command)(proc_name, command_args)
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/tool.py", line 277, in _print_tree_impl
    else PartialParserConfig()                                                                                                                                   
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/_parser/entrypoints.py", line 76, in parse_module
    detect_default_newline=True,
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/_parser/entrypoints.py", line 51, in _parse
    result = parser.parse()                                                     
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/_parser/base_parser.py", line 111, in parse
    self._add_token(token)        
  File "/opt/instagram/virtualenv/lib/python3.7/site-packages/fbcode/libcst/_parser/base_parser.py", line 191, in _add_token
    raw_column=token.start_pos[1],
libcst._exceptions.ParserSyntaxError: Syntax Error @ 2:21.       
Incomplete input. Encountered 'for', but expected '!', ':', or '}'.
                                                                                                                                                                 
print(f"List of: {y for y in x}")
```

Comments:

Patch:
diff --git a/libcst/_parser/conversions/expression.py b/libcst/_parser/conversions/expression.py
index 8edbf26..b7e5c18 100644
--- a/libcst/_parser/conversions/expression.py
+++ b/libcst/_parser/conversions/expression.py
@@ -1038,12 +1038,12 @@ def convert_fstring_equality(
 
 @with_production(
     "fstring_expr",
-    "'{' testlist [ fstring_equality ] [ fstring_conversion ] [ fstring_format_spec ] '}'",
+    "'{' testlist_comp_tuple [ fstring_equality ] [ fstring_conversion ] [ fstring_format_spec ] '}'",
     version=">=3.8",
 )
 @with_production(
     "fstring_expr",
-    "'{' testlist [ fstring_conversion ] [ fstring_format_spec ] '}'",
+    "'{' testlist_comp_tuple [ fstring_conversion ] [ fstring_format_spec ] '}'",
     version="<=3.7",
 )
 def convert_fstring_expr(
@@ -1441,8 +1441,16 @@ def convert_arg_assign_comp_for(
         elt, for_in = children
         return Arg(value=GeneratorExp(elt.value, for_in, lpar=(), rpar=()))
     else:
-        # "key = value" assignment argument
         lhs, equal, rhs = children
+        # "key := value" assignment; positional
+        if equal.string == ":=":
+            val = convert_namedexpr_test(config, children)
+            if not isinstance(val, WithLeadingWhitespace):
+                raise Exception(
+                    f"convert_namedexpr_test returned {val!r}, not WithLeadingWhitespace"
+                )
+            return Arg(value=val.value)
+        # "key = value" assignment; keyword argument
         return Arg(
             keyword=lhs.value,
             equal=AssignEqual(


Test patch:
diff --git a/libcst/_nodes/tests/test_atom.py b/libcst/_nodes/tests/test_atom.py
index 452c9f7..1a14e37 100644
--- a/libcst/_nodes/tests/test_atom.py
+++ b/libcst/_nodes/tests/test_atom.py
@@ -668,6 +668,35 @@ class AtomTest(CSTNodeTest):
                 "parser": parse_expression,
                 "expected_position": CodeRange((1, 1), (1, 4)),
             },
+            # Generator expression (doesn't make sense, but legal syntax)
+            {
+                "node": cst.FormattedString(
+                    start='f"',
+                    parts=[
+                        cst.FormattedStringExpression(
+                            expression=cst.GeneratorExp(
+                                elt=cst.Name(
+                                    value="x",
+                                ),
+                                for_in=cst.CompFor(
+                                    target=cst.Name(
+                                        value="x",
+                                    ),
+                                    iter=cst.Name(
+                                        value="y",
+                                    ),
+                                ),
+                                lpar=[],
+                                rpar=[],
+                            ),
+                        ),
+                    ],
+                    end='"',
+                ),
+                "code": 'f"{x for x in y}"',
+                "parser": parse_expression,
+                "expected_position": None,
+            },
             # Concatenated strings
             {
                 "node": cst.ConcatenatedString(
diff --git a/libcst/_nodes/tests/test_namedexpr.py b/libcst/_nodes/tests/test_namedexpr.py
index 4ba1485..3949bbe 100644
--- a/libcst/_nodes/tests/test_namedexpr.py
+++ b/libcst/_nodes/tests/test_namedexpr.py
@@ -101,6 +101,71 @@ class NamedExprTest(CSTNodeTest):
                 "parser": _parse_statement_force_38,
                 "expected_position": None,
             },
+            # Function args
+            {
+                "node": cst.Call(
+                    func=cst.Name(value="f"),
+                    args=[
+                        cst.Arg(
+                            value=cst.NamedExpr(
+                                target=cst.Name(value="y"),
+                                value=cst.Integer(value="1"),
+                                whitespace_before_walrus=cst.SimpleWhitespace(""),
+                                whitespace_after_walrus=cst.SimpleWhitespace(""),
+                            )
+                        ),
+                    ],
+                ),
+                "code": "f(y:=1)",
+                "parser": _parse_expression_force_38,
+                "expected_position": None,
+            },
+            # Whitespace handling on args is fragile
+            {
+                "node": cst.Call(
+                    func=cst.Name(value="f"),
+                    args=[
+                        cst.Arg(
+                            value=cst.Name(value="x"),
+                            comma=cst.Comma(
+                                whitespace_after=cst.SimpleWhitespace("  ")
+                            ),
+                        ),
+                        cst.Arg(
+                            value=cst.NamedExpr(
+                                target=cst.Name(value="y"),
+                                value=cst.Integer(value="1"),
+                                whitespace_before_walrus=cst.SimpleWhitespace("   "),
+                                whitespace_after_walrus=cst.SimpleWhitespace("    "),
+                            ),
+                            whitespace_after_arg=cst.SimpleWhitespace("     "),
+                        ),
+                    ],
+                ),
+                "code": "f(x,  y   :=    1     )",
+                "parser": _parse_expression_force_38,
+                "expected_position": None,
+            },
+            {
+                "node": cst.Call(
+                    func=cst.Name(value="f"),
+                    args=[
+                        cst.Arg(
+                            value=cst.NamedExpr(
+                                target=cst.Name(value="y"),
+                                value=cst.Integer(value="1"),
+                                whitespace_before_walrus=cst.SimpleWhitespace("   "),
+                                whitespace_after_walrus=cst.SimpleWhitespace("    "),
+                            ),
+                            whitespace_after_arg=cst.SimpleWhitespace("     "),
+                        ),
+                    ],
+                    whitespace_before_args=cst.SimpleWhitespace("  "),
+                ),
+                "code": "f(  y   :=    1     )",
+                "parser": _parse_expression_force_38,
+                "expected_position": None,
+            },
         )
     )
     def test_valid(self, **kwargs: Any) -> None: