Repository: Instagram/LibCST (language: python)

Issue:
Tokenizer error when triple-quoted f-string inside single-quoted f-string
This is related to PEP 701 (updated f-string grammar).

When a multiline f-string is inside a non-multiline f-string, the tokenizer throws an error. So, for the following code:
```python
f"some {f"""multiline
allowed {x}"""} string"
```

The output is:
```
Failed to tokenize file: tokenizer error: unterminated string literal
```

I'm running the tokenizer using the following function:
```rust
libcst_native::tokenize(&contents)
```

Looking at the code, I think the problem is that the `allow_multiline` variable should only check for the latest f-string node on the stack and not _all_ of them:

https://github.com/Instagram/LibCST/blob/9eab2f037fa3680e0627d23038457b65dbb4078f/native/libcst/src/tokenizer/core/mod.rs#L897


Discussion / hints:

Fix patch:
diff --git a/native/libcst/src/tokenizer/core/mod.rs b/native/libcst/src/tokenizer/core/mod.rs
index 067c1cf9e..477bd8680 100644
--- a/native/libcst/src/tokenizer/core/mod.rs
+++ b/native/libcst/src/tokenizer/core/mod.rs
@@ -363,7 +363,8 @@ impl<'t> TokState<'t> {
                     self.text_pos.next();
                     self.at_bol = true;
                     if self.split_fstring
-                        && !self.fstring_stack.iter().all(|node| node.allow_multiline())
+                        && self.fstring_stack.last().map(|node| node.allow_multiline())
+                            == Some(false)
                     {
                         Err(TokError::UnterminatedString)
                     } else if self.blank_line || !self.paren_stack.is_empty() {
@@ -895,7 +896,8 @@ impl<'t> TokState<'t> {
         is_in_format_spec: bool,
         is_raw_string: bool,
     ) -> Result<Option<TokType>, TokError<'t>> {
-        let allow_multiline = self.fstring_stack.iter().all(|node| node.allow_multiline());
+        let allow_multiline =
+            self.fstring_stack.last().map(|node| node.allow_multiline()) == Some(true);
         let mut in_named_unicode: bool = false;
         let mut ok_result = Ok(None); // value to return if we reach the end and don't error out
         'outer: loop {
diff --git a/native/libcst/src/tokenizer/core/string_types.rs b/native/libcst/src/tokenizer/core/string_types.rs
index d14d13f5b..b04ccb397 100644
--- a/native/libcst/src/tokenizer/core/string_types.rs
+++ b/native/libcst/src/tokenizer/core/string_types.rs
@@ -105,7 +105,7 @@ impl FStringNode {
     }
 
     pub fn allow_multiline(&self) -> bool {
-        self.quote_size == StringQuoteSize::Triple
+        self.quote_size == StringQuoteSize::Triple || self.is_in_expr()
     }
 
     pub fn is_in_expr(&self) -> bool {


Test patch:
diff --git a/native/libcst/tests/fixtures/super_strings.py b/native/libcst/tests/fixtures/super_strings.py
index c1471a459..25eee0e04 100644
--- a/native/libcst/tests/fixtures/super_strings.py
+++ b/native/libcst/tests/fixtures/super_strings.py
@@ -27,6 +27,16 @@
 
 print(f"{self.ERASE_CURRENT_LINE}{self._human_seconds(elapsed_time)} {percent:.{self.pretty_precision}f}% complete, {self.estimate_completion(elapsed_time, finished, left)} estimated for {left} files to go...")
 
+f"{"\n".join()}"
+
+f"___{
+    x
+}___"
+
+f"___{(
+    x
+)}___"
+
 f'\{{\}}'
 f"regexp_like(path, '.*\{file_type}$')"
 f"\lfoo"
@@ -38,4 +48,3 @@
 f"{'':*^{1:{1}}}"
 f"{'':*^{1:{1:{1}}}}"
 f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"
-