Repository: Instagram/LibCST (language: python)

Issue:
Escaping opening braces in f-strings causes a tokenizer error
> Not sure if is the same issue, but this also breaks the native parser: `f"regexp_like(path, '.*\{file_type}$')"`

> This is another example: f"\{{\}}"

_Originally posted by @lpetre in https://github.com/Instagram/LibCST/issues/668#issuecomment-1083354887_


Discussion / hints:

Fix patch:
diff --git a/native/libcst/src/tokenizer/core/mod.rs b/native/libcst/src/tokenizer/core/mod.rs
index 9a8695556..09b4a0048 100644
--- a/native/libcst/src/tokenizer/core/mod.rs
+++ b/native/libcst/src/tokenizer/core/mod.rs
@@ -940,7 +940,25 @@ impl<'t> TokState<'t> {
                             // skip escaped char (e.g. \', \", or newline/line continuation)
                             self.text_pos.next();
                         }
-                    } else {
+                    } else if let Some(
+                        '\n'
+                        | '\\'
+                        | '\''
+                        | '"'
+                        | 'a'
+                        | 'b'
+                        | 'f'
+                        | 'n'
+                        | 'r'
+                        | 't'
+                        | 'v'
+                        | 'x'
+                        | '0'..='9'
+                        | 'N'
+                        | 'u'
+                        | 'U',
+                    ) = self.text_pos.peek()
+                    {
                         // skip escaped char
                         let next_ch = self.text_pos.next();
                         // check if this is a \N sequence


Test patch:
diff --git a/native/libcst/src/tokenizer/tests.rs b/native/libcst/src/tokenizer/tests.rs
index 300e6d299..200d170a0 100644
--- a/native/libcst/src/tokenizer/tests.rs
+++ b/native/libcst/src/tokenizer/tests.rs
@@ -617,6 +617,34 @@ fn test_split_fstring() {
     );
 }
 
+#[test]
+fn test_fstring_escapes() {
+    let config = TokConfig {
+        split_fstring: true,
+        ..default_config()
+    };
+    assert_eq!(
+        tokenize_all("f'\\{{\\}}'", &config),
+        Ok(vec![
+            (TokType::FStringStart, "f'"),
+            (TokType::FStringString, "\\{{\\}}"),
+            (TokType::FStringEnd, "'"),
+        ])
+    );
+    assert_eq!(
+        tokenize_all(r#"f"regexp_like(path, '.*\{file_type}$')""#, &config),
+        Ok(vec![
+            (TokType::FStringStart, "f\""),
+            (TokType::FStringString, "regexp_like(path, '.*\\"),
+            (TokType::Op, "{"),
+            (TokType::Name, "file_type"),
+            (TokType::Op, "}"),
+            (TokType::FStringString, "$')"),
+            (TokType::FStringEnd, "\""),
+        ])
+    );
+}
+
 #[test]
 fn test_operator() {
     assert_eq!(
diff --git a/native/libcst/tests/fixtures/super_strings.py b/native/libcst/tests/fixtures/super_strings.py
index 76b1b2ddd..d993f5abd 100644
--- a/native/libcst/tests/fixtures/super_strings.py
+++ b/native/libcst/tests/fixtures/super_strings.py
@@ -26,3 +26,7 @@
 _(f"ok { expr = !r: aosidjhoi } end")
 
 print(f"{self.ERASE_CURRENT_LINE}{self._human_seconds(elapsed_time)} {percent:.{self.pretty_precision}f}% complete, {self.estimate_completion(elapsed_time, finished, left)} estimated for {left} files to go...")
+
+f'\{{\}}'
+f"regexp_like(path, '.*\{file_type}$')"
+f"\lfoo"