Repository: Instagram/LibCST (language: python)

Issue:
Native parser crashes on `rf"\\"`
```python
import contextlib, os
from libcst import parse_expression

@contextlib.contextmanager
def native_parser():
    var = os.environ.get("LIBCST_PARSER_TYPE")
    try:
        os.environ["LIBCST_PARSER_TYPE"] = "native"
        yield
    finally:
        os.environ.pop("LIBCST_PARSER_TYPE")
        if var is not None:
            os.environ["LIBCST_PARSER_TYPE"] = var

string_literal = r'rf"\\"'
parse_expression(string_literal)  # fine :-)
with native_parser():
    parse_expression(string_literal)  # fails :-(
```




Discussion / hints:

Fix patch:
diff --git a/native/libcst/src/tokenizer/core/mod.rs b/native/libcst/src/tokenizer/core/mod.rs
index 89877f7e1..7c0f0788e 100644
--- a/native/libcst/src/tokenizer/core/mod.rs
+++ b/native/libcst/src/tokenizer/core/mod.rs
@@ -973,8 +973,8 @@ impl<'t> TokState<'t> {
                 }
                 (Some('\\'), _) if is_raw_string => {
                     self.text_pos.next();
-                    if let Some('"' | '\'') = self.text_pos.peek() {
-                        // these aren't end of string markers, skip them
+                    // skip escaped end-of-string marker or backslash
+                    if let Some('"' | '\'' | '\\') = self.text_pos.peek() {
                         self.text_pos.next();
                     }
                 }


Test patch:
diff --git a/native/libcst/src/tokenizer/tests.rs b/native/libcst/src/tokenizer/tests.rs
index 69deaaf3a..4e8ce4d3e 100644
--- a/native/libcst/src/tokenizer/tests.rs
+++ b/native/libcst/src/tokenizer/tests.rs
@@ -529,6 +529,10 @@ fn test_string_prefix() {
         tokenize_all(r#"r"\"""#, &default_config()),
         Ok(vec![(TokType::String, r#"r"\"""#)]),
     );
+    assert_eq!(
+        tokenize_all(r#"r'\\'"#, &default_config()),
+        Ok(vec![(TokType::String, r#"r'\\'"#)]),
+    );
     let config = TokConfig {
         split_fstring: true,
         ..default_config()
@@ -549,6 +553,14 @@ fn test_string_prefix() {
             (TokType::FStringEnd, "\""),
         ]),
     );
+    assert_eq!(
+        tokenize_all(r#"rf'\\'"#, &config),
+        Ok(vec![
+            (TokType::FStringStart, "rf'"),
+            (TokType::FStringString, r#"\\"#),
+            (TokType::FStringEnd, "'"),
+        ]),
+    );
 }
 
 #[test]