Repository: RustPython/RustPython (language: rust)

Issue:
Some valid named unicode characters cause syntax errors
`"\N{BACKSPACE}another cool trick"` yields a syntax error in RustPython, but runs without error on CPython.

See: https://github.com/charliermarsh/ruff/issues/3207.


Discussion / hints:
For reference, Python does throw errors on obviously-invalid named unicode characters like `"\N{AAA} another cool trick"`.
Seems like this is coming from `unicode_names2`. `unicode_names2::character("BACKSPACE")` doesn't return a character.
Related question (out of curiosity): What does `unicode_names2::name('\x08')` return?
It's also a `None`, it doesn't support any of the control characters (so `'\N{LINE FEED}'`, `'\N{NUL}'` etc, all raise SyntaxErrors, this is probably intentional).
After a bit of searching it appears it doesn't support [name aliases](http://www.unicode.org/Public/12.1.0/ucd/NameAliases.txt) (which all these control characters fall into). I personally don't know of any libs that support it but we could just grab that file, parse it and have `unicodedata` and the string parser fall back to it. (or, even better, just build a static map for it)
[unic-ucd-name_aliases](https://docs.rs/unic-ucd-name_aliases/) supports getting alias from the name but not the other way.
https://github.com/progval/unicode_names2/issues/11
unicode_names2 will take patch
https://github.com/progval/unicode_names2/issues/11#issuecomment-1445037320

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 9b811d063b..57721dc7d1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2933,9 +2933,11 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
 
 [[package]]
 name = "unicode_names2"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301"
+version = "0.6.0"
+source = "git+https://github.com/youknowone/unicode_names2.git?tag=v0.6.0+character-alias#4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde"
+dependencies = [
+ "phf",
+]
 
 [[package]]
 name = "utf8parse"
diff --git a/Cargo.toml b/Cargo.toml
index f305420ba9..cb5603171a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,6 +55,7 @@ static_assertions = "1.1"
 syn = "1.0.91"
 thiserror = "1.0"
 thread_local = "1.1.4"
+unicode_names2 = { version = "0.6.0", git = "https://github.com/youknowone/unicode_names2.git", tag = "v0.6.0+character-alias" }
 widestring = "0.5.1"
 
 [features]
diff --git a/compiler/parser/Cargo.toml b/compiler/parser/Cargo.toml
index 56f3d161f5..40bf75e508 100644
--- a/compiler/parser/Cargo.toml
+++ b/compiler/parser/Cargo.toml
@@ -27,10 +27,10 @@ log = { workspace = true }
 num-bigint = { workspace = true }
 num-traits = { workspace = true }
 thiserror = { workspace = true }
+unicode_names2 = { workspace = true }
 
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
-unicode_names2 = "0.5.0"
 lalrpop-util = "0.19.8"
 phf = "0.11.1"
 rustc-hash = "1.1.0"
diff --git a/compiler/parser/src/string.rs b/compiler/parser/src/string.rs
index a8be77782e..0cd03df666 100644
--- a/compiler/parser/src/string.rs
+++ b/compiler/parser/src/string.rs
@@ -1048,4 +1048,28 @@ mod tests {
         let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    macro_rules! test_aliases_parse {
+        ($($name:ident: $alias:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = format!(r#""\N{{{0}}}""#, $alias);
+                let parse_ast = parse_program(&source, "<test>").unwrap();
+                insta::assert_debug_snapshot!(parse_ast);
+            }
+        )*
+        }
+    }
+
+    test_aliases_parse! {
+        test_backspace_alias: "BACKSPACE",
+        test_bell_alias: "BEL",
+        test_carriage_return_alias: "CARRIAGE RETURN",
+        test_delete_alias: "DELETE",
+        test_escape_alias: "ESCAPE",
+        test_form_feed_alias: "FORM FEED",
+        test_hts_alias: "HTS",
+        test_character_tabulation_with_justification_alias: "CHARACTER TABULATION WITH JUSTIFICATION",
+    }
 }
diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml
index ea326516fd..6b76f65c8c 100644
--- a/stdlib/Cargo.toml
+++ b/stdlib/Cargo.toml
@@ -57,7 +57,7 @@ sha3 = "0.10.1"
 blake2 = "0.10.4"
 
 ## unicode stuff
-unicode_names2 = "0.5.0"
+unicode_names2 = { workspace = true }
 # TODO: use unic for this; needed for title case:
 # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939
 unicode-casing = "0.1.0"
diff --git a/vm/Cargo.toml b/vm/Cargo.toml
index 487b3eec89..841b5c5191 100644
--- a/vm/Cargo.toml
+++ b/vm/Cargo.toml
@@ -81,7 +81,7 @@ sre-engine = "0.4.1"
 # sre-engine = { path = "../../sre-engine" }
 
 ## unicode stuff
-unicode_names2 = "0.5.1"
+unicode_names2 = { workspace = true }
 # TODO: use unic for this; needed for title case:
 # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939
 unicode-casing = "0.1.0"


Test patch:
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap
new file mode 100644
index 0000000000..eea9786970
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 15,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{8}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap
new file mode 100644
index 0000000000..cf04fd68fd
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 9,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{7}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap
new file mode 100644
index 0000000000..33a84d2291
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 21,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 21,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\r",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap
new file mode 100644
index 0000000000..282706f8b9
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 45,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 45,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{89}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap
new file mode 100644
index 0000000000..06e74d248a
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 12,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 12,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{7f}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap
new file mode 100644
index 0000000000..751456db8c
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 12,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 12,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{1b}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap
new file mode 100644
index 0000000000..b91c10ebe2
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 15,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{c}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/compiler/parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap
new file mode 100644
index 0000000000..73f12074eb
--- /dev/null
+++ b/compiler/parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 9,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{88}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]