Repository: beeware/briefcase

Issue:
Non-latin formal app names cause problems
If you use non-latin characters (e.g., `学口算`, or anything outside the ASCII set) in the formal app name, `briefcase new` fails. Similar problems occur if you manually define `pyproject.toml`, and then run `briefcase create`.

This is because briefcase tries to create a "Class Name", as well as other artefacts by processing the formal app name; these conversions fail with non-latin names.

**To Reproduce**
Steps to reproduce the behavior:
1. Run `briefcase new`
2. Specify an app name of `学口算`.

or
1. Run `briefcase new`
2. Modify `pyproject.toml` so that the formal app name is `学口算`
3. Run `briefcase create`

**Expected behavior**

The formal name is human-readable, so any unicode string should be valid.



Comments:

Patch:
diff --git a/changes/612.bugfix.rst b/changes/612.bugfix.rst
new file mode 100644
index 00000000..ca7b1ed2
--- /dev/null
+++ b/changes/612.bugfix.rst
@@ -0,0 +1,1 @@
+When the formal name uses non-Latin characters, the suggested Class and App names are now valid.
diff --git a/src/briefcase/commands/new.py b/src/briefcase/commands/new.py
index 451abec7..bed4f8dd 100644
--- a/src/briefcase/commands/new.py
+++ b/src/briefcase/commands/new.py
@@ -1,6 +1,7 @@
 import os
 import re
 import subprocess
+import unicodedata
 from email.utils import parseaddr
 from typing import Optional
 from urllib.parse import urlparse
@@ -79,9 +80,36 @@ class NewCommand(BaseCommand):
         :param formal_name: The formal name
         :returns: The app's class name
         """
-        class_name = re.sub('[^0-9a-zA-Z_]+', '', formal_name)
-        if class_name[0].isdigit():
+        # Identifiers (including class names) can be unicode.
+        # https://docs.python.org/3/reference/lexical_analysis.html#identifiers
+        xid_start = {
+            "Lu",  # uppercase letters
+            "Ll",  # lowercase letters
+            "Lt",  # titlecase letters
+            "Lm",  # modifier letters
+            "Lo",  # other letters
+            "Nl",  # letter numbers
+        }
+        xid_continue = xid_start.union({
+            "Mn",  # nonspacing marks
+            "Mc",  # spacing combining marks
+            "Nd",  # decimal number
+            "Pc",  # connector punctuations
+        })
+
+        # Normalize to NFKC form, then remove any character that isn't
+        # in the allowed categories, or is the underscore character
+        class_name = ''.join(
+            ch for ch in unicodedata.normalize('NFKC', formal_name)
+            if unicodedata.category(ch) in xid_continue
+            or ch in {'_'}
+        )
+
+        # If the first character isn't in the 'start' character set,
+        # and it isn't already an underscore, prepend an underscore.
+        if unicodedata.category(class_name[0]) not in xid_start and class_name[0] != '_':
             class_name = '_' + class_name
+
         return class_name
 
     def make_app_name(self, formal_name):
@@ -91,7 +119,14 @@ class NewCommand(BaseCommand):
         :param formal_name: The formal name
         :returns: The candidate app name
         """
-        return re.sub('[^0-9a-zA-Z_]+', '', formal_name).lstrip('_').lower()
+        normalized = unicodedata.normalize('NFKD', formal_name)
+        stripped = re.sub('[^0-9a-zA-Z_]+', '', normalized).lstrip('_')
+        if stripped:
+            return stripped.lower()
+        else:
+            # If stripping removes all the content,
+            # use a dummy app name as the suggestion.
+            return 'myapp'
 
     def validate_app_name(self, candidate):
         """


Test patch:
diff --git a/tests/commands/new/test_make_app_name.py b/tests/commands/new/test_make_app_name.py
index 0e314413..84a39e0d 100644
--- a/tests/commands/new/test_make_app_name.py
+++ b/tests/commands/new/test_make_app_name.py
@@ -8,6 +8,13 @@ import pytest
         ('Hello World!', 'helloworld'),
         ('Hello! World', 'helloworld'),
         ('Hello-World', 'helloworld'),
+
+        # Internationalized names that can be unicode-simplified
+        ('Hallo Vögel', 'hallovogel'),
+        ('Bonjour Garçon', 'bonjourgarcon'),
+
+        # Internationalized names that cannot be unicode-simplified
+        ('你好 世界', 'myapp'),
     ]
 )
 def test_make_app_name(new_command, formal_name, candidate):
diff --git a/tests/commands/new/test_make_class_name.py b/tests/commands/new/test_make_class_name.py
index 79d2eee7..4f08966d 100644
--- a/tests/commands/new/test_make_class_name.py
+++ b/tests/commands/new/test_make_class_name.py
@@ -4,12 +4,38 @@ import pytest
 @pytest.mark.parametrize(
     'formal_name, candidate',
     [
+        # Some simple cases
         ('Hello World', 'HelloWorld'),
         ('Hello World!', 'HelloWorld'),
         ('Hello! World', 'HelloWorld'),
         ('Hello_World', 'Hello_World'),
         ('Hello-World', 'HelloWorld'),
-        ('24 Jump Street', '_24JumpStreet'),
+
+        # Startint with a number
+        ('24 Jump Street', '_24JumpStreet'),  # Unicode category Nd
+        # Starting with an underscore
+        ('Hello_World', 'Hello_World'),
+        ('_Hello_World', '_Hello_World'),
+
+        # Unicode names
+        ('你好 世界', '你好世界'),
+        ('Hallo Vögel', 'HalloVögel'),
+        ('Bonjour Garçon', 'BonjourGarçon'),
+
+        # Unicode codepoints that can be at the start of an identifier
+        ('\u02EC World', '\u02ECWorld'),  # Unicode category Lm
+        ('\u3006 World', '\u3006World'),  # Unicode category Lo
+        ('\u3021 World', '\u3021World'),  # Unicode category Nl
+        # ('\u2118 World', '\u2118World'),  # in Other_ID_Start
+
+        # Unicode codepoints that cannot be at the start of an identifer
+        ('\u20E1 World', '_\u20E1World'),  # Unicode Category Mn
+        ('\u0903 World', '_\u0903World'),  # Unicode Category Mc
+        ('\u2040 World', '_\u2040World'),  # Category Pc
+        # ('\u00B7 World', '_\u00B7World'),  # in Other_ID_Continue
+
+        # Characters that are converted by NFKC normalization
+        ('\u2135 World', '\u05d0World'),  # Unicode category Lo
     ]
 )
 def test_make_class_name(new_command, formal_name, candidate):