Repository: hedyorg/hedy

Issue:
Be able to use Chinese characters as variable names
@Grace666-code points out that it is not possible to use Chinese characters as variables. For example:

```
你好世界 is 你好世界
```


Discussion:
Note: I tried with Cyrillic characters and the same problem happens.

Resolving patch:
diff --git a/app.py b/app.py
--- a/app.py
+++ b/app.py
@@ -48,8 +48,9 @@
     'sw': 'Swahili',
     'hu': 'Magyar',
     'el': 'Ελληνικά',
-    "zh": "简体中文",
-    'cs': 'Čeština'
+    'zh': "简体中文",
+    'cs': 'Čeština',
+    'bn': 'বাংলা'
 }
 
 LEVEL_DEFAULTS = collections.defaultdict(courses.NoSuchDefaults)
diff --git a/hedy.py b/hedy.py
--- a/hedy.py
+++ b/hedy.py
@@ -5,6 +5,7 @@
 import sys
 import utils
 from collections import namedtuple
+import hashlib
 
 
 # Some useful constants
@@ -47,6 +48,21 @@
 #  It will return '' if the invalid command does not contain any known command.
 #
 
+def hash_var(name):
+    import re
+    pattern = re.compile('[^a-zA-Z0-9_]')
+
+    if name in reserved_words or pattern.match(name) != None:
+        # hash "illegal" var names
+        # being reservered keywords
+        # or non-latin vars to comply with Skulpt, which does not implement PEP3131 :(
+        # prepend with v for when hash starts with a number
+
+        hash_object = hashlib.md5(name.encode())
+        return "v" + hash_object.hexdigest()
+    else:
+        return name
+
 def closest_command(invalid_command, known_commands):
     # First search for 100% match of known commands
     min_position = len(invalid_command)
@@ -179,7 +195,6 @@ def list_access(self, args):
         else:
             return listname + '[' + args[1] + ']'
 
-
     # additions Laura, to be checked for higher levels:
     def list_access_var(self, args):
         return args[0]
@@ -197,9 +212,8 @@ def var_access(self, args):
 
     def var(self, args):
         # the var itself (when in an assignment) should be added
-        # if it happens to be a keyword in Python, prefix with _
         name = args[0]
-        return "_" + name if name in reserved_words else name
+        return name
 
     def punctuation(self, args):
         #is never a variable (but should be removed from the tree or it will be seen as one!)
@@ -208,6 +222,83 @@ def punctuation(self, args):
     def __default__(self, args, children, meta):
         return self.filter_ask_assign(children)
 
+class AllAssignmentCommandsHashed(Transformer):
+    # returns a list of variable and list access
+    # so these can be excluded when printing
+
+    #this version returns all hashe var names
+
+    def filter_ask_assign(self, args):
+        ask_assign = []
+        for a in args:
+            # strings (vars remaining in the tree) are added directly
+            if type(a) is str:
+                ask_assign.append(a)
+            #lists are seached further for string members (vars)
+            elif type(a) is list:
+                sub_a_ask_assign = self.filter_ask_assign(a)
+                for sub_a in sub_a_ask_assign:
+                    ask_assign.append(sub_a)
+        return ask_assign
+
+    def for_loop(self, args):
+      # for loop iterator is a var so should be added to the list of vars
+      iterator = str(args[0])
+      iterator_hashed = hash_var(iterator)
+      return [iterator_hashed] + self.filter_ask_assign(args)
+
+    def input(self, args):
+        #return left side of the =
+        return hash_var(args[0])
+
+    def ask(self, args):
+        #try is needed cause in level 1 sk has not variable in front
+        try:
+            return hash_var(args[0])
+        except:
+            return None
+
+    def assign(self, args):
+        return hash_var(args[0])
+
+    def assign_list(self, args):
+        return hash_var(args[0])
+
+    # list access is accessing a variable, so must be escaped
+    # for example we print(dieren[1]) not print('dieren[1]')
+    def list_access(self, args):
+        listname = hash_var(args[0])
+        if args[1] == 'random':
+            return 'random.choice(' + listname + ')'
+        else:
+            return listname + '[' + args[1] + ']'
+
+    # additions Laura, to be checked for higher levels:
+    def list_access_var(self, args):
+        return hash_var(args[0])
+
+    def change_list_item(self, args):
+        return hash_var(args[0])
+
+    def text(self, args):
+        #text never contains a variable
+        return None
+
+    def var_access(self, args):
+        # just accessing (printing) a variable does not count toward the lookup table
+        return None
+
+    def var(self, args):
+        # the var itself (when in an assignment) should be added
+        name = hash_var(args[0])
+        return name
+
+    def punctuation(self, args):
+        #is never a variable (but should be removed from the tree or it will be seen as one!)
+        return None
+
+    def __default__(self, args, children, meta):
+        return self.filter_ask_assign(children)
 
 
 def are_all_arguments_true(args):
@@ -383,7 +474,9 @@ def punctuation(self, args):
         return ''.join([str(c) for c in args])
     def var(self, args):
         name = ''.join(args)
-        return "_" + name if name in reserved_words else name
+        name = args[0]
+        return hash_var(name)
+        # return "_" + name if name in reserved_words else name
     def print(self, args):
         all_arguments_converted = []
         i = 0
@@ -442,8 +535,8 @@ def quoted(s):
 class ConvertToPython_3(ConvertToPython_2):
 
     def var_access(self, args):
-        name = ''.join(args)
-        return "_" + name if name in reserved_words else name
+        name = args[0]
+        return hash_var(name)
 
     def text(self, args):
         return ''.join([str(c) for c in args])
@@ -455,7 +548,7 @@ def print(self, args):
         if unquoted_in_lookup == [] or all(unquoted_in_lookup):
             return "print(" + '+'.join(args) + ')'
         else:
-            # I would like to raise normally but that is caught by the tranformer :(
+            # I would like to raise normally but that is caught by the transformer :(
             return f"HedyException:{args[0]}"
             #raise HedyException('Var Undefined', name=args[0])
 
@@ -475,7 +568,7 @@ def indent(s):
 
 class ConvertToPython_4(ConvertToPython_3):
     def list_access_var(self, args):
-        var = args[0]
+        var = hash_var(args[0])
         if args[2].data == 'random':
             return var + '=random.choice(' + args[1] + ')'
         else:
@@ -643,7 +736,7 @@ def assign_list(self, args):
         return parameter + " = [" + ", ".join(values) + "]"
 
     def list_access_var(self, args):
-        var = args[0]
+        var = hash_var(args[0])
         if not isinstance(args[2], str):
             if args[2].data == 'random':
                 return var + '=random.choice(' + args[1] + ')'
@@ -671,11 +764,12 @@ def assign(self, args):  # TODO: needs to be merged with 6, when 6 is improved t
                 if "'" in value or 'random.choice' in value:  # TODO: should be a call to wrap nonvarargument is quotes!
                     return parameter + " = " + value
                 else:
-                    # FH, June 21 the addition of _true/false is a bit of a hack. cause they are first seen as vars that at reserved words, they egt and _ and we undo that here.
+                    # FH, June 21 the addition of _true/false is a bit of a hack. cause they are first seen as vars that at reserved words, they are then hashed and we undo that here
                     # could/should be fixed in the grammar!
-                    if value == 'true' or value == 'True' or value == '_True':
+
+                    if value == 'true' or value == 'True' or value == hash_var('True') or value == hash_var('true'):
                         return parameter + " = True"
-                    elif value == 'false' or value == 'False' or value == '_False':
+                    elif value == 'false' or value == 'False' or value == hash_var('False') or value == hash_var('false'):
                         return parameter + " = False"
                     else:
                         return parameter + " = '" + value + "'"
@@ -934,7 +1028,7 @@ def get_parser(level, sub):
     if existing and not utils.is_debug_mode():
         return existing
     grammar = create_grammar(level, sub)
-    ret = Lark(grammar)
+    ret = Lark(grammar, regex=True)
     PARSER_CACHE[key] = ret
     return ret
 
@@ -1084,6 +1178,11 @@ def transpile_inner(input_string, level, sub=0):
         abstract_syntaxtree = ExtractAST().transform(program_root)
         lookup_table = AllAssignmentCommands().transform(abstract_syntaxtree)
 
+        #also add hashes to list
+        hashed_lookups = AllAssignmentCommandsHashed().transform(abstract_syntaxtree)
+
+        lookup_table += hashed_lookups
+
     except UnexpectedCharacters as e:
         try:
             location = e.line, e.column


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -17,19 +17,19 @@ def test_extract_default_rounds(self):
     salt = bcrypt.gensalt ().decode ('utf-8')
     self.assertEqual(12, utils.extract_bcrypt_rounds(salt))
 
-  def test_load_yaml_speed(self):
-    n = 50
-    file = 'coursedata/adventures/hu.yaml'
-
-    start = time.time()
-    for _ in range(n):
-      original_data = utils.load_yaml_uncached(file)
-    original_seconds = time.time() - start
-
-    start = time.time()
-    for _ in range(n):
-      cached_data = utils.load_yaml_pickled(file)
-    cached_seconds = time.time() - start
-
-    self.assertEqual(original_data, cached_data)
-    print(f'YAML loading takes {original_seconds / n} seconds, unpickling takes {cached_seconds / n} ({original_seconds / cached_seconds}x faster)')
\ No newline at end of file
+  # def test_load_yaml_speed(self):
+  #   #   n = 50
+  #   #   file = 'coursedata/adventures/hu.yaml'
+  #   #
+  #   #   start = time.time()
+  #   #   for _ in range(n):
+  #   #     original_data = utils.load_yaml_uncached(file)
+  #   #   original_seconds = time.time() - start
+  #   #
+  #   #   start = time.time()
+  #   #   for _ in range(n):
+  #   #     cached_data = utils.load_yaml_pickled(file)
+  #   #   cached_seconds = time.time() - start
+  #   #
+  #   #   self.assertEqual(original_data, cached_data)
+  #   #   print(f'YAML loading takes {original_seconds / n} seconds, unpickling takes {cached_seconds / n} ({original_seconds / cached_seconds}x faster)')
\ No newline at end of file
diff --git a/tests/tests_level_02.py b/tests/tests_level_02.py
--- a/tests/tests_level_02.py
+++ b/tests/tests_level_02.py
@@ -336,6 +336,20 @@ def test_allow_use_of_quotes_in_echo(self):
     self.assertEqual(expected, result.code)
     self.assertEqual(False, result.has_turtle)
 
+  def test_allow_hungarian_vars(self):
+    code = textwrap.dedent("""\
+      állatok is kutya, macska, kenguru
+      print állatok at random""")
+
+    result = hedy.transpile(code, self.level)
+
+    expected = textwrap.dedent("""\
+    v79de0191e90551f058d466c5e8c267ff = ['kutya', 'macska', 'kenguru']
+    print(random.choice(v79de0191e90551f058d466c5e8c267ff))""")
+
+    self.assertEqual(expected, result.code)
+    self.assertEqual(False, result.has_turtle)
+
   def test_two_spaces_after_print(self):
     code = "print        hallo!"
     result = hedy.transpile(code, self.level)
diff --git a/tests/tests_level_03.py b/tests/tests_level_03.py
--- a/tests/tests_level_03.py
+++ b/tests/tests_level_03.py
@@ -118,15 +118,17 @@ def test_name_with_underscore(self):
     self.assertEqual(False, result.has_turtle)
 
   def test_name_that_is_keyword(self):
+    hashed_var = hedy.hash_var("for")
+
     code = textwrap.dedent("""\
     for is Hedy
     print 'ik heet ' for """)
 
     result = hedy.transpile(code, self.level)
 
-    expected = textwrap.dedent("""\
-    _for = 'Hedy'
-    print('ik heet '+_for)""")
+    expected = textwrap.dedent(f"""\
+    {hashed_var} = 'Hedy'
+    print('ik heet '+{hashed_var})""")
 
     self.assertEqual(expected, result.code)
     self.assertEqual(False, result.has_turtle)
@@ -315,6 +317,57 @@ def test_two_spaces_after_print(self):
       self.assertEqual(False, result.has_turtle)
 
 
+  def test_bengali_assign(self):
+    hashed_var = hedy.hash_var("নাম")
+
+    code = textwrap.dedent("""\
+    নাম is হেডি""")
+
+    result = hedy.transpile(code, self.level)
+
+    expected = textwrap.dedent(f"""\
+    {hashed_var} = 'হেডি'""")
+
+    self.assertEqual(expected, result.code)
+    self.assertEqual(False, result.has_turtle)
+
+  def test_bengali_assign_and_use(self):
+    hashed_var = hedy.hash_var("নাম")
+
+    code = textwrap.dedent("""\
+    নাম is হেডি
+    print 'আমার নাম is ' নাম """)
+
+    result = hedy.transpile(code, self.level)
+
+    expected = textwrap.dedent(f"""\
+    {hashed_var} = 'হেডি'
+    print('আমার নাম is '+{hashed_var})""")
+
+    self.assertEqual(expected, result.code)
+
+  def test_chinese_assign_and_use(self):
+    hashed_var = hedy.hash_var("你好世界")
+
+    code = textwrap.dedent("""\
+    你好世界 is 你好世界
+    print 你好世界""")
+
+    result = hedy.transpile(code, self.level)
+
+    expected = textwrap.dedent(f"""\
+    {hashed_var} = '你好世界'
+    print({hashed_var})""")
+
+    self.assertEqual(expected, result.code)
+
+
+
+
+
+
+
+
 
 
 
diff --git a/tests/tests_level_04.py b/tests/tests_level_04.py
--- a/tests/tests_level_04.py
+++ b/tests/tests_level_04.py
@@ -367,3 +367,11 @@ def test_transpile_other(self):
         result = hedy.transpile(code, self.level)
       self.assertEqual(str(context.exception), 'Invalid')
 
+  # def test_list_find_issue(self):
+  #   #'list' object has no attribute 'find'
+  #   # FH dd sept 2021 for later fixing!
+  #   code = textwrap.dedent("""\
+  #     নাম is ask আপনার নাম কি?
+  #     if নাম is হেডি print 'ভালো!' else print 'মন্দ'\"""")
+  #
+  #
diff --git a/tests/tests_level_07.py b/tests/tests_level_07.py
--- a/tests/tests_level_07.py
+++ b/tests/tests_level_07.py
@@ -136,6 +136,8 @@ def test_repeat_with_indent(self):
 
     self.assertEqual(expected, result.code)
 
+
+
   def test_repeat_with_variable_print(self):
     code = textwrap.dedent("""\
     n is 5
@@ -160,6 +162,30 @@ def test_repeat_with_variable_print(self):
 
     self.assertEqual(expected_output, run_code(result))
 
+  def test_repeat_with_non_latin_variable_print(self):
+    code = textwrap.dedent("""\
+    állatok is 5
+    repeat állatok times
+        print 'me wants a cookie!'""")
+
+    result = hedy.transpile(code, self.level)
+
+    expected = textwrap.dedent("""\
+    v79de0191e90551f058d466c5e8c267ff = '5'
+    for i in range(int(v79de0191e90551f058d466c5e8c267ff)):
+      print('me wants a cookie!')""")
+
+    self.assertEqual(expected, result.code)
+
+    expected_output = textwrap.dedent("""\
+    me wants a cookie!
+    me wants a cookie!
+    me wants a cookie!
+    me wants a cookie!
+    me wants a cookie!""")
+
+    self.assertEqual(expected_output, run_code(result))
+
   def test_repeat_nested_in_if(self):
     code = textwrap.dedent("""\
     kleur is groen
diff --git a/tests/tests_level_13.py b/tests/tests_level_13.py
--- a/tests/tests_level_13.py
+++ b/tests/tests_level_13.py
@@ -311,13 +311,13 @@ def test_if_under_else_in_for(self):
 
     def test_bool_true(self):
         code = textwrap.dedent("""\
-    ja is True
-    if ja is True:
-        print('ja')""")
+        ja is True
+        if ja is True:
+            print('ja')""")
         expected = textwrap.dedent("""\
-    ja = True
-    if ja == True:
-      print('ja')""")
+        ja = True
+        if ja == True:
+          print('ja')""")
 
         result = hedy.transpile(code, self.level)
         self.assertEqual(expected, result.code)