Repository: hedyorg/hedy

Issue:
Add tests for all adventure, start and demo programs
Sometimes little errors happen in the code we ship in various examples, f.e. a typo, or when we change the grammar. Going forward, these should all be tested too. 

This entails more or less:

* Parse adventure yamls and extract all code between ```'s
* Parse level defaults yamls and extract all start code 
* Parse level defaults yamls and extract all demo code in all commands

Parse all these as a unittest and see if they all parse (in a later stage we could even define outcome to test against, but for now just seeing that they parse is ok)

We also need to think about proper reporting, so not just a failing test but "story adventure fails in Hungarian" as output.




Discussion:

Resolving patch:
diff --git a/utils.py b/utils.py
--- a/utils.py
+++ b/utils.py
@@ -9,7 +9,10 @@
 import random
 from ruamel import yaml
 from website import querylog
-
+import commonmark
+commonmark_parser = commonmark.Parser ()
+commonmark_renderer = commonmark.HtmlRenderer ()
+from bs4 import BeautifulSoup
 
 IS_WINDOWS = os.name == 'nt'
 
@@ -234,3 +237,10 @@ def mstoisostring(date):
 # https://stackoverflow.com/a/2257449
 def random_id_generator(size=6, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits):
     return ''.join (random.choice (chars) for _ in range (size))
+
+# This function takes a markdown string and returns a list with each of the HTML elements obtained
+# by rendering the markdown into HTML.
+def markdown_to_html_tags (markdown):
+    _html = commonmark_renderer.render(commonmark_parser.parse (markdown))
+    soup = BeautifulSoup(_html, 'html.parser')
+    return soup.find_all ()


Test patch:
diff --git a/tests/tests_adventures.py b/tests/tests_adventures.py
new file mode 100644
--- /dev/null
+++ b/tests/tests_adventures.py
@@ -0,0 +1,60 @@
+import os
+import re
+import utils
+import hedy
+import unittest
+
+# Set the current directory to the root Hedy folder
+os.chdir(os.path.join (os.getcwd (), __file__.replace (os.path.basename (__file__), '')))
+
+path = '../coursedata/adventures'
+files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith ('.yaml')]
+
+def check_code(filename, level, field_name, code, adventure_name):
+    # We ignore empty code snippets or those of length 1
+    if len(code) == 0:
+        return True
+    try:
+        hedy.transpile(code, int(level))
+    except Exception as E:
+        #shorter file name for beter readability
+        filename_shorter = filename.split("/")[3]
+        language = filename_shorter.split(".")[0]
+        if E.args[0] != 'Has Blanks':  # code with blanks is ok!
+            error = f'{language}: adventure {adventure_name} - level #{level} - {field_name}. Error: {E.args[0]}'
+            print(error)
+            return error
+    return True
+
+class TestsAdventurePrograms(unittest.TestCase):
+
+    def test_adventure_snippets(self):
+        adventure_fails = []
+
+        for f in files:
+            f = os.path.join (path, f)
+            yaml = utils.load_yaml_uncached (f)
+
+            for adventure in yaml ['adventures'].values ():
+                for level_number in adventure['levels']:
+                    level = adventure['levels'][level_number]
+                    adventure_name = adventure['name']
+
+                    code_snippet_counter = 0
+                    # code snippets inside story_text
+                    for tag in utils.markdown_to_html_tags(level['story_text']):
+                        if tag.name != 'pre' or not tag.contents[0]:
+                            continue
+                        code_snippet_counter += 1
+                        code = tag.contents[0].contents[0]
+
+                        result = check_code(f, level_number, 'story_text code snippet #' + str (code_snippet_counter), code, adventure_name)
+                        if result != True:
+                            adventure_fails.append(result)
+
+                    # start_code
+                    result = check_code(f, level_number, 'start_code', level ['start_code'], adventure_name)
+                    if result != True:
+                        adventure_fails.append(result)
+
+        self.assertEqual(0, len(adventure_fails))
diff --git a/tests/tests_defaults.py b/tests/tests_defaults.py
new file mode 100644
--- /dev/null
+++ b/tests/tests_defaults.py
@@ -0,0 +1,60 @@
+import os
+import re
+import utils
+import hedy
+import unittest
+
+# Set the current directory to the root Hedy folder
+os.chdir(os.path.join (os.getcwd (), __file__.replace (os.path.basename (__file__), '')))
+
+path = '../coursedata/level-defaults'
+files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith ('.yaml')]
+
+def check_code(filename, level, field_name, code):
+    # We ignore empty code snippets
+    if len(code) == 0:
+        return True
+    try:
+        hedy.transpile(code, int(level))
+    except Exception as E:
+        if E.args[0] != 'Has Blanks':  # code with blanks is ok!
+            filename_shorter = filename.split("/")[3]
+            language = filename_shorter.split(".")[0]
+            error = f'{language}: level #{level} - {field_name}. Error: {E.args[0]}'
+            # We print the error for readability, since otherwise they get accumulated on a long list
+            print (error)
+            return error
+    return True
+
+class TestsLevelDefaultsPrograms(unittest.TestCase):
+
+    def test_level_defaults_snippets(self):
+        level_default_fails = []
+
+        for file in files:
+            file = os.path.join (path, file)
+            yaml = utils.load_yaml_uncached (file)
+
+            for level in yaml:
+                # start_code
+                result = check_code(file, level, 'start_code', yaml [level] ['start_code'])
+                if result != True:
+                    level_default_fails.append(result)
+                # commands.k.demo_code
+                for k, command in enumerate(yaml[level]['commands']):
+                    command_text_short = command['explanation'][0:10]
+                    result = check_code(file, level, 'command ' + command_text_short + ' demo_code', command['demo_code'])
+                    if result != True:
+                        level_default_fails.append(result)
+
+                # code snippets inside intro_text
+                code_snippet_counter = 0
+                for tag in utils.markdown_to_html_tags (yaml [level] ['intro_text']):
+                    if tag.name != 'pre' or not tag.contents [0]:
+                        continue
+                    code_snippet_counter += 1
+                    result = check_code(file, level, 'intro_text snippet #' + str (code_snippet_counter), tag.contents [0].contents [0])
+                    if result != True:
+                        level_default_fails.append(result)
+
+        self.assertEqual(0, len(level_default_fails))