Repository: oppia/oppia

Issue:
Create a function in html_cleaner.py, with tests, that extracts all components from an HTML string.
We want to understand the structure of the HTML strings stored in the content, feedback and fallbacks of states (and in other places where HTML is used, e.g. the multiple choice interaction). For example, one of the things we would like to do is to derive a list of all image components used in the exploration, which involves examining the HTML strings and extracting the image components from them.

The aim of this issue is therefore to write a function in html_cleaner.py (which we might want to rename to html_services.py) called get_rte_components(html_string). This function would return a list of dicts, each representing an RTE component (so, with keys 'id' and 'customization_args') that could then be analyzed further. Tests should also be written for this function.

/cc @juansaba for info, since this relates to work on images.


Discussion:
@seanlip Sorry for the naive question, but can you elaborate on what you mean by HTML strings?
@juansaba These just refer to the contents of rich-text editors (which may include image, video, etc. components). This content is stored as (a strict subset of) HTML, with custom tags for the components.

Resolving patch:
diff --git a/appengine_config.py b/appengine_config.py
--- a/appengine_config.py
+++ b/appengine_config.py
@@ -80,6 +80,7 @@ def save(self):
     os.path.join(ROOT_PATH, 'third_party', 'graphy-1.0.0'),
     os.path.join(ROOT_PATH, 'third_party', 'requests-2.10.0'),
     os.path.join(ROOT_PATH, 'third_party', 'simplejson-3.7.1'),
+    os.path.join(ROOT_PATH, 'third_party', 'beautifulsoup4-4.6.0'),
 ]
 
 # In the GAE production environment, numpy is automatically provided, and
diff --git a/core/domain/html_cleaner.py b/core/domain/html_cleaner.py
--- a/core/domain/html_cleaner.py
+++ b/core/domain/html_cleaner.py
@@ -16,10 +16,13 @@
 
 """HTML sanitizing service."""
 
+import HTMLParser
+import json
 import logging
 import urlparse
 
 import bleach
+import bs4
 
 from core.domain import rte_component_registry
 
@@ -107,3 +110,37 @@ def strip_html_tags(html):
         stripped out.
     """
     return bleach.clean(html, tags=[], attributes={}, strip=True)
+
+
+def get_rte_components(html_string):
+    """Extracts the RTE components from an HTML string.
+
+    Args:
+        html: str. An HTML string.
+
+    Returns:
+        list(dict). A list of dictionaries, each representing an RTE component.
+        Each dict in the list contains:
+        - id: str. The name of the component, i.e. 'oppia-noninteractive-link'.
+        - customization_args: dict. Customization arg specs for the component.
+    """
+    parser = HTMLParser.HTMLParser()
+    components = []
+    soup = bs4.BeautifulSoup(html_string, 'html.parser')
+    oppia_custom_tag_attrs = (
+        rte_component_registry.Registry.get_tag_list_with_attrs())
+    for tag_name in oppia_custom_tag_attrs:
+        component_tags = soup.find_all(tag_name)
+        for component_tag in component_tags:
+            component = {'id': tag_name}
+            customization_args = {}
+            for attr in oppia_custom_tag_attrs[tag_name]:
+                # Unescape special HTML characters such as '&quot;'
+                attr_val = parser.unescape(component_tag[attr])
+                # Adds escapes so that things like '\frac' aren't
+                # interpreted as special characters.
+                attr_val = attr_val.encode('unicode_escape')
+                customization_args[attr] = json.loads(attr_val)
+            component['customization_args'] = customization_args
+            components.append(component)
+    return components
diff --git a/scripts/pre_commit_linter.py b/scripts/pre_commit_linter.py
--- a/scripts/pre_commit_linter.py
+++ b/scripts/pre_commit_linter.py
@@ -157,6 +157,7 @@
     os.path.join(_PARENT_DIR, 'oppia_tools', 'selenium-2.53.2'),
     os.path.join('third_party', 'gae-pipeline-1.9.17.0'),
     os.path.join('third_party', 'bleach-1.2.2'),
+    os.path.join('third_party', 'beautifulsoup4-4.6.0'),
     os.path.join('third_party', 'gae-mapreduce-1.9.17.0'),
 ]
 for path in _PATHS_TO_INSERT:


Test patch:
diff --git a/core/domain/html_cleaner_test.py b/core/domain/html_cleaner_test.py
--- a/core/domain/html_cleaner_test.py
+++ b/core/domain/html_cleaner_test.py
@@ -126,3 +126,57 @@ def test_strip_html_tags(self):
 
         for datum in test_data:
             self.assertEqual(html_cleaner.strip_html_tags(datum[0]), datum[1])
+
+
+class RteComponentExtractorUnitTests(test_utils.GenericTestBase):
+    '''Test the RTE component extractor.'''
+
+    def test_get_rte_components(self):
+        test_data = (
+            '<p>Test text&nbsp;'
+            '<oppia-noninteractive-math '
+            'raw_latex-with-value="&amp;quot;\\frac{x}{y}&amp;quot;">'
+            '</oppia-noninteractive-math></p><p>&nbsp;'
+            '<oppia-noninteractive-link '
+            'text-with-value="&amp;quot;Link&amp;quot;" '
+            'url-with-value="&amp;quot;https://www.example.com&amp;quot;">'
+            '</oppia-noninteractive-link>.</p>'
+            '<p>Video</p>'
+            '<p><oppia-noninteractive-video autoplay-with-value="false" '
+            'end-with-value="0" start-with-value="0" '
+            'video_id-with-value="&amp;quot;'
+            'https://www.youtube.com/watch?v=Ntcw0H0hwPU&amp;quot;">'
+            '</oppia-noninteractive-video><br></p>'
+        )
+
+        expected_components = [
+            {
+                'customization_args': {
+                    'text-with-value': u'Link',
+                    'url-with-value': u'https://www.example.com'},
+                'id': 'oppia-noninteractive-link'
+            },
+            {
+                'customization_args': {
+                    'start-with-value': 0,
+                    'end-with-value': 0,
+                    'video_id-with-value': (
+                        u'https://www.youtube.com/watch?'
+                        u'v=Ntcw0H0hwPU'),
+                    'autoplay-with-value': False
+                },
+                'id': 'oppia-noninteractive-video'
+            },
+            {
+                'customization_args': {
+                    'raw_latex-with-value': u'\\frac{x}{y}'
+                },
+                'id': 'oppia-noninteractive-math'
+            }
+        ]
+
+        components = html_cleaner.get_rte_components(test_data)
+
+        self.assertEqual(len(components), len(expected_components))
+        for component in components:
+            self.assertIn(component, expected_components)
diff --git a/core/tests/gae_suite.py b/core/tests/gae_suite.py
--- a/core/tests/gae_suite.py
+++ b/core/tests/gae_suite.py
@@ -55,6 +55,7 @@
     os.path.join(THIRD_PARTY_DIR, 'html5lib-python-0.95'),
     os.path.join(THIRD_PARTY_DIR, 'requests-2.10.0'),
     os.path.join(THIRD_PARTY_DIR, 'simplejson-3.7.1'),
+    os.path.join(THIRD_PARTY_DIR, 'beautifulsoup4-4.6.0'),
 ]
 
 _PARSER = argparse.ArgumentParser()