Repository: inducer/relate

Issue:
lxml dependency, HTML parse during markup render
Hi @dzhuang, I just spotted that 

* we've picked up a dependency on lxml, and
* each markup render involves a full HTML parse and re-stringify, just to get rid of some `<pre>` tags.

I'm not a fan of either of these two points, and I would like to see them fixed. Markup rendering is already way too slow. And lxml is not lightweight at all, requiring libxml2 and libxslt2. I've tried hard to keep Relate's dependencies to Python-only. On top of that, lxml is a bad tool for this job, with non-WHATWG-compliant HTML5 parsing. (`html5lib` exists for a reason)


Discussion:
I'll try to fix this.

Resolving patch:
diff --git a/course/content.py b/course/content.py
--- a/course/content.py
+++ b/course/content.py
@@ -912,18 +912,6 @@ def expand_markup(
     return text
 
 
-def unwrap_relate_tmp_pre_tag(html_string):
-    # type: (Text) -> (Text)
-
-    from lxml.html import fromstring, tostring
-    tree = fromstring(html_string)
-
-    for node in tree.iterdescendants("pre"):
-        if "relate_tmp_pre" in node.attrib.get("class", ""):
-            node.drop_tag()
-    return tostring(tree, encoding="unicode")
-
-
 def markup_to_html(
         course,  # type: Optional[Course]
         repo,  # type: Repo_ish
@@ -976,11 +964,13 @@ def markup_to_html(
         return ""
 
     from course.mdx_mathjax import MathJaxExtension
+    from course.utils import NBConvertExtension
     import markdown
 
     extensions = [
         LinkFixerExtension(course, commit_sha, reverse_func=reverse_func),
         MathJaxExtension(),
+        NBConvertExtension(),
         "markdown.extensions.extra",
     ]
 
@@ -998,9 +988,6 @@ def markup_to_html(
         extensions=extensions,
         output_format="html5")
 
-    if result.strip():
-        result = unwrap_relate_tmp_pre_tag(result)
-
     assert isinstance(result, six.text_type)
     if cache_key is not None:
         def_cache.add(cache_key, result, None)
diff --git a/course/utils.py b/course/utils.py
--- a/course/utils.py
+++ b/course/utils.py
@@ -28,6 +28,7 @@
 
 import six
 import datetime  # noqa
+import markdown
 
 from django.shortcuts import (  # noqa
         render, get_object_or_404)
@@ -1322,6 +1323,22 @@ def _render_notebook_from_source(
 
         return body
 
+
+NBCONVERT_PRE_OPEN_RE = re.compile(r"<pre\s*>\s*<relate_ipynb\s*>")
+NBCONVERT_PRE_CLOSE_RE = re.compile(r"</relate_ipynb\s*>\s*</pre\s*>")
+
+
+class NBConvertHTMLPostprocessor(markdown.postprocessors.Postprocessor):
+    def run(self, text):
+        text = NBCONVERT_PRE_OPEN_RE.sub("", text)
+        text = NBCONVERT_PRE_CLOSE_RE.sub("", text)
+        return text
+
+
+class NBConvertExtension(markdown.Extension):
+    def extendMarkdown(self, md, md_globals):  # noqa
+        md.postprocessors['relate_nbconvert'] = NBConvertHTMLPostprocessor(md)
+
 # }}}
 
 # vim: foldmethod=marker
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -27,7 +27,6 @@
           "markdown",
           "dulwich",
           "pyyaml",
-          "lxml",
           "nbconvert>=5.2.1",
           "pymbolic",
           "sympy",


Test patch:
diff --git a/tests/test_content.py b/tests/test_content.py
--- a/tests/test_content.py
+++ b/tests/test_content.py
@@ -240,6 +240,7 @@ def test_view_flow_with_cache_improperly_configured(self):
 CODE_CELL_IN_STR_PATTERN = '<div class="prompt input_prompt">In[%s]:</div>'
 CODE_CELL_PRINT_STR1 = "This is function1"
 CODE_CELL_PRINT_STR2 = "This is function2"
+RELATE_IPYNB_CONVERT_PRE_WRAPPER_TAG_NAME = "relate_ipynb"
 
 
 def strip_nbsp(s):
@@ -261,6 +262,7 @@ def assertIsValidNbConversion(self, response):  # noqa
         self.assertNotContains(response, "```")
         self.assertNotContains(response, "# First Title of Test NoteBook")
         self.assertNotContains(response, "# Second Title of Test NoteBook")
+        self.assertNotContains(response, RELATE_IPYNB_CONVERT_PRE_WRAPPER_TAG_NAME)
 
     def setUp(self):
         super(NbconvertRenderTestMixin, self).setUp()