Repository: getnikola/nikola

Issue:
Markdown compiler has performance issues
As noted in https://getnikola.com/blog/markdown-can-affect-performance.html, the markdown compiler has very poor performance when compiling a large number of markdown files.


Discussion:

Resolving patch:
diff --git a/nikola/plugins/compile/markdown/__init__.py b/nikola/plugins/compile/markdown/__init__.py
--- a/nikola/plugins/compile/markdown/__init__.py
+++ b/nikola/plugins/compile/markdown/__init__.py
@@ -30,11 +30,12 @@
 
 import io
 import os
+import threading
 
 try:
-    from markdown import markdown
+    from markdown import Markdown
 except ImportError:
-    markdown = None  # NOQA
+    Markdown = None  # NOQA
     nikola_extension = None
     gist_extension = None
     podcast_extension = None
@@ -43,40 +44,60 @@
 from nikola.utils import makedirs, req_missing, write_metadata
 
 
+class ThreadLocalMarkdown(threading.local):
+    """Convert Markdown to HTML using per-thread Markdown objects.
+
+    See discussion in #2661.
+    """
+
+    def __init__(self, extensions):
+        """Create a Markdown instance."""
+        self.markdown = Markdown(extensions=extensions, output_format="html5")
+
+    def convert(self, data):
+        """Convert data to HTML and reset internal state."""
+        result = self.markdown.convert(data)
+        self.markdown.reset()
+        return result
+
+
 class CompileMarkdown(PageCompiler):
     """Compile Markdown into HTML."""
 
     name = "markdown"
     friendly_name = "Markdown"
     demote_headers = True
-    extensions = []
     site = None
 
     def set_site(self, site):
         """Set Nikola site."""
         super(CompileMarkdown, self).set_site(site)
         self.config_dependencies = []
+        extensions = []
         for plugin_info in self.get_compiler_extensions():
             self.config_dependencies.append(plugin_info.name)
-            self.extensions.append(plugin_info.plugin_object)
+            extensions.append(plugin_info.plugin_object)
             plugin_info.plugin_object.short_help = plugin_info.description
 
-        self.config_dependencies.append(str(sorted(site.config.get("MARKDOWN_EXTENSIONS"))))
+        site_extensions = self.site.config.get("MARKDOWN_EXTENSIONS")
+        self.config_dependencies.append(str(sorted(site_extensions)))
+        extensions.extend(site_extensions)
+        if Markdown is not None:
+            self.converter = ThreadLocalMarkdown(extensions)
 
     def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
         """Compile Markdown into HTML strings."""
-        if markdown is None:
+        if Markdown is None:
             req_missing(['markdown'], 'build this site (compile Markdown)')
-        self.extensions += self.site.config.get("MARKDOWN_EXTENSIONS")
         if not is_two_file:
             _, data = self.split_metadata(data)
-        output = markdown(data, self.extensions, output_format="html5")
+        output = self.converter.convert(data)
         output, shortcode_deps = self.site.apply_shortcodes(output, filename=source_path, with_dependencies=True, extra_context={'post': post})
         return output, shortcode_deps
 
     def compile(self, source, dest, is_two_file=True, post=None, lang=None):
         """Compile the source file into HTML and save as dest."""
-        if markdown is None:
+        if Markdown is None:
             req_missing(['markdown'], 'build this site (compile Markdown)')
         makedirs(os.path.dirname(dest))
         with io.open(dest, "w+", encoding="utf8") as out_file:


Test patch: