Repository: jupyter/nbconvert
Pull request #1957 (2023-03-03 13:50:40)

add mermaidjs 10.2.3
Adds [mermaidjs diagram support](https://mermaid.js.org/), which is the most commonly-available, open source ascii-based diagram tool, supporting a large number of broadly-useful diagrams.

The motivation is that this syntax <code>```mermaid</code> is supported natively by a large number of existing platforms under a consistent syntax, most notably GitHub, which theoretically we track. 

## References

- related [JupyterLab PR](https://github.com/jupyterlab/jupyterlab/pull/14102#issuecomment-1453134883)

## Changes

- [x] adds a CDN URL for mermaidjs
- [x] adds via the recommended `<script type="module"` 
  - > this supports the lazy loading without e.g. requirejs for [95% of browser market share](https://caniuse.com/es6-module)
- [x] only load/start mermaid assets when mermaid actually found

## User-facing changes
- example output with rich display and markdown cells
  - [gist](https://gist.github.com/bollwyvl/b250b5ad584819b68923c97a45d68b22)
- this would _not_ yet be supported in
  - notebook classic
  - bespoke renderers of notebooks, such as the GitLab notebook viewer
  - other jupyter clients (beyond any linked PRs above)

bollwyvl: Will look into the failing tests.
maartenbreddels: Nice to see this happen, we also have support for mermaid in [solara](https://github.com/widgetti/solara/), so we will keep an eye on this and see if we can track the same version number.

Patch:
diff --git a/nbconvert/exporters/html.py b/nbconvert/exporters/html.py
index 03d1462d..30c10b7f 100644
--- a/nbconvert/exporters/html.py
+++ b/nbconvert/exporters/html.py
@@ -121,6 +121,15 @@ class HTMLExporter(TemplateExporter):
         """,
     ).tag(config=True)
 
+    mermaid_js_url = Unicode(
+        "https://cdnjs.cloudflare.com/ajax/libs/mermaid/10.2.3/mermaid.esm.min.mjs",
+        help="""
+        URL to load MermaidJS from.
+
+        Defaults to loading from cdnjs.
+        """,
+    )
+
     jquery_url = Unicode(
         "https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js",
         help="""
@@ -303,6 +312,7 @@ class HTMLExporter(TemplateExporter):
         resources["include_url"] = resources_include_url
         resources["require_js_url"] = self.require_js_url
         resources["mathjax_url"] = self.mathjax_url
+        resources["mermaid_js_url"] = self.mermaid_js_url
         resources["jquery_url"] = self.jquery_url
         resources["jupyter_widgets_base_url"] = self.jupyter_widgets_base_url
         resources["widget_renderer_url"] = self.widget_renderer_url
diff --git a/nbconvert/exporters/latex.py b/nbconvert/exporters/latex.py
index d3ae76ef..a23013cc 100644
--- a/nbconvert/exporters/latex.py
+++ b/nbconvert/exporters/latex.py
@@ -3,12 +3,14 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import os
 
 from traitlets import default
 from traitlets.config import Config
 
 from nbconvert.filters.filter_links import resolve_references
 from nbconvert.filters.highlight import Highlight2Latex
+from nbconvert.filters.pandoc import ConvertExplicitlyRelativePaths
 
 from .templateexporter import TemplateExporter
 
@@ -77,6 +79,16 @@ class LatexExporter(TemplateExporter):
         )
         self.register_filter("highlight_code", highlight_code)
 
+        # Need to make sure explicit relative paths are visible to latex for pdf conversion
+        # https://github.com/jupyter/nbconvert/issues/1998
+        nb_path = resources.get("metadata", {}).get("path") if resources else None
+        texinputs = os.path.abspath(nb_path) if nb_path else os.getcwd()
+        convert_explicitly_relative_paths = self.filters.get(
+            "convert_explicitly_relative_paths",
+            ConvertExplicitlyRelativePaths(texinputs=texinputs, parent=self),
+        )
+        self.register_filter("convert_explicitly_relative_paths", convert_explicitly_relative_paths)
+
         return super().from_notebook_node(nb, resources, **kw)
 
     def _create_environment(self):
diff --git a/nbconvert/filters/markdown_mistune.py b/nbconvert/filters/markdown_mistune.py
index d7501b65..a7a9f05e 100644
--- a/nbconvert/filters/markdown_mistune.py
+++ b/nbconvert/filters/markdown_mistune.py
@@ -176,6 +176,10 @@ class IPythonRenderer(HTMLRenderer):
         """Handle block code."""
         lang = ""
         lexer = None
+
+        if info and info.startswith("mermaid"):
+            return self.block_mermaidjs(code)
+
         if info:
             try:
                 lang = info.strip().split(None, 1)[0]
@@ -190,6 +194,14 @@ class IPythonRenderer(HTMLRenderer):
         formatter = HtmlFormatter()
         return highlight(code, lexer, formatter)
 
+    def block_mermaidjs(self, code, info=None):
+        """Handle mermaid syntax."""
+        return (
+            """<div class="jp-Mermaid"><pre class="mermaid">\n"""
+            f"""{code.strip()}"""
+            """\n</pre></div>"""
+        )
+
     def block_html(self, html):
         """Handle block html."""
         if self.embed_images:
diff --git a/nbconvert/filters/pandoc.py b/nbconvert/filters/pandoc.py
index 09eecfab..165e98ba 100644
--- a/nbconvert/filters/pandoc.py
+++ b/nbconvert/filters/pandoc.py
@@ -1,4 +1,12 @@
-"""Convert between any two formats using pandoc."""
+"""
+Convert between any two formats using pandoc,
+and related filters
+"""
+import os
+
+from pandocfilters import Image, applyJSONFilters  # type:ignore
+
+from nbconvert.utils.base import NbConvertBase
 from nbconvert.utils.pandoc import pandoc
 
 
@@ -23,3 +31,48 @@ def convert_pandoc(source, from_format, to_format, extra_args=None):
         Output as returned by pandoc.
     """
     return pandoc(source, from_format, to_format, extra_args=extra_args)
+
+
+# When converting to pdf, explicitly relative references
+# like "./" and "../" doesn't work with TEXINPUTS.
+# So we need to convert them to absolute paths.
+# See https://github.com/jupyter/nbconvert/issues/1998
+class ConvertExplicitlyRelativePaths(NbConvertBase):
+    """A converter that handles relative path references."""
+
+    def __init__(self, texinputs=None, **kwargs):
+        """Initialize the converter."""
+        # texinputs should be the directory of the notebook file
+        self.nb_dir = os.path.abspath(texinputs) if texinputs else ""
+        self.ancestor_dirs = self.nb_dir.split("/")
+        super().__init__(**kwargs)
+
+    def __call__(self, source):
+        """Invoke the converter."""
+        # If this is not set for some reason, we can't do anything,
+        if self.nb_dir:
+            return applyJSONFilters([self.action], source)
+        return source
+
+    def action(self, key, value, frmt, meta):
+        """Perform the action."""
+        # Convert explicitly relative paths:
+        # ./path -> path  (This should be visible to the latex engine since TEXINPUTS already has .)
+        # ../path -> /abs_path
+        # assuming all relative references are at the start of a given path
+        if key == "Image":
+            # Image seems to have this composition, according to https://github.com/jgm/pandoc-types
+            attr, caption, [filename, typedef] = value
+
+            if filename[:2] == "./":
+                filename = filename[2:]
+            elif filename[:3] == "../":
+                n_up = 0
+                while filename[:3] == "../":
+                    n_up += 1
+                    filename = filename[3:]
+                ancestors = "/".join(self.ancestor_dirs[:-n_up]) + "/"
+                filename = ancestors + filename
+            return Image(attr, caption, [filename, typedef])
+        # If not image, return "no change"
+        return None
diff --git a/share/templates/classic/index.html.j2 b/share/templates/classic/index.html.j2
index ec575b22..87a8578a 100644
--- a/share/templates/classic/index.html.j2
+++ b/share/templates/classic/index.html.j2
@@ -19,6 +19,12 @@
 {%- block html_head_js_requirejs -%}
 <script src="{{ resources.require_js_url }}"></script>
 {%- endblock html_head_js_requirejs -%}
+{%- block html_head_js_mermaidjs -%}
+<script type="module">
+  import mermaid from '{{ resources.mermaid_js_url }}';
+  mermaid.initialize({ startOnLoad: true });
+</script>
+{%- endblock html_head_js_mermaidjs -%}
 {%- endblock html_head_js -%}
 
 {% block jupyter_widgets %}
diff --git a/share/templates/lab/index.html.j2 b/share/templates/lab/index.html.j2
index 4a6bb866..8c566529 100644
--- a/share/templates/lab/index.html.j2
+++ b/share/templates/lab/index.html.j2
@@ -1,5 +1,6 @@
 {%- extends 'base.html.j2' -%}
 {% from 'mathjax.html.j2' import mathjax %}
+{% from 'mermaidjs.html.j2' import mermaid_js %}
 {% from 'jupyter_widgets.html.j2' import jupyter_widgets %}
 
 {%- block header -%}
@@ -149,6 +150,10 @@ body[data-format='mobile'] .jp-OutputArea-child .jp-OutputArea-output {
 {{ mathjax(resources.mathjax_url) }}
 {%- endblock html_head_js_mathjax -%}
 
+{%- block html_head_js_mermaidjs -%}
+{{ mermaid_js(resources.mermaid_js_url) }}
+{%- endblock html_head_js_mermaidjs -%}
+
 {%- block html_head_css -%}
 {%- endblock html_head_css -%}
 
diff --git a/share/templates/lab/mermaidjs.html.j2 b/share/templates/lab/mermaidjs.html.j2
new file mode 100644
index 00000000..06c18479
--- /dev/null
+++ b/share/templates/lab/mermaidjs.html.j2
@@ -0,0 +1,114 @@
+{%- macro mermaid_js(
+url="https://cdnjs.cloudflare.com/ajax/libs/mermaid/10.0.2/mermaid.esm.min.mjs"
+) -%}
+<script type="module">
+  document.addEventListener("DOMContentLoaded", async () => {
+    const diagrams = document.querySelectorAll(".jp-Mermaid > pre.mermaid");
+    // do not load mermaidjs if not needed
+    if (!diagrams.length) {
+      return;
+    }
+    const mermaid = (await import("{{ url }}")).default;
+
+    mermaid.initialize({
+      maxTextSize: 100000,
+      startOnLoad: false,
+      fontFamily: window
+        .getComputedStyle(document.body)
+        .getPropertyValue("--jp-ui-font-family"),
+      theme: document.querySelector("body[data-jp-theme-light='true']")
+        ? "default"
+        : "dark",
+    });
+
+    let _nextMermaidId = 0;
+
+    function makeMermaidImage(svg) {
+      const img = document.createElement('img');
+      const maxWidth = svg.match(/max-width: (\d+)/);
+      if (maxWidth && maxWidth[1]) {
+        const width = parseInt(maxWidth[1]);
+        if (width && !Number.isNaN(width) && Number.isFinite(width)) {
+          img.width = width;
+        }
+      }
+      img.setAttribute('src', `data:image/svg+xml,${encodeURIComponent(svg)}`);
+      return img;
+    }
+
+    async function makeMermaidError(text) {
+      let errorMessage = '';
+      try {
+        await mermaid.parse(text);
+      } catch (err) {
+        errorMessage = `${err}`;
+      }
+
+      const result = document.createElement('details');
+      const summary = document.createElement('summary');
+      const pre = document.createElement('pre');
+      const code = document.createElement('code');
+      code.innerText = text;
+      pre.appendChild(code);
+      summary.appendChild(pre);
+      result.appendChild(summary);
+
+      const warning = document.createElement('pre');
+      warning.innerText = errorMessage;
+      result.appendChild(warning);
+      return result;
+    }
+
+    async function renderOneMarmaid(src) {
+      const id = `jp-mermaid-${_nextMermaidId++}`;
+      const parent = src.parentNode;
+      let raw = src.textContent.trim();
+      const el = document.createElement("div");
+      el.style.visibility = "hidden";
+      document.body.appendChild(el);
+      let result = null;
+      try {
+        const { svg } = await mermaid.render(id, raw, el);
+        result = makeMermaidImage(svg);
+      } catch (err) {
+        parent.classList.add("jp-mod-warning");
+        result = await makeMermaidError(raw);
+      } finally {
+        el.remove();
+      }
+      parent.classList.add("jp-RenderedMermaid");
+      parent.appendChild(result);
+    }
+
+    void Promise.all([...diagrams].map(renderOneMarmaid));
+  });
+</script>
+<style>
+  .jp-RenderedMarkdown .jp-Mermaid:not(.jp-RenderedMermaid) {
+    display: none;
+  }
+  .jp-RenderedMarkdown .jp-RenderedMermaid.jp-mod-warning {
+    width: auto;
+    padding: 10px;
+    border: var(--jp-border-width) solid var(--jp-warn-color2);
+    border-radius: var(--jp-border-radius);
+    color: var(--jp-ui-font-color1);
+    font-size: var(--jp-ui-font-size1);
+    white-space: pre-wrap;
+    word-wrap: break-word;
+  }
+  .jp-RenderedMarkdown .jp-RenderedMermaid.jp-mod-warning details > pre {
+    margin-top: 1em;
+  }
+  .jp-RenderedMarkdown .jp-RenderedMermaid.jp-mod-warning summary {
+    color: var(--jp-warn-color2);
+  }
+  .jp-RenderedMarkdown .jp-RenderedMermaid.jp-mod-warning summary > pre {
+    display: inline-block;
+  }
+  .jp-RenderedMermaid > .mermaid {
+    display: none;
+  }
+</style>
+<!-- End of mermaid configuration -->
+{%- endmacro %}
diff --git a/share/templates/latex/document_contents.tex.j2 b/share/templates/latex/document_contents.tex.j2
index 4dc172b1..61935c69 100644
--- a/share/templates/latex/document_contents.tex.j2
+++ b/share/templates/latex/document_contents.tex.j2
@@ -65,7 +65,7 @@
 
 % Render markdown
 ((* block markdowncell scoped *))
-    ((( cell.source | citation2latex | strip_files_prefix | convert_pandoc('markdown+tex_math_double_backslash', 'json',extra_args=[]) | resolve_references | convert_pandoc('json','latex'))))
+    ((( cell.source | citation2latex | strip_files_prefix | convert_pandoc('markdown+tex_math_double_backslash', 'json',extra_args=[]) | resolve_references | convert_explicitly_relative_paths | convert_pandoc('json','latex'))))
 ((* endblock markdowncell *))
 
 % Don't display unknown types
diff --git a/share/templates/reveal/index.html.j2 b/share/templates/reveal/index.html.j2
index c30af4b4..84cb6528 100644
--- a/share/templates/reveal/index.html.j2
+++ b/share/templates/reveal/index.html.j2
@@ -30,6 +30,12 @@
 {%- block html_head_js_requirejs -%}
 <script src="{{ resources.require_js_url }}"></script>
 {%- endblock html_head_js_requirejs -%}
+{%- block html_head_js_mermaidjs -%}
+<script type="module">
+  import mermaid from '{{ resources.mermaid_js_url }}';
+  mermaid.initialize({ startOnLoad: true });
+</script>
+{%- endblock html_head_js_mermaidjs -%}
 {%- endblock html_head_js -%}
 
 {% block jupyter_widgets %}


Test patch:
diff --git a/nbconvert/filters/tests/test_markdown.py b/nbconvert/filters/tests/test_markdown.py
index a6643282..c271969e 100644
--- a/nbconvert/filters/tests/test_markdown.py
+++ b/nbconvert/filters/tests/test_markdown.py
@@ -247,6 +247,19 @@ i.e. the $i^{th}$""",
                 tokens[index],
             )
 
+    def test_mermaid_markdown(self):
+        code = """flowchart LR
+            chicken --> egg --> chicken"""
+        case = f"""```mermaid\n  {code}\n```"""
+
+        output_check = (
+            """<div class="jp-Mermaid"><pre class="mermaid">\n"""
+            f"""{code.strip()}"""
+            """\n</pre></div>"""
+        )
+
+        self._try_markdown(markdown2html, case, output_check)
+
     def _try_markdown(self, method, test, tokens):
         results = method(test)
         if isinstance(tokens, (str,)):
diff --git a/nbconvert/filters/tests/test_pandoc.py b/nbconvert/filters/tests/test_pandoc.py
new file mode 100644
index 00000000..21d9bb9c
--- /dev/null
+++ b/nbconvert/filters/tests/test_pandoc.py
@@ -0,0 +1,59 @@
+"""
+Module with tests for Pandoc filters
+"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import json
+
+from ...tests.base import TestsBase
+from ...tests.utils import onlyif_cmds_exist
+from ..pandoc import ConvertExplicitlyRelativePaths, convert_pandoc
+
+
+class TestPandocFilters(TestsBase):
+    @onlyif_cmds_exist("pandoc")
+    def test_convert_explicitly_relative_paths(self):
+        """
+        Do the image links in a markdown file located in dir get processed correctly?
+        """
+        inp_dir = "/home/user/src"
+        fltr = ConvertExplicitlyRelativePaths(texinputs=inp_dir)
+
+        # pairs of input, expected
+        tests = {
+            # TEXINPUTS is enough, abs_path not needed
+            "im.png": "im.png",
+            "./im.png": "im.png",
+            "./images/im.png": "images/im.png",
+            # TEXINPUTS is not enough, abs_path needed
+            "../im.png": "/home/user/im.png",
+            "../images/im.png": "/home/user/images/im.png",
+            "../../images/im.png": "/home/images/im.png",
+        }
+
+        # this shouldn't be modified by the filter
+        # since it is a code block inside markdown,
+        # not an image link itself
+        fake = """
+        ```
+        \\includegraphics{../fake.png}
+        ```
+        """
+
+        # convert to markdown image
+        def foo(filename):
+            return f"![]({filename})"
+
+        # create input markdown and convert to pandoc json
+        inp = convert_pandoc(fake + "\n".join(map(foo, tests.keys())), "markdown", "json")
+        expected = convert_pandoc(fake + "\n".join(map(foo, tests.values())), "markdown", "json")
+        # Do this to fix string formatting
+        expected = json.dumps(json.loads(expected))
+        self.assertEqual(expected, fltr(inp))
+
+    def test_convert_explicitly_relative_paths_no_texinputs(self):
+        # no texinputs should lead to just returning
+        fltr = ConvertExplicitlyRelativePaths(texinputs="")
+        self.assertEqual("test", fltr("test"))