Repository: jupyter/nbconvert
Pull request #2167 (2024-07-30 14:56:57)

Allow including `text/x-rst` outputs in rst conversion, transition away from `text/restructuredtext`
Outputs in a notebook can include arbitrary mimetypes. From IPython this can be done with `_repr_mimebundle_` method, for instance. Most of the export formats that nbconvert knows about have a corresponding template block for output already in that format, e.g. `data_latex` or `data_html`. However, there's no template block for rst output.

I've partially generalised this, by adding a `data_native` block to the display priority template rather than a specific `data_rst`. This is meant to hold output in the exporter's native format (its `output_mimetype`), iff there isn't already a dedicated block for that format. I admit that's not the tidiest definition, but it seemed the best way to avoid breaking anything.

The exporter class then has to customise `filter_data_type` to accept its additional mimetype. I've done this in `RSTExporter`. It could be generalised in `TemplateExporter`, but that runs more risk of surprising behaviour for subclasses, which are unlikely to have a `data_native` block in their templates.

takluyver: The remaining test failure is because [a pip bug](https://github.com/pypa/pip/issues/12781) broke installing pandocfilters 1.4.1. That's already been fixed in pip 24.2, but the environment that hatch creates still seeems to get pip 24.1. :face_exhaling: 
takluyver: I've got all the CI passing now.

Patch:
diff --git a/nbconvert/exporters/rst.py b/nbconvert/exporters/rst.py
index 23061f14..07984d1e 100644
--- a/nbconvert/exporters/rst.py
+++ b/nbconvert/exporters/rst.py
@@ -6,6 +6,7 @@
 from traitlets import default
 from traitlets.config import Config
 
+from ..filters import DataTypeFilter
 from .templateexporter import TemplateExporter
 
 
@@ -22,9 +23,24 @@ class RSTExporter(TemplateExporter):
     def _template_name_default(self):
         return "rst"
 
-    output_mimetype = "text/restructuredtext"
+    @default("raw_mimetypes")
+    def _raw_mimetypes_default(self):
+        # Up to summer 2024, nbconvert had a mistaken output_mimetype.
+        # Listing that as an extra option here maintains compatibility for
+        # notebooks with raw cells marked as that mimetype.
+        return [self.output_mimetype, "text/restructuredtext", ""]
+
+    output_mimetype = "text/x-rst"
     export_from_notebook = "reST"
 
+    def default_filters(self):
+        """Override filter_data_type to use native rst outputs"""
+        dtf = DataTypeFilter()
+        dtf.display_data_priority = [self.output_mimetype, *dtf.display_data_priority]
+        filters = dict(super().default_filters())
+        filters["filter_data_type"] = dtf
+        return filters.items()
+
     @property
     def default_config(self):
         c = Config(
diff --git a/nbconvert/exporters/templateexporter.py b/nbconvert/exporters/templateexporter.py
index c09e6583..42220cbd 100644
--- a/nbconvert/exporters/templateexporter.py
+++ b/nbconvert/exporters/templateexporter.py
@@ -407,6 +407,7 @@ class TemplateExporter(Exporter):
         """
         nb_copy, resources = super().from_notebook_node(nb, resources, **kw)
         resources.setdefault("raw_mimetypes", self.raw_mimetypes)
+        resources.setdefault("output_mimetype", self.output_mimetype)
         resources["global_content_filter"] = {
             "include_code": not self.exclude_code_cell,
             "include_markdown": not self.exclude_markdown,
@@ -565,13 +566,19 @@ class TemplateExporter(Exporter):
         for template_name in template_names:
             for base_dir in self.extra_template_basedirs:
                 path = os.path.join(base_dir, template_name)
-                if not prune or os.path.exists(path):
-                    paths.append(path)
+                try:
+                    if not prune or os.path.exists(path):
+                        paths.append(path)
+                except PermissionError:
+                    pass
             for root_dir in root_dirs:
                 base_dir = os.path.join(root_dir, "nbconvert", "templates")
                 path = os.path.join(base_dir, template_name)
-                if not prune or os.path.exists(path):
-                    paths.append(path)
+                try:
+                    if not prune or os.path.exists(path):
+                        paths.append(path)
+                except PermissionError:
+                    pass
 
         for root_dir in root_dirs:
             # we include root_dir for when we want to be very explicit, e.g.
diff --git a/nbconvert/filters/markdown.py b/nbconvert/filters/markdown.py
index 7e4bee9f..fe0cb10e 100644
--- a/nbconvert/filters/markdown.py
+++ b/nbconvert/filters/markdown.py
@@ -8,6 +8,10 @@ markdown within Jinja templates.
 
 import re
 
+from packaging.version import Version
+
+from nbconvert.utils.pandoc import get_pandoc_version
+
 try:
     from .markdown_mistune import markdown2html_mistune
 
@@ -66,7 +70,16 @@ def markdown2html_pandoc(source, extra_args=None):
 
 def markdown2asciidoc(source, extra_args=None):
     """Convert a markdown string to asciidoc via pandoc"""
-    extra_args = extra_args or ["--atx-headers"]
+
+    # Prior to version 3.0, pandoc supported the --atx-headers flag.
+    # For later versions, we must instead pass --markdown-headings=atx.
+    # See https://pandoc.org/releases.html#pandoc-3.0-2023-01-18
+    atx_args = ["--atx-headers"]
+    pandoc_version = get_pandoc_version()
+    if pandoc_version and Version(pandoc_version) >= Version("3.0"):
+        atx_args = ["--markdown-headings=atx"]
+
+    extra_args = extra_args or atx_args
     asciidoc = convert_pandoc(source, "markdown", "asciidoc", extra_args=extra_args)
     # workaround for https://github.com/jgm/pandoc/issues/3068
     if "__" in asciidoc:
diff --git a/pyproject.toml b/pyproject.toml
index fd87d8cd..9d876813 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,7 +27,7 @@ urls = {Homepage = "https://jupyter.org"}
 requires-python = ">=3.8"
 dependencies = [
     "beautifulsoup4",
-    "bleach!=5.0.0",
+    "bleach[css]!=5.0.0",
     "defusedxml",
     "importlib_metadata>=3.6;python_version<\"3.10\"",
     "jinja2>=3.0",
@@ -40,7 +40,6 @@ dependencies = [
     "packaging",
     "pandocfilters>=1.4.1",
     "pygments>=2.4.1",
-    "tinycss2",
     "traitlets>=5.1",
 ]
 dynamic = ["version"]
diff --git a/share/templates/base/display_priority.j2 b/share/templates/base/display_priority.j2
index 27c3c87e..c24639e9 100644
--- a/share/templates/base/display_priority.j2
+++ b/share/templates/base/display_priority.j2
@@ -38,6 +38,9 @@
         {%- elif type == 'application/vnd.jupyter.widget-view+json' -%}
             {%- block data_widget_view -%}
             {%- endblock -%}
+        {%- elif type == resources.output_mimetype -%}
+            {%- block data_native -%}
+            {%- endblock -%}
         {%- else -%}
             {%- block data_other -%}
             {%- endblock -%}
diff --git a/share/templates/rst/conf.json b/share/templates/rst/conf.json
index cf2f30a0..f1dd78aa 100644
--- a/share/templates/rst/conf.json
+++ b/share/templates/rst/conf.json
@@ -1,6 +1,6 @@
 {
   "base_template": "base",
   "mimetypes": {
-    "text/restructuredtext": true
+    "text/x-rst": true
   }
 }
diff --git a/share/templates/rst/index.rst.j2 b/share/templates/rst/index.rst.j2
index bda5c5fb..c374cd24 100644
--- a/share/templates/rst/index.rst.j2
+++ b/share/templates/rst/index.rst.j2
@@ -44,6 +44,10 @@
 {{ output.text | indent }}
 {% endblock stream %}
 
+{% block data_native %}
+{{ output.data['text/x-rst'] }}
+{% endblock data_native %}
+
 {% block data_svg %}
 .. image:: {{ output.metadata.filenames['image/svg+xml'] | urlencode }}
 {% endblock data_svg %}


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index a0c8d3ec..0b652494 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -122,6 +122,8 @@ jobs:
 
       - name: Run tests
         run: |
+          # Attempt to work around https://github.com/pypa/pip/issues/12781
+          PIP_CONSTRAINT= hatch env run -e test -- pip install 'pip>=24.2'
           xvfb-run --auto-servernum hatch run test:nowarn || xvfb-run --auto-servernum hatch run test:nowarn --lf
 
   test_prereleases:
diff --git a/tests/exporters/files/rst_output.ipynb b/tests/exporters/files/rst_output.ipynb
new file mode 100644
index 00000000..86dee7d6
--- /dev/null
+++ b/tests/exporters/files/rst_output.ipynb
@@ -0,0 +1,75 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "d5202f0f-21b8-4509-a9e2-45caf1c7db7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from textwrap import indent\n",
+    "\n",
+    "\n",
+    "class Note:\n",
+    "    def __init__(self, text):\n",
+    "        self.text = text\n",
+    "\n",
+    "    def _repr_html_(self):\n",
+    "        return f'<div style=\"font-weight: bold; font-size: 16pt;\">{self.text}</div>'\n",
+    "\n",
+    "    def _repr_mimebundle_(self, include=None, exclude=None):\n",
+    "        return {\"text/x-rst\": \".. note::\\n\\n\" + indent(self.text, \"    \")}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5145b05f-3a07-4cff-8738-516a9c27cb58",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div style=\"font-weight: bold; font-size: 16pt;\">Testing testing</div>"
+      ],
+      "text/plain": [
+       "<__main__.Note at 0x7f457c0250d0>"
+      ],
+      "text/x-rst": [
+       ".. note::\n",
+       "\n",
+       "    Testing testing"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "Note(\"Testing testing\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/tests/exporters/test_asciidoc.py b/tests/exporters/test_asciidoc.py
index e2a2ae35..0298bf90 100644
--- a/tests/exporters/test_asciidoc.py
+++ b/tests/exporters/test_asciidoc.py
@@ -50,6 +50,10 @@ class TestASCIIDocExporter(ExportersTestsBase):
         assert re.findall(in_regex, output)
         assert re.findall(out_regex, output)
 
+        # Assert that the Markdown header from our test notebook made it into the output.
+        # This can fail when nbconvert invokes pandoc incorrectly, as in issue #2017.
+        assert "== NumPy and Matplotlib examples" in output
+
     @onlyif_cmds_exist("pandoc")
     def test_export_no_prompt(self):
         """
diff --git a/tests/exporters/test_rst.py b/tests/exporters/test_rst.py
index 4a5f5c3c..6e8b24a1 100644
--- a/tests/exporters/test_rst.py
+++ b/tests/exporters/test_rst.py
@@ -68,3 +68,14 @@ class TestRSTExporter(ExportersTestsBase):
         assert ":width:" in attr_string
         assert ":height:" in attr_string
         assert "px" in attr_string
+
+    def test_rst_output(self):
+        """
+        Is native text/x-rst output included when converting
+        """
+        (output, resources) = RSTExporter().from_filename(
+            self._get_notebook(nb_name="rst_output.ipynb")
+        )
+        assert len(output) > 0
+        assert "\n.. note::" in output
+        assert ".. raw:: html" not in output  # rst should shadow html output