Repository: jupyter/nbconvert
Pull request #2203 (2025-01-06 21:39:22)

Prevent leading whitespace in markdown code blocks from being stripped
This PR prevents leading whitespace from being stripped from markdown code blocks before they are parsed for highlighting. For certain languages, leading whitespace is syntactically significant. Fixes https://github.com/jupyter/nbviewer/issues/1021.

I tried to go as far back as I could through the git history to figure out why this was added, but it appears in the earliest tag, `4.0.0`. If there _is_ a reason to strip leading whitespaces, it's not explicitly tested for in the test suite. If you have any context around this choice, I'd be interested in hearing about it.

I also added a test to check that whitespaces aren't stripped from code blocks. I've also added a `RUF001` rule to ignore uses of ambiguous unicode characters in `tests/exporters/test_html.py`, because of the "ɩ" which is apparently a valid APL function.

krassowski: Does it also fix https://github.com/jupyter/nbconvert/issues/2156?
peytondmurray: > Does it also fix https://github.com/jupyter/nbconvert/issues/2156?

I don't think so - this PR only addresses code blocks inside markdown. The issue above :point_up: is about being able to split statements across multiple code cells - for example, putting a single (empty) `if`, `with`, or `while` etc in one cell, and having indented code in subsequent cells further down be part of the scope of the first cell:

![image](https://github.com/user-attachments/assets/09bf29e2-25cf-4c28-a58a-2b2b82c245be)

Maybe this is a discussion for that issue, but I'm not sure that makes sense to do here - at the moment, each cell needs to be a syntactically complete snippet of python code, which isn't true in the use case provided.

About your other suggestion: I like it, I'll add the option to pass arbitrary lexer options to the pygments lexers now!
peytondmurray: I've added a test for the lexer options. The user can revert to the old behavior by setting

```python
HTMLExporter(lexer_options={"stripall": True})
```

The option should be documented by traitlets, so I haven't included anything extra about reverting back to the old behavior. But if some additional documentation would be helpful please let me know and I can add an explicit section about this to the docs.
divyansshhh: I see the PR is being worked on, is there an ETA on when it would be released? 

Patch:
diff --git a/nbconvert/exporters/html.py b/nbconvert/exporters/html.py
index d63699c0..56724b06 100644
--- a/nbconvert/exporters/html.py
+++ b/nbconvert/exporters/html.py
@@ -15,6 +15,7 @@ import markupsafe
 from bs4 import BeautifulSoup
 from jupyter_core.paths import jupyter_path
 from traitlets import Bool, Unicode, default, validate
+from traitlets import Dict as TraitletsDict
 from traitlets.config import Config
 
 if tuple(int(x) for x in jinja2.__version__.split(".")[:3]) < (3, 0, 0):
@@ -183,6 +184,14 @@ class HTMLExporter(TemplateExporter):
 
     output_mimetype = "text/html"
 
+    lexer_options = TraitletsDict(
+        {},
+        help=(
+            "Options to be passed to the pygments lexer for highlighting markdown code blocks. "
+            "See https://pygments.org/docs/lexers/#available-lexers for available options."
+        ),
+    ).tag(config=True)
+
     @property
     def default_config(self):
         c = Config(
@@ -239,6 +248,7 @@ class HTMLExporter(TemplateExporter):
             path=path,
             anchor_link_text=self.anchor_link_text,
             exclude_anchor_links=self.exclude_anchor_links,
+            **self.lexer_options,
         )
         return MarkdownWithMath(renderer=renderer).render(source)
 
diff --git a/nbconvert/filters/highlight.py b/nbconvert/filters/highlight.py
index d29a4759..3a85bd17 100644
--- a/nbconvert/filters/highlight.py
+++ b/nbconvert/filters/highlight.py
@@ -136,7 +136,9 @@ class Highlight2Latex(NbConvertBase):
         return latex
 
 
-def _pygments_highlight(source, output_formatter, language="ipython", metadata=None):
+def _pygments_highlight(
+    source, output_formatter, language="ipython", metadata=None, **lexer_options
+):
     """
     Return a syntax-highlighted version of the input source
 
@@ -149,6 +151,10 @@ def _pygments_highlight(source, output_formatter, language="ipython", metadata=N
         language to highlight the syntax of
     metadata : NotebookNode cell metadata
         metadata of the cell to highlight
+    lexer_options : dict
+        Options to pass to the pygments lexer. See
+        https://pygments.org/docs/lexers/#available-lexers for more information about
+        valid lexer options
     """
     from pygments import highlight
     from pygments.lexers import get_lexer_by_name
@@ -179,7 +185,7 @@ def _pygments_highlight(source, output_formatter, language="ipython", metadata=N
 
     if lexer is None:
         try:
-            lexer = get_lexer_by_name(language, stripall=True)
+            lexer = get_lexer_by_name(language, **lexer_options)
         except ClassNotFound:
             warn("No lexer found for language %r. Treating as plain text." % language, stacklevel=2)
             from pygments.lexers.special import TextLexer
diff --git a/nbconvert/filters/markdown_mistune.py b/nbconvert/filters/markdown_mistune.py
index fb882816..ea46bba0 100644
--- a/nbconvert/filters/markdown_mistune.py
+++ b/nbconvert/filters/markdown_mistune.py
@@ -293,6 +293,7 @@ class IPythonRenderer(HTMLRenderer):
         anchor_link_text: str = "¶",
         path: str = "",
         attachments: Optional[Dict[str, Dict[str, str]]] = None,
+        **lexer_options,
     ):
         """Initialize the renderer."""
         super().__init__(escape, allow_harmful_protocols)
@@ -300,6 +301,7 @@ class IPythonRenderer(HTMLRenderer):
         self.exclude_anchor_links = exclude_anchor_links
         self.anchor_link_text = anchor_link_text
         self.path = path
+        self.lexer_options = lexer_options
         if attachments is not None:
             self.attachments = attachments
         else:
@@ -317,7 +319,7 @@ class IPythonRenderer(HTMLRenderer):
             try:
                 if info.strip().split(None, 1):
                     lang = info.strip().split(maxsplit=1)[0]
-                    lexer = get_lexer_by_name(lang, stripall=True)
+                    lexer = get_lexer_by_name(lang, **self.lexer_options)
             except ClassNotFound:
                 code = f"{lang}\n{code}"
                 lang = None
diff --git a/pyproject.toml b/pyproject.toml
index 9d876813..06fd056f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -224,7 +224,6 @@ ignore = [
   "T201",   # `print` found
   "RUF012", # Mutable class attributes should be annotated
   "UP031",  # Use format specifiers instead of percent format
-
 ]
 unfixable = [
   "T201",   # Don't touch print statements
@@ -253,6 +252,7 @@ unfixable = [
 "nbconvert/__init__.py" = ["F401", "F403"]
 # PLR2004 Magic value used in comparison
 "nbconvert/filters/ansi.py" = ["PLR2004"]
+"tests/exporters/test_html.py" = ["RUF001"]
 
 [tool.interrogate]
 ignore-init-module=true


Test patch:
diff --git a/tests/exporters/test_html.py b/tests/exporters/test_html.py
index 810de47e..f1bfb69b 100644
--- a/tests/exporters/test_html.py
+++ b/tests/exporters/test_html.py
@@ -5,6 +5,7 @@
 
 import re
 
+import pytest
 from nbformat import v4
 from traitlets.config import Config
 
@@ -262,3 +263,50 @@ class TestHTMLExporter(ExportersTestsBase):
         (output, resources) = exporter.from_filename(self._get_notebook())
 
         assert '<html lang="en">' in output
+
+
+@pytest.mark.parametrize(
+    ("lexer_options"),
+    [
+        {"stripall": True},
+        {"stripall": False},
+        {},
+    ],
+)
+def test_syntax_highlight_leading_whitespace(lexer_options):
+    """Test that syntax highlight doesn't strip leading spaces."""
+    nb = v4.reads(r"""
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "29da71a9-ae40-4098-8c3b-31a98e79fc12",
+   "metadata": {},
+   "source": [
+    "```APL\n",
+    "      1+2×⍳3\n",
+    "3 5 7\n",
+    "```\n",
+    "\n",
+    "```\n",
+    "      1+2×⍳3\n",
+    "3 5 7\n",
+    "```"
+   ]
+  }
+ ],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
+    """)
+    output, _ = HTMLExporter(lexer_options=lexer_options).from_notebook_node(nb)
+    # Check that the second code block has the leading spaces
+    assert "<pre><code>      1+2×⍳3\n3 5 7\n</code></pre>" in output
+
+    if lexer_options.get("stripall"):
+        # Check that the APL-formatted code block has leading spaces stripped
+        assert '<span class="w">      </span>' not in output
+    else:
+        # Check that the APL-formatted code block has the leading spaces
+        assert '<span class="w">      </span>' in output