Repository: holoviz/panel

Issue:
Test documentation markdown documentation python
With the switch of some of the documentation from notebook files to markdown files, it makes sense to add a way to test all the code blocks to verify it is able to run from top to bottom. 


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -142,6 +142,7 @@ def run(self):
     'ipython >=7.0',
     'holoviews',
     'diskcache',
+    "markdown-it-py",
     # Temporary pins (jupyter_bokeh needs updates)
     'ipywidgets <8.0'
 ]


Test patch:
diff --git a/panel/tests/test_docs.py b/panel/tests/test_docs.py
--- a/panel/tests/test_docs.py
+++ b/panel/tests/test_docs.py
@@ -1,24 +1,32 @@
 """
-These tests verify that all of the panes, layouts, and widgets defined by panel are
+The ref_available tests verify that all of the panes, layouts, and widgets defined by panel are
 represented in the reference gallery.
+
+The doc_available tests check that python in markdown files can be run top to bottom.
+
 """
-import os
+import ast
 
 from inspect import isclass
+from pathlib import Path
+from subprocess import run
 
 import pytest
 
 import panel as pn
 
-here = os.path.abspath(os.path.dirname(__file__))
-ref = os.path.join(here, '..', '..', 'examples', 'reference')
-docs_available = pytest.mark.skipif(not os.path.isdir(ref), reason="docs not found")
+REF_PATH = Path(__file__).parents[2] / "examples" / "reference"
+ref_available = pytest.mark.skipif(not REF_PATH.is_dir(), reason="folder 'examples/reference' not found")
 
+DOC_PATH = Path(__file__).parents[2] / "doc"
+doc_files = sorted(DOC_PATH.rglob("*.md"))
+doc_available = pytest.mark.skipif(not DOC_PATH.is_dir(), reason="folder 'doc' not found")
 
-@docs_available
+
+@ref_available
 def test_layouts_are_in_reference_gallery():
-    exceptions = set(['ListPanel', 'Panel'])
-    docs = {os.path.splitext(f)[0] for f in os.listdir(os.path.join(ref, 'layouts'))}
+    exceptions = {"ListPanel", "Panel"}
+    docs = {f.with_suffix("").name for f in (REF_PATH / "layouts").iterdir()}
 
     def is_panel_layout(attr):
         layout = getattr(pn.layout, attr)
@@ -28,10 +36,14 @@ def is_panel_layout(attr):
     assert layouts - exceptions - docs == set()
 
 
-@docs_available
+@ref_available
 def test_widgets_are_in_reference_gallery():
-    exceptions = set(['CompositeWidget', 'Widget', 'ToggleGroup', 'NumberInput', 'Spinner'])
-    docs = {os.path.splitext(f)[0] for g in ('indicators', 'widgets') for f in os.listdir(os.path.join(ref, g))}
+    exceptions = {"CompositeWidget", "Widget", "ToggleGroup", "NumberInput", "Spinner"}
+    docs = {
+        f.with_suffix("").name
+        for g in ("indicators", "widgets")
+        for f in (REF_PATH / g).iterdir()
+    }
 
     def is_panel_widget(attr):
         widget = getattr(pn.widgets, attr)
@@ -41,10 +53,10 @@ def is_panel_widget(attr):
     assert widgets - exceptions - docs == set()
 
 
-@docs_available
+@ref_available
 def test_panes_are_in_reference_gallery():
-    exceptions = set(['PaneBase', 'YT', 'RGGPlot', 'Interactive', 'ICO'])
-    docs = {os.path.splitext(f)[0] for f in os.listdir(os.path.join(ref, 'panes'))}
+    exceptions = {"PaneBase", "YT", "RGGPlot", "Interactive", "ICO"}
+    docs = {f.with_suffix("").name for f in (REF_PATH / "panes").iterdir()}
 
     def is_panel_pane(attr):
         pane = getattr(pn.pane, attr)
@@ -52,3 +64,26 @@ def is_panel_pane(attr):
 
     panes = set(filter(is_panel_pane, dir(pn.pane)))
     assert panes - exceptions - docs == set()
+
+
+@doc_available
+@pytest.mark.parametrize(
+    "file", doc_files, ids=[str(f.relative_to(DOC_PATH)) for f in doc_files]
+)
+def test_markdown_codeblocks(file):
+    from markdown_it import MarkdownIt
+
+    exceptions = ("await", "pn.serve", "django")
+
+    md_ast = MarkdownIt().parse(file.read_text(encoding="utf-8"))
+    lines = ""
+    for n in md_ast:
+        if n.tag == "code" and n.info is not None:
+            if "pyodide" in n.info.lower() or "python" in n.info.lower():
+                if ">>>" not in n.content:
+                    lines += n.content
+    if lines:
+        ast.parse(lines)
+
+    if lines and not any(w in lines for w in exceptions):
+        run(["python", "-c", lines], timeout=30, check=True)