Repository: vega/altair

Issue:
Add strict option to altair sphinx extension
Hi!

I'm a big fan of altair and the sphinx extension that comes with it. 

One thing that I sometimes stumble over is that when I have an rst page the documentation build doesn't fail when the Python code snippets have errors. Would you be open to adding a `:strict:` flag to the code snippets so that they raise when an error occurs instead of skipping the node? I have something like this in mind: https://github.com/altair-viz/altair/compare/master...jtilly:issue-2550

I'm happy to make a PR for this or consider alternative solutions.

Thanks!


Discussion:

Resolving patch:
diff --git a/altair/sphinxext/altairgallery.py b/altair/sphinxext/altairgallery.py
--- a/altair/sphinxext/altairgallery.py
+++ b/altair/sphinxext/altairgallery.py
@@ -102,6 +102,7 @@
 
 .. altair-plot::
     {% if code_below %}:code-below:{% endif %}
+    {% if strict %}:strict:{% endif %}
 
     {{ code | indent(4) }}
 
@@ -253,7 +254,7 @@ def main(app):
 
     gallery_ref = app.builder.config.altair_gallery_ref
     gallery_title = app.builder.config.altair_gallery_title
-    examples = populate_examples(gallery_ref=gallery_ref, code_below=True)
+    examples = populate_examples(gallery_ref=gallery_ref, code_below=True, strict=False)
 
     if not os.path.exists(target_dir):
         os.makedirs(target_dir)
diff --git a/altair/sphinxext/altairplot.py b/altair/sphinxext/altairplot.py
--- a/altair/sphinxext/altairplot.py
+++ b/altair/sphinxext/altairplot.py
@@ -37,6 +37,7 @@
         :alt: text  # Alternate text when plot cannot be rendered
         :links: editor source export  # specify one or more of these options
         :chart-var-name: chart  # name of variable in namespace containing output
+        :strict: # if set, then code with errors will raise instead of being skipped
 
 
 Additionally, this extension introduces a global configuration
@@ -142,6 +143,7 @@ class AltairPlotDirective(Directive):
         "alt": unchanged,
         "links": validate_links,
         "chart-var-name": unchanged,
+        "strict": flag,
     }
 
     def run(self):
@@ -150,6 +152,7 @@ def run(self):
 
         show_code = "hide-code" not in self.options
         code_below = "code-below" in self.options
+        strict = "strict" in self.options
 
         if not hasattr(env, "_altair_namespaces"):
             env._altair_namespaces = {}
@@ -191,6 +194,7 @@ def run(self):
         )
         plot_node["output"] = self.options.get("output", "plot")
         plot_node["chart-var-name"] = self.options.get("chart-var-name", None)
+        plot_node["strict"] = strict
 
         if "alt" in self.options:
             plot_node["alt"] = self.options["alt"]
@@ -216,13 +220,14 @@ def html_visit_altair_plot(self, node):
             chart = eval_block(node["code"], namespace)
         stdout = f.getvalue()
     except Exception as e:
-        warnings.warn(
-            "altair-plot: {}:{} Code Execution failed:"
-            "{}: {}".format(
-                node["rst_source"], node["rst_lineno"], e.__class__.__name__, str(e)
-            )
+        message = "altair-plot: {}:{} Code Execution failed:" "{}: {}".format(
+            node["rst_source"], node["rst_lineno"], e.__class__.__name__, str(e)
         )
-        raise nodes.SkipNode
+        if node["strict"]:
+            raise ValueError(message) from e
+        else:
+            warnings.warn(message)
+            raise nodes.SkipNode
 
     chart_name = node["chart-var-name"]
     if chart_name is not None:


Test patch: