Repository: sagemath/sage

Issue:
Auto-translate doctests to pure Python, with import statements
I think we would be able to generate `import` statements from the `# optional` (or `# needs` - #35779) annotations.

The necessary cross-referencing information is going to be in sage.features.sagemath.
See:
- #35749 (needs review)
(doc preview: https://deploy-preview-35749--sagemath-tobias.netlify.app/reference/misc/sage/features/sagemath.html#sage.features.sagemath.all_features)

> This would also have the huge advantage to support users that want to use sage as a normal python package without loading the sage environment.

+1. 

I could imagine that the code blocks in the formatted HTML could offers tabs for different views, as is done for example in https://setuptools.pypa.io/en/latest/userguide/package_discovery.html: The example in monolithic Sage (goes through preparser, no imports), and the example in plain Python (some prettified version of the preparsed example + imports). This would also address @williamstein's comment (https://groups.google.com/g/sage-devel/c/utA0N1it0Eo/m/79aD4-RQAAAJ)

_Originally posted by @mkoeppe in https://github.com/sagemath/sage/issues/35750#issuecomment-1594840729_
            


Discussion:
The sphinx-inline-tabs package for this type of presentation in the HTML documentation is added in #37056

Resolving patch:
diff --git a/src/sage_docbuild/__main__.py b/src/sage_docbuild/__main__.py
--- a/src/sage_docbuild/__main__.py
+++ b/src/sage_docbuild/__main__.py
@@ -289,6 +289,9 @@ def setup_parser():
     standard.add_argument("--no-plot", dest="no_plot",
                           action="store_true",
                           help="do not include graphics auto-generated using the '.. plot' markup")
+    standard.add_argument("--no-preparsed-examples", dest="no_preparsed_examples",
+                          action="store_true",
+                          help="do not show preparsed versions of EXAMPLES blocks")
     standard.add_argument("--include-tests-blocks", dest="skip_tests", default=True,
                           action="store_false",
                           help="include TESTS blocks in the reference manual")
@@ -478,6 +481,8 @@ def excepthook(*exc_info):
         build_options.ALLSPHINXOPTS += "-n "
     if args.no_plot:
         os.environ['SAGE_SKIP_PLOT_DIRECTIVE'] = 'yes'
+    if args.no_preparsed_examples:
+        os.environ['SAGE_PREPARSED_DOC'] = 'no'
     if args.live_doc:
         os.environ['SAGE_LIVE_DOC'] = 'yes'
     if args.skip_tests:
diff --git a/src/sage_docbuild/conf.py b/src/sage_docbuild/conf.py
--- a/src/sage_docbuild/conf.py
+++ b/src/sage_docbuild/conf.py
@@ -39,6 +39,9 @@
 # General configuration
 # ---------------------
 
+SAGE_LIVE_DOC = os.environ.get('SAGE_LIVE_DOC', 'no')
+SAGE_PREPARSED_DOC = os.environ.get('SAGE_PREPARSED_DOC', 'yes')
+
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
@@ -57,7 +60,7 @@
 
 jupyter_execute_default_kernel = 'sagemath'
 
-if os.environ.get('SAGE_LIVE_DOC', 'no') == 'yes':
+if SAGE_LIVE_DOC == 'yes':
     SAGE_JUPYTER_SERVER = os.environ.get('SAGE_JUPYTER_SERVER', 'binder')
     if SAGE_JUPYTER_SERVER.startswith('binder'):
         # format: "binder" or
@@ -230,7 +233,7 @@ def sphinx_plot(graphics, **kwds):
 # console lexers. 'ipycon' is the IPython console, which is what we want
 # for most code blocks: anything with "sage:" prompts. For other IPython,
 # like blocks which might appear in a notebook cell, use 'ipython'.
-highlighting.lexers['ipycon'] = IPythonConsoleLexer(in1_regex=r'sage: ', in2_regex=r'[.][.][.][.]: ')
+highlighting.lexers['ipycon'] = IPythonConsoleLexer(in1_regex=r'(sage:|>>>)', in2_regex=r'([.][.][.][.]:|[.][.][.])')
 highlighting.lexers['ipython'] = IPyLexer()
 highlight_language = 'ipycon'
 
@@ -305,7 +308,7 @@ def set_intersphinx_mappings(app, config):
 multidocs_is_master = True
 
 # https://sphinx-copybutton.readthedocs.io/en/latest/use.html
-copybutton_prompt_text = r"sage: |[.][.][.][.]: |\$ "
+copybutton_prompt_text = r"sage: |[.][.][.][.]: |>>> |[.][.][.] |\$ "
 copybutton_prompt_is_regexp = True
 copybutton_exclude = '.linenos, .c1'  # exclude single comments (in particular, # optional!)
 copybutton_only_copy_prompt_lines = True
@@ -789,8 +792,6 @@ class will be properly documented inside its surrounding class.
     return skip
 
 
-from jupyter_sphinx.ast import JupyterCellNode, CellInputNode
-
 class SagecodeTransform(SphinxTransform):
     """
     Transform a code block to a live code block enabled by jupyter-sphinx.
@@ -828,29 +829,87 @@ def apply(self):
         if self.app.builder.tags.has('html') or self.app.builder.tags.has('inventory'):
             for node in self.document.traverse(nodes.literal_block):
                 if node.get('language') is None and node.astext().startswith('sage:'):
-                    source = node.rawsource
-                    lines = []
-                    for line in source.splitlines():
-                        newline = line.lstrip()
-                        if newline.startswith('sage: ') or newline.startswith('....: '):
-                            lines.append(newline[6:])
-                    cell_node = JupyterCellNode(
-                                execute=False,
-                                hide_code=True,
-                                hide_output=True,
-                                emphasize_lines=[],
-                                raises=False,
-                                stderr=True,
-                                code_below=False,
-                                classes=["jupyter_cell"])
-                    cell_input = CellInputNode(classes=['cell_input','live-doc'])
-                    cell_input += nodes.literal_block(
-                        text='\n'.join(lines),
-                        linenos=False,
-                        linenostart=1)
-                    cell_node += cell_input
-
-                    node.parent.insert(node.parent.index(node) + 1, cell_node)
+                    from docutils.nodes import container as Container, label as Label, literal_block as LiteralBlock, Text
+                    from sphinx_inline_tabs._impl import TabContainer
+                    parent = node.parent
+                    index = parent.index(node)
+                    if isinstance(node.previous_sibling(), TabContainer):
+                        # Make sure not to merge inline tabs for adjacent literal blocks
+                        parent.insert(index, Text(''))
+                        index += 1
+                    parent.remove(node)
+                    # Tab for Sage code
+                    container = TabContainer("", type="tab", new_set=False)
+                    textnodes = [Text('Sage')]
+                    label = Label("", "", *textnodes)
+                    container += label
+                    content = Container("", is_div=True, classes=["tab-content"])
+                    content += node
+                    container += content
+                    parent.insert(index, container)
+                    if SAGE_PREPARSED_DOC == 'yes':
+                        # Tab for preparsed version
+                        from sage.repl.preparse import preparse
+                        container = TabContainer("", type="tab", new_set=False)
+                        textnodes = [Text('Python')]
+                        label = Label("", "", *textnodes)
+                        container += label
+                        content = Container("", is_div=True, classes=["tab-content"])
+                        example_lines = []
+                        preparsed_lines = ['>>> from sage.all import *']
+                        for line in node.rawsource.splitlines() + ['']:  # one extra to process last example
+                            newline = line.lstrip()
+                            if newline.startswith('....: '):
+                                example_lines.append(newline[6:])
+                            else:
+                                if example_lines:
+                                    preparsed_example = preparse('\n'.join(example_lines))
+                                    prompt = '>>> '
+                                    for preparsed_line in preparsed_example.splitlines():
+                                        preparsed_lines.append(prompt + preparsed_line)
+                                        prompt = '... '
+                                    example_lines = []
+                                if newline.startswith('sage: '):
+                                    example_lines.append(newline[6:])
+                                else:
+                                    preparsed_lines.append(line)
+                        preparsed = '\n'.join(preparsed_lines)
+                        preparsed_node = LiteralBlock(preparsed, preparsed, language='ipycon')
+                        content += preparsed_node
+                        container += content
+                        parent.insert(index + 1, container)
+                    if SAGE_LIVE_DOC == 'yes':
+                        # Tab for Jupyter-sphinx cell
+                        from jupyter_sphinx.ast import JupyterCellNode, CellInputNode
+                        source = node.rawsource
+                        lines = []
+                        for line in source.splitlines():
+                            newline = line.lstrip()
+                            if newline.startswith('sage: ') or newline.startswith('....: '):
+                                lines.append(newline[6:])
+                        cell_node = JupyterCellNode(
+                                    execute=False,
+                                    hide_code=False,
+                                    hide_output=True,
+                                    emphasize_lines=[],
+                                    raises=False,
+                                    stderr=True,
+                                    code_below=False,
+                                    classes=["jupyter_cell"])
+                        cell_input = CellInputNode(classes=['cell_input','live-doc'])
+                        cell_input += nodes.literal_block(
+                            text='\n'.join(lines),
+                            linenos=False,
+                            linenostart=1)
+                        cell_node += cell_input
+                        container = TabContainer("", type="tab", new_set=False)
+                        textnodes = [Text('Sage Live')]
+                        label = Label("", "", *textnodes)
+                        container += label
+                        content = Container("", is_div=True, classes=["tab-content"])
+                        content += cell_node
+                        container += content
+                        parent.insert(index + 1, container)
 
 
 # This replaces the setup() in sage.misc.sagedoc_conf
@@ -864,7 +923,7 @@ def setup(app):
         app.connect('autodoc-process-docstring', skip_TESTS_block)
     app.connect('autodoc-skip-member', skip_member)
     app.add_transform(SagemathTransform)
-    if os.environ.get('SAGE_LIVE_DOC', 'no') == 'yes':
+    if SAGE_LIVE_DOC == 'yes' or SAGE_PREPARSED_DOC == 'yes':
         app.add_transform(SagecodeTransform)
 
     # When building the standard docs, app.srcdir is set to SAGE_DOC_SRC +


Test patch: