Repository: GenericMappingTools/pygmt

Issue:
The dynamically generated images are not shown in the documentation
In https://github.com/GenericMappingTools/pygmt/pull/3379, we enabled the `myst-nb` extension to generate images dynamically from codes in Markdown files.

In the RTD preview (https://pygmt-dev--3379.org.readthedocs.build/en/3379/install.html#testing-your-install), the image was shown correctly, but for the documentation built by GitHub Actions, the image is not shown (see https://www.pygmt.org/dev/install.html#testing-your-install).

The difference is that, when building documentation, RTD calls `sphinx-build` directly, while in the "Docs" workflow, we run `make -C doc clean all`, which set `PYGMT_USE_EXTERNAL_DISPLAY="false"` before calling `sphinx-build`:

https://github.com/GenericMappingTools/pygmt/blob/0faf52c2410601f551e7e8661cafdd28889dd0c0/doc/Makefile#L36

The complicated thing is:

Gallery examples are executed by Sphinx-Gallery, which executes the Python codes using the `compile`/`exec` functions. Thus, the IPython kernel is not available and images are opened using external viewers by default. We have to disable it by setting `PYGMT_USE_EXTERNAL_DISPLAY="false"`. The PyGMTScraper class keeps track of all `Figure` instances then call `Figure.show()` and save the images using `Figure.savefig`.

Instead, code cells in MyST Markdown files are executed by the myst-nb extension. As I understand it, it has an IPython Kernel, so the default display method is `"notebook"` and can't be `"none"`.

So, the solution should be straightforward. `PYGMT_USE_EXTERNAL_DISPLAY="false"` should only have effects when the default method is `"external"`. 




Discussion:
seisman: I'm self-proving this PR so that we can continue other PRs.

Patch:
diff --git a/pygmt/figure.py b/pygmt/figure.py
index 0509e765..05cba50e 100644
--- a/pygmt/figure.py
+++ b/pygmt/figure.py
@@ -6,6 +6,7 @@ import base64
 import os
 from pathlib import Path, PurePath
 from tempfile import TemporaryDirectory
+from typing import Literal
 
 try:
     import IPython
@@ -26,27 +27,49 @@ from pygmt.helpers import (
     use_alias,
 )
 
+
+def _get_default_display_method() -> Literal["external", "notebook", "none"]:
+    """
+    Get the default method to display preview images.
+
+    The function checks the current environment and determines the most suitable method
+    to display preview images when calling :meth:`pygmt.Figure.show`. Valid display
+    methods are:
+
+    - ``"external"``: External PDF preview using the default PDF viewer
+    - ``"notebook"``: Inline PNG preview in the current notebook
+    - ``"none"``: Disable image preview
+
+    The default display method is ``"notebook"`` in the Jupyter notebook environment,
+    and ``"external"`` in other cases.
+
+    Setting environment variable **PYGMT_USE_EXTERNAL_DISPLAY** to ``"false"`` can
+    disable image preview in external viewers. It's useful when running the tests and
+    building the documentation to avoid popping up windows.
+
+    Returns
+    -------
+    method
+        The default display method.
+    """
+    # Check if an IPython kernel is running.
+    if _HAS_IPYTHON and (ipy := IPython.get_ipython()) and "IPKernelApp" in ipy.config:
+        return "notebook"
+    # Check if the environment variable PYGMT_USE_EXTERNAL_DISPLAY is set to "false".
+    if os.environ.get("PYGMT_USE_EXTERNAL_DISPLAY", "true").lower() == "false":
+        return "none"
+    # Fallback to using the external viewer.
+    return "external"
+
+
 # A registry of all figures that have had "show" called in this session.
 # This is needed for the sphinx-gallery scraper in pygmt/sphinx_gallery.py
 SHOWED_FIGURES = []
-
-# Configurations for figure display
+# Configurations for figure display.
 SHOW_CONFIG = {
-    "method": "external",  # Open in an external viewer by default
+    "method": _get_default_display_method(),  # The image preview display method.
 }
 
-# Show figures in Jupyter notebooks if available
-if _HAS_IPYTHON:
-    get_ipython = IPython.get_ipython()
-    if get_ipython and "IPKernelApp" in get_ipython.config:  # Jupyter Notebook enabled
-        SHOW_CONFIG["method"] = "notebook"
-
-# Set environment variable PYGMT_USE_EXTERNAL_DISPLAY to 'false' to disable
-# external display. Use it when running the tests and building the docs to
-# avoid popping up windows.
-if os.environ.get("PYGMT_USE_EXTERNAL_DISPLAY", "true").lower() == "false":
-    SHOW_CONFIG["method"] = "none"
-
 
 class Figure:
     """


Test patch:
diff --git a/pygmt/tests/test_figure.py b/pygmt/tests/test_figure.py
index 042c9876..aadaaacf 100644
--- a/pygmt/tests/test_figure.py
+++ b/pygmt/tests/test_figure.py
@@ -4,7 +4,6 @@ Test the behavior of the Figure class.
 Doesn't include the plotting commands which have their own test files.
 """
 
-import importlib
 from pathlib import Path
 
 import numpy as np
@@ -12,9 +11,15 @@ import numpy.testing as npt
 import pytest
 from pygmt import Figure, set_display
 from pygmt.exceptions import GMTError, GMTInvalidInput
+from pygmt.figure import _get_default_display_method
 from pygmt.helpers import GMTTempFile
 
-HAS_IPYTHON = bool(importlib.util.find_spec("IPython"))
+try:
+    import IPython
+
+    _HAS_IPYTHON = True
+except ImportError:
+    _HAS_IPYTHON = False
 
 
 def test_figure_region():
@@ -307,7 +312,7 @@ def test_figure_savefig_worldfile():
                 fig.savefig(fname=imgfile.name, worldfile=True)
 
 
-@pytest.mark.skipif(not HAS_IPYTHON, reason="run when IPython is installed")
+@pytest.mark.skipif(not _HAS_IPYTHON, reason="run when IPython is installed")
 def test_figure_show():
     """
     Test that show creates the correct file name and deletes the temp dir.
@@ -347,7 +352,7 @@ def test_figure_show_invalid_method():
         fig.show(method="test")
 
 
-@pytest.mark.skipif(HAS_IPYTHON, reason="run without IPython installed")
+@pytest.mark.skipif(_HAS_IPYTHON, reason="run without IPython installed")
 def test_figure_show_notebook_error_without_ipython():
     """
     Test to check if an error is raised when display method is 'notebook', but IPython
@@ -390,3 +395,48 @@ def test_figure_unsupported_xshift_yshift():
         fig.plot(x=1, y=1, style="c3c", yshift="3c")
     with pytest.raises(GMTInvalidInput):
         fig.plot(x=1, y=1, style="c3c", Y="3c")
+
+
+class TestGetDefaultDisplayMethod:
+    """
+    Test the _get_default_display_method function.
+    """
+
+    def test_default_display_method(self, monkeypatch):
+        """
+        Default display method is "external" if PYGMT_USE_EXTERNAL_DISPLAY is undefined.
+        """
+        monkeypatch.delenv("PYGMT_USE_EXTERNAL_DISPLAY", raising=False)
+        assert _get_default_display_method() == "external"
+
+    def test_disable_external_display(self, monkeypatch):
+        """
+        Setting PYGMT_USE_EXTERNAL_DISPLAY to "false" should disable external display.
+        """
+        monkeypatch.setenv("PYGMT_USE_EXTERNAL_DISPLAY", "false")
+        assert _get_default_display_method() == "none"
+
+    @pytest.mark.skipif(not _HAS_IPYTHON, reason="Run when IPython is installed")
+    def test_notebook_display(self, monkeypatch):
+        """
+        Default display method is "notebook" when an IPython kernel is running.
+        """
+
+        class MockIPython:
+            """
+            A simple mock class to simulate an IPython instance.
+            """
+
+            def __init__(self):
+                self.config = {"IPKernelApp": True}
+
+        # Mock IPython.get_ipython() to return a MockIPython instance.
+        mock_ipython = MockIPython()
+        monkeypatch.setattr(IPython, "get_ipython", lambda: mock_ipython)
+
+        # Default display method should be "notebook" when an IPython kernel is running.
+        assert _get_default_display_method() == "notebook"
+
+        # PYGMT_USE_EXTERNAL_DISPLAY should not affect notebook display.
+        monkeypatch.setenv("PYGMT_USE_EXTERNAL_DISPLAY", "false")
+        assert _get_default_display_method() == "notebook"