Repository: yt-project/yt

Issue:
Enzo_64 dataset not loadable by load_sample()
### Bug report

As I was updating the quickstart notebooks, I came upon a failure for `load_sample()` being unable to load the `Enzo_64` dataset.  See [here](https://yt-project.org/docs/dev/quickstart/simple_visualization.html).  When I attempt this command locally, yt automatically downloads and untars the dataset, but it seems like it's trying to look at a filename called `None`.  No problems in loading the dataset manually with `load()`.

**Code for reproduction**

```python
ds = yt.load_sample("Enzo_64", "DD0043/data0043")
s = yt.SlicePlot(ds, "z", ["density", "velocity_magnitude"], center="max")
s.set_cmap("velocity_magnitude", "kamae")
s.zoom(10.0)
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-12-7024538d34af> in <module>
----> 1 ds = yt.load_sample("Enzo_64", "DD0043/data0043")
      2 s = yt.SlicePlot(ds, "z", ["density", "velocity_magnitude"], center="max")
      3 s.set_cmap("velocity_magnitude", "kamae")
      4 s.zoom(10.0)

/var/jenkins_home/workspace/yt_docs/yt/loaders.py in load_sample(fn, progressbar, timeout, **kwargs)
   1363         if timeout is not None:
   1364             mylog.info("Ignoring the `timeout` keyword argument received.")
-> 1365         return load(loadable_path, **kwargs)
   1366 
   1367     try:

/var/jenkins_home/workspace/yt_docs/yt/loaders.py in load(fn, *args, **kwargs)
     78     # either in the current dir or yt.config.ytcfg['data_dir_directory']
     79     if not fn.startswith("http"):
---> 80         fn = str(lookup_on_disk_data(fn))
     81 
     82     candidates = []

/var/jenkins_home/workspace/yt_docs/yt/sample_data/api.py in lookup_on_disk_data(fn)
    162     if not alt_path.exists():
    163         err_msg += f"\n(Also tried '{alt_path}')."
--> 164         raise FileNotFoundError(err_msg)
    165 
    166     return alt_path

FileNotFoundError: No such file or directory: '/mnt/yt/Enzo_64/None'.
(Also tried '/mnt/yt/Enzo_64/None').
```




Discussion:

Resolving patch:
diff --git a/yt/loaders.py b/yt/loaders.py
--- a/yt/loaders.py
+++ b/yt/loaders.py
@@ -6,6 +6,7 @@
 import sys
 import tarfile
 from pathlib import Path
+from typing import Optional
 from urllib.parse import urlsplit
 
 import numpy as np
@@ -1302,20 +1303,21 @@ def flatten(l):
 
 
 # --- Loader for yt sample datasets ---
-def load_sample(fn=None, progressbar: bool = True, timeout=None, **kwargs):
+def load_sample(
+    fn: Optional[str] = None, progressbar: bool = True, timeout=None, **kwargs
+):
     """
     Load sample data with yt.
 
     This is a simple wrapper around `yt.load` to include fetching
     data with pooch from remote source.
 
-    yt sample data can be found at:
-    https://yt-project.org/data.
-
     The data registry table can be retrieved and visualized using
     `yt.sample_data.api.get_data_registry_table()`.
-
-    This function requires pandas and pooch to be installed.
+    The `filename` column contains usable keys that can be passed
+    as the first positional argument to `load_sample`.
+    Some data samples contain series of datasets. It may be required to
+    supply the relative path to a specific dataset.
 
     Parameters
     ----------
@@ -1331,9 +1333,18 @@ def load_sample(fn=None, progressbar: bool = True, timeout=None, **kwargs):
         `None` means "no limit". This parameter is directly passed to down to
         requests.get via pooch.HTTPDownloader
 
-    Any additional keyword argument is passed down to `yt.load`.
-    Note that in case of collision with predefined keyword arguments as set in
+    Notes
+    -----
+    - This function is experimental as of yt 4.0.0, do not rely on its exact behaviour.
+    - Any additional keyword argument is passed down to `yt.load`.
+    - In case of collision with predefined keyword arguments as set in
     the data registry, the ones passed to this function take priority.
+    - Datasets with slashes '/' in their names can safely be used even on Windows.
+      On the contrary, paths using backslashes '\' won't work outside of Windows, so
+      it is recommended to favour the UNIX convention ('/') in scripts that are meant
+      to be cross-platform.
+    - This function requires pandas and pooch.
+    - Corresponding sample data live at https://yt-project.org/data
     """
 
     if fn is None:
@@ -1352,7 +1363,12 @@ def load_sample(fn=None, progressbar: bool = True, timeout=None, **kwargs):
 
     pooch_logger = pooch.utils.get_logger()
 
-    topdir, _, specific_file = str(fn).partition(os.path.sep)
+    # normalize path for platform portability
+    # for consistency with yt.load, we also convert to str explicitly,
+    # which gives us support Path objects for free
+    fn = str(fn).replace("/", os.path.sep)
+
+    topdir, _, specific_file = fn.partition(os.path.sep)
 
     registry_table = get_data_registry_table()
     # PR 3089
@@ -1369,9 +1385,10 @@ def load_sample(fn=None, progressbar: bool = True, timeout=None, **kwargs):
     except IndexError as err:
         raise KeyError(f"Could not find '{fn}' in the registry.") from err
 
-    if not specs["load_name"]:
+    load_name = specific_file or specs["load_name"]
+    if not load_name:
         raise ValueError(
-            "Registry appears to be corrupted: could not find a 'load_name' entry for this dataset."
+            "Missing a 'load_name' entry for this dataset. This may be fixed by requesting the full path of the loadable file."
         )
 
     if not isinstance(specs["load_kwargs"], dict):
@@ -1384,28 +1401,21 @@ def load_sample(fn=None, progressbar: bool = True, timeout=None, **kwargs):
 
     kwargs = {**specs["load_kwargs"], **kwargs}
 
-    try:
-        data_dir = lookup_on_disk_data(fn)
-    except FileNotFoundError:
-        mylog.info("'%s' is not available locally. Looking up online.", fn)
-    else:
+    save_dir = _get_test_data_dir_path()
+
+    data_path = save_dir.joinpath(fn)
+    if data_path.exists():
         # if the data is already available locally, `load_sample`
         # only acts as a thin wrapper around `load`
-        loadable_path = data_dir.joinpath(specs["load_name"], specific_file)
-        mylog.info("Sample dataset found in '%s'", data_dir)
+        appendum = specific_file or specs["load_name"]
+        if appendum not in str(data_path):
+            data_path = data_path.joinpath(appendum)
+        mylog.info("Sample dataset found in '%s'", data_path)
         if timeout is not None:
             mylog.info("Ignoring the `timeout` keyword argument received.")
-        return load(loadable_path, **kwargs)
+        return load(data_path, **kwargs)
 
-    try:
-        save_dir = _get_test_data_dir_path()
-        assert save_dir.is_dir()
-    except (OSError, AssertionError):
-        mylog.warning(
-            "yt test data directory is not properly set up. "
-            "Data will be saved to the current work directory instead."
-        )
-        save_dir = Path.cwd()
+    mylog.info("'%s' is not available locally. Looking up online.", fn)
 
     # effectively silence the pooch's logger and create our own log instead
     pooch_logger.setLevel(100)
@@ -1430,9 +1440,11 @@ def load_sample(fn=None, progressbar: bool = True, timeout=None, **kwargs):
     else:
         os.replace(tmp_file, save_dir)
 
-    loadable_path = Path.joinpath(save_dir, fn, specs["load_name"], specific_file)
+    loadable_path = Path.joinpath(save_dir, fn)
+    if not specs["load_name"] in str(loadable_path):
+        loadable_path = loadable_path.joinpath(specs["load_name"], specific_file)
 
     if specific_file and not loadable_path.exists():
-        raise ValueError(f"Could not find file '{specific_file}'.")
+        raise ValueError(f"Could not find file '{loadable_path}'.")
 
     return load(loadable_path, **kwargs)
diff --git a/yt/sample_data/api.py b/yt/sample_data/api.py
--- a/yt/sample_data/api.py
+++ b/yt/sample_data/api.py
@@ -7,6 +7,7 @@
 from itertools import chain
 from pathlib import Path
 from typing import Optional, Union
+from warnings import warn
 
 import pkg_resources
 
@@ -112,9 +113,8 @@ def get_data_registry_table():
     )
 
     # load local data
-    pooch_json = json.load(
-        pkg_resources.resource_stream("yt", "sample_data_registry.json")
-    )
+    with pkg_resources.resource_stream("yt", "sample_data_registry.json") as fh:
+        pooch_json = json.load(fh)
     pooch_table = pd.DataFrame(pooch_json.values())
 
     # merge tables
@@ -132,7 +132,15 @@ def get_data_registry_table():
 def _get_test_data_dir_path():
     from yt.config import ytcfg
 
-    return Path(ytcfg.get("yt", "test_data_dir"))
+    p = Path(ytcfg.get("yt", "test_data_dir"))
+    if p.is_dir():
+        return p
+    warn(
+        "Storage directory from yt config doesn't exist "
+        f"(currently set to '{p}'). "
+        "Current working directory will be used instead."
+    )
+    return Path.cwd()
 
 
 def lookup_on_disk_data(fn):
@@ -171,9 +179,8 @@ def _get_pooch_instance():
     data_registry = get_data_registry_table()
     cache_storage = _get_test_data_dir_path() / "yt_download_cache"
 
-    sample_data_registry = json.load(
-        pkg_resources.resource_stream("yt", "sample_data_registry.json")
-    )
+    with pkg_resources.resource_stream("yt", "sample_data_registry.json") as fh:
+        sample_data_registry = json.load(fh)
     registry = {k: v["hash"] for k, v in sample_data_registry.items()}
     return pooch.create(
         path=cache_storage, base_url="https://yt-project.org/data/", registry=registry


Test patch:
diff --git a/yt/tests/test_load_sample.py b/yt/tests/test_load_sample.py
--- a/yt/tests/test_load_sample.py
+++ b/yt/tests/test_load_sample.py
@@ -1,12 +1,12 @@
 import logging
 import os
+import sys
 import textwrap
 
 import numpy as np
 import pytest
 
 from yt.config import ytcfg
-from yt.frontends.enzo.api import EnzoDataset
 from yt.loaders import load_sample
 from yt.sample_data.api import get_data_registry_table
 from yt.testing import requires_module_pytest
@@ -40,42 +40,56 @@ def capturable_logger(caplog):
 
 @requires_module_pytest("pandas", "pooch")
 @pytest.mark.usefixtures("capturable_logger")
-def test_load_sample_small_dataset(tmp_data_dir, caplog):
-    ds = load_sample("ToroShockTube", progressbar=False, timeout=30)
-    assert isinstance(ds, EnzoDataset)
+@pytest.mark.parametrize(
+    "fn, archive, exact_loc, class_",
+    [
+        (
+            "ToroShockTube",
+            "ToroShockTube.tar.gz",
+            "ToroShockTube/DD0001/data0001",
+            "EnzoDataset",
+        ),
+        (
+            "KeplerianDisk",
+            "KeplerianDisk.tar.gz",
+            "KeplerianDisk/disk.out1.00000.athdf",
+            "AthenaPPDataset",
+        ),
+        # trying with an exact name as well
+        (
+            "KeplerianDisk/disk.out1.00000.athdf",
+            "KeplerianDisk.tar.gz",
+            "KeplerianDisk/disk.out1.00000.athdf",
+            "AthenaPPDataset",
+        ),
+    ],
+)
+def test_load_sample_small_dataset(
+    fn, archive, exact_loc, class_: str, tmp_data_dir, caplog
+):
+    ds = load_sample(fn, progressbar=False, timeout=30)
+    assert type(ds).__name__ == class_
 
     text = textwrap.dedent(
         f"""
-            'ToroShockTube' is not available locally. Looking up online.
-            Downloading from https://yt-project.org/data/ToroShockTube.tar.gz
+            '{fn.replace('/', os.path.sep)}' is not available locally. Looking up online.
+            Downloading from https://yt-project.org/data/{archive}
             Untaring downloaded file to '{str(tmp_data_dir)}'
-            Parameters: current_time              = 0.2
-            Parameters: domain_dimensions         = [100   1   1]
-            Parameters: domain_left_edge          = [0. 0. 0.]
-            Parameters: domain_right_edge         = [1. 1. 1.]
-            Parameters: cosmological_simulation   = 0
         """
     ).strip("\n")
     expected = [("yt", 20, message) for message in text.split("\n")]
-    assert caplog.record_tuples == expected
+    assert caplog.record_tuples[:3] == expected
 
     caplog.clear()
     # loading a second time should not result in a download request
-    ds2 = load_sample("ToroShockTube")
-    assert isinstance(ds2, EnzoDataset)
+    ds2 = load_sample(fn)
+    assert type(ds2).__name__ == class_
 
-    text = textwrap.dedent(
-        f"""
-            Sample dataset found in '{os.path.join(str(tmp_data_dir), 'ToroShockTube')}'
-            Parameters: current_time              = 0.2
-            Parameters: domain_dimensions         = [100   1   1]
-            Parameters: domain_left_edge          = [0. 0. 0.]
-            Parameters: domain_right_edge         = [1. 1. 1.]
-            Parameters: cosmological_simulation   = 0
-        """
-    ).strip("\n")
-    expected = [("yt", 20, message) for message in text.split("\n")]
-    assert caplog.record_tuples == expected
+    assert caplog.record_tuples[0] == (
+        "yt",
+        20,
+        f"Sample dataset found in '{os.path.join(str(tmp_data_dir), *exact_loc.split('/'))}'",
+    )
 
 
 @requires_module_pytest("pandas", "pooch")
@@ -122,3 +136,35 @@ def test_unknown_filename():
     with pytest.raises(KeyError) as err:
         load_sample(fake_name)
         assert err.exc == f"Could not find '{fake_name}' in the registry."
+
+
+@pytest.fixture()
+def fake_data_dir_in_a_vaccum(tmp_path):
+    pre_test_data_dir = ytcfg["yt", "test_data_dir"]
+    ytcfg.set("yt", "test_data_dir", "/does/not/exist")
+    origin = os.getcwd()
+    os.chdir(tmp_path)
+
+    yield
+
+    ytcfg.set("yt", "test_data_dir", pre_test_data_dir)
+    os.chdir(origin)
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith("win"),
+    reason="can't figure out how to match the warning message in a cross-platform way",
+)
+@requires_module_pytest("pandas", "pooch")
+@pytest.mark.usefixtures("fake_data_dir_in_a_vaccum")
+def test_data_dir_broken():
+    # check that load_sample still works if the test_data_dir
+    # isn't properly set, in which case we should dl to the
+    # cwd and issue a warning.
+    msg = (
+        r"Storage directory from yt config doesn't exist "
+        r"\(currently set to '/does/not/exist'\)\. "
+        r"Current working directory will be used instead\."
+    )
+    with pytest.warns(UserWarning, match=msg):
+        load_sample("ToroShockTube")