Repository: yt-project/yt

Issue:
support pathlike arguments for user-specificed files/directories
### Bug report

So in https://github.com/yt-project/yt/pull/2405 I've added support for loading datafiles specified as `pathlib.Path` objects. I'm realizing the global api would feel more consistent if that was also supported by other functions accepting files/directories as arguments.

I can currently think of 2 places where this change would apply, `yt.utilities_load_sample()` and `yt.funcs.enable_plugins()`.

Please report any other similar api parts that would benefit from this change.

(In the spirit of the dev meeting we had yesterday and the recent email distributed to users, I took the liberty of creating an "api-consistency" label)


Discussion:
So one place we could add support for pathlike arguments would be `DatasetSeries.from_filenames`, which is part of `yt.load()`.

Resolving patch:
diff --git a/yt/data_objects/time_series.py b/yt/data_objects/time_series.py
--- a/yt/data_objects/time_series.py
+++ b/yt/data_objects/time_series.py
@@ -12,7 +12,7 @@
 from yt.data_objects.particle_trajectories import ParticleTrajectories
 from yt.funcs import ensure_list, issue_deprecation_warning, iterable, mylog
 from yt.units.yt_array import YTArray, YTQuantity
-from yt.utilities.exceptions import YTException, YTUnidentifiedDataType
+from yt.utilities.exceptions import YTException
 from yt.utilities.object_registries import (
     analysis_task_registry,
     data_object_registry,
@@ -55,22 +55,6 @@ def _eval(params, ds):
     return cls
 
 
-def get_filenames_from_glob_pattern(outputs):
-    """
-    Helper function to DatasetSeries.__new__
-    handle a special case where "outputs" is assumed to be really a pattern string
-    """
-    pattern = outputs
-    epattern = os.path.expanduser(pattern)
-    data_dir = ytcfg.get("yt", "test_data_dir")
-    # if not match if found from the current work dir,
-    # we try to match the pattern from the test data dir
-    file_list = glob.glob(epattern) or glob.glob(os.path.join(data_dir, epattern))
-    if not file_list:
-        raise OSError(f"No match found for pattern : {pattern}")
-    return sorted(file_list)
-
-
 attrs = (
     "refine_by",
     "dimensionality",
@@ -113,10 +97,11 @@ class DatasetSeries:
 
     Parameters
     ----------
-    outputs : list or pattern
+    outputs : list of filenames, or pattern
         A list of filenames, for instance ["DD0001/DD0001", "DD0002/DD0002"],
         or a glob pattern (i.e. containing wildcards '[]?!*') such as "DD*/DD*.index".
         In the latter case, results are sorted automatically.
+        Filenames and patterns can be of type str, os.Pathlike or bytes.
     parallel : True, False or int
         This parameter governs the behavior when .piter() is called on the
         resultant DatasetSeries object.  If this is set to False, the time
@@ -162,13 +147,12 @@ def __init_subclass__(cls, *args, **kwargs):
             mylog.debug("Registering simulation: %s as %s", code_name, cls)
 
     def __new__(cls, outputs, *args, **kwargs):
-        if isinstance(outputs, str):
-            outputs = get_filenames_from_glob_pattern(outputs)
-        ret = super(DatasetSeries, cls).__new__(cls)
         try:
-            ret._pre_outputs = outputs[:]
-        except TypeError as e:
-            raise YTUnidentifiedDataType(outputs, *args, **kwargs) from e
+            outputs = cls._get_filenames_from_glob_pattern(outputs)
+        except TypeError:
+            pass
+        ret = super(DatasetSeries, cls).__new__(cls)
+        ret._pre_outputs = outputs[:]
         return ret
 
     def __init__(
@@ -199,14 +183,30 @@ def _null(x):
         self.parallel = parallel
         self.kwargs = kwargs
 
+    @staticmethod
+    def _get_filenames_from_glob_pattern(outputs):
+        """
+        Helper function to DatasetSeries.__new__
+        handle a special case where "outputs" is assumed to be really a pattern string
+        """
+        pattern = outputs
+        epattern = os.path.expanduser(pattern)
+        data_dir = ytcfg.get("yt", "test_data_dir")
+        # if not match if found from the current work dir,
+        # we try to match the pattern from the test data dir
+        file_list = glob.glob(epattern) or glob.glob(os.path.join(data_dir, epattern))
+        if not file_list:
+            raise FileNotFoundError(f"No match found for pattern : {pattern}")
+        return sorted(file_list)
+
     def __iter__(self):
         # We can make this fancier, but this works
         for o in self._pre_outputs:
-            if isinstance(o, str):
+            try:
                 ds = self._load(o, **self.kwargs)
                 self._setup_function(ds)
                 yield ds
-            else:
+            except TypeError:
                 yield o
 
     def __getitem__(self, key):
@@ -218,9 +218,11 @@ def __getitem__(self, key):
                 self._pre_outputs[key], parallel=self.parallel, **self.kwargs
             )
         o = self._pre_outputs[key]
-        if isinstance(o, str):
+        try:
             o = self._load(o, **self.kwargs)
             self._setup_function(o)
+        except TypeError:
+            pass
         return o
 
     def __len__(self):
@@ -535,7 +537,7 @@ def __init__(self, parameter_filename, find_outputs=False):
         """
 
         if not os.path.exists(parameter_filename):
-            raise OSError(parameter_filename)
+            raise FileNotFoundError(parameter_filename)
         self.parameter_filename = parameter_filename
         self.basename = os.path.basename(parameter_filename)
         self.directory = os.path.dirname(parameter_filename)


Test patch:
diff --git a/yt/data_objects/tests/test_time_series.py b/yt/data_objects/tests/test_time_series.py
--- a/yt/data_objects/tests/test_time_series.py
+++ b/yt/data_objects/tests/test_time_series.py
@@ -1,27 +1,65 @@
-import os
 import tempfile
 from pathlib import Path
 
-from yt.data_objects.time_series import get_filenames_from_glob_pattern
+from yt.data_objects.time_series import DatasetSeries
 from yt.testing import assert_raises
+from yt.utilities.exceptions import YTUnidentifiedDataType
 
 
 def test_pattern_expansion():
     file_list = [f"fake_data_file_{str(i).zfill(4)}" for i in range(10)]
 
     with tempfile.TemporaryDirectory() as tmpdir:
+        tmp_path = Path(tmpdir)
         for file in file_list:
-            (Path(tmpdir) / file).touch()
+            (tmp_path / file).touch()
 
-        pattern = os.path.join(tmpdir, "fake_data_file_*")
-        found = get_filenames_from_glob_pattern(pattern)
-        assert found == [os.path.join(tmpdir, file) for file in file_list]
+        pattern = tmp_path / "fake_data_file_*"
+        expected = [str(tmp_path / file) for file in file_list]
+        found = DatasetSeries._get_filenames_from_glob_pattern(pattern)
+        assert found == expected
 
-        found2 = get_filenames_from_glob_pattern(Path(pattern))
-        assert found2 == found
+        found2 = DatasetSeries._get_filenames_from_glob_pattern(Path(pattern))
+        assert found2 == expected
 
 
 def test_no_match_pattern():
     with tempfile.TemporaryDirectory() as tmpdir:
-        pattern = os.path.join(tmpdir, "fake_data_file_*")
-        assert_raises(OSError, get_filenames_from_glob_pattern, pattern)
+        pattern = Path(tmpdir).joinpath("fake_data_file_*")
+        assert_raises(
+            FileNotFoundError, DatasetSeries._get_filenames_from_glob_pattern, pattern
+        )
+
+
+def test_init_fake_dataseries():
+
+    file_list = [f"fake_data_file_{str(i).zfill(4)}" for i in range(10)]
+    with tempfile.TemporaryDirectory() as tmpdir:
+        pfile_list = [Path(tmpdir) / file for file in file_list]
+        sfile_list = [str(file) for file in pfile_list]
+        for file in pfile_list:
+            file.touch()
+        pattern = Path(tmpdir) / "fake_data_file_*"
+
+        # init from str pattern
+        ts = DatasetSeries(pattern)
+        assert ts._pre_outputs == sfile_list
+
+        # init from Path pattern
+        ppattern = Path(pattern)
+        ts = DatasetSeries(ppattern)
+        assert ts._pre_outputs == sfile_list
+
+        # init form str list
+        ts = DatasetSeries(sfile_list)
+        assert ts._pre_outputs == sfile_list
+
+        # init form Path list
+        ts = DatasetSeries(pfile_list)
+        assert ts._pre_outputs == pfile_list
+
+        # rejected input type (str repr of a list) "[file1, file2, ...]"
+        assert_raises(FileNotFoundError, DatasetSeries, str(file_list))
+
+        # finally, check that ts[0] fails to actually load
+        assert_raises(YTUnidentifiedDataType, ts.__getitem__, 0)