Repository: ecmwf/earthkit-data

Issue:
xarray converter fails with a list of ZIP files containing a single-file ZIP
### What happened?

The code below fails, but it works if I exclude 2025 or if only 2025 is selected. I believe there's a bug that gets triggered when earthkit.data tries to open a list of ZIP files containing a single-file ZIP (i.e., a ZIP archive with only one file inside).

(I will try to add a simpler MRE later to confirm my hypothesis)

### What are the steps to reproduce the bug?

```python
import earthkit.data

collection_id = "reanalysis-oras5"
request = {
    "product_type": ["operational"],
    "vertical_resolution": "single_level",
    "variable": ["ocean_heat_content_for_the_upper_300m"],
    "year": ["2023", "2024", "2025"],
    "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
}

ds = earthkit.data.from_source("cds", collection_id, **request, split_on="year").to_xarray()
```

### Version

0.13.1

### Platform (OS and architecture)

Darwin MacBook-Pro-di-Bopen.local 24.3.0 Darwin Kernel Version 24.3.0: Thu Jan  2 20:24:24 PST 2025; root:xnu-11215.81.4~3/RELEASE_ARM64_T6030 arm64

### Relevant log output

```shell
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[2], line 12
      3 collection_id = "reanalysis-oras5"
      4 request = {
      5     "product_type": ["operational"],
      6     "vertical_resolution": "single_level",
   (...)      9     "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
     10 }
---> 12 ds = earthkit.data.from_source("cds", collection_id, **request, split_on="year").to_xarray()

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/earthkit/data/sources/multi.py:109, in MultiSource.to_xarray(self, **kwargs)
    108 def to_xarray(self, **kwargs):
--> 109     return make_merger(self.merger, self.sources).to_xarray(**kwargs)

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/earthkit/data/mergers/__init__.py:109, in DefaultMerger.to_xarray(self, **kwargs)
    106 def to_xarray(self, **kwargs):
    107     from .xarray import merge
--> 109     return merge(
    110         sources=self.sources,
    111         paths=self.paths,
    112         reader_class=self.reader_class,
    113         **kwargs,
    114     )

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/earthkit/data/mergers/xarray.py:75, in merge(sources, paths, reader_class, **kwargs)
     73 if paths is not None:
     74     if reader_class is not None and hasattr(reader_class, "to_xarray_multi_from_paths"):
---> 75         return reader_class.to_xarray_multi_from_paths(
     76             paths,
     77             **options,
     78         )
     80     LOG.debug(f"xr.open_mfdataset with options={options}")
     81     return xr.open_mfdataset(paths, **options)

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/earthkit/data/readers/netcdf/__init__.py:73, in NetCDFReader.to_xarray_multi_from_paths(cls, paths, **kwargs)
     70 if not options:
     71     options = dict(**kwargs)
---> 73 return xr.open_mfdataset(
     74     paths,
     75     **options,
     76 )

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/xarray/backends/api.py:1634, in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
   1631     open_ = open_dataset
   1632     getattr_ = getattr
-> 1634 datasets = [open_(p, **open_kwargs) for p in paths1d]
   1635 closers = [getattr_(ds, "_close") for ds in datasets]
   1636 if preprocess is not None:

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/xarray/backends/api.py:1634, in <listcomp>(.0)
   1631     open_ = open_dataset
   1632     getattr_ = getattr
-> 1634 datasets = [open_(p, **open_kwargs) for p in paths1d]
   1635 closers = [getattr_(ds, "_close") for ds in datasets]
   1636 if preprocess is not None:

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/xarray/backends/api.py:667, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
    664     kwargs.update(backend_kwargs)
    666 if engine is None:
--> 667     engine = plugins.guess_engine(filename_or_obj)
    669 if from_array_kwargs is None:
    670     from_array_kwargs = {}

File ~/miniforge3/envs/earthkit-data/lib/python3.11/site-packages/xarray/backends/plugins.py:194, in guess_engine(store_spec)
    186 else:
    187     error_msg = (
    188         "found the following matches with the input file in xarray's IO "
    189         f"backends: {compatible_engines}. But their dependencies may not be installed, see:\n"
    190         "https://docs.xarray.dev/en/stable/user-guide/io.html \n"
    191         "https://docs.xarray.dev/en/stable/getting-started-guide/installing.html"
    192     )
--> 194 raise ValueError(error_msg)

ValueError: did not find a match in any of xarray's currently installed IO backends ['netcdf4', 'scipy', 'cfgrib', 'earthkit']. Consider explicitly selecting one of the installed engines via the ``engine`` parameter, or installing additional IO dependencies, see:
https://docs.xarray.dev/en/stable/getting-started-guide/installing.html
https://docs.xarray.dev/en/stable/user-guide/io.html
```

### Accompanying data

_No response_

### Organisation

B-Open / EQC

Hints from issue comments:
@malmans2: Here is a simpler MRE:
```python
import tempfile
import zipfile

import earthkit.data
import xarray as xr

N = 3
ds = xr.DataArray(range(N), coords={"dim_0": range(N)}, name="foo").to_dataset()

with tempfile.TemporaryDirectory() as tmpdir:
    zipnames = set()
    for i, ds_i in ds.groupby("dim_0"):
        ds_i.to_netcdf(ncname := f"{tmpdir}/{i}.nc")

        zipnames.add(zipname := f"{tmpdir}/{i // 2}.zip")
        with zipfile.ZipFile(zipname, "a") as zipf:
            zipf.write(ncname)

    ek_ds = earthkit.data.from_source("file", list(zipnames)).to_xarray()
```

It only crashes when there's a single-file ZIP in the list (i.e., it crashes when `N=3` but succeed when `N=4`)@sandorkertesz: @malmans2 thank you very much for reporting this issue and providing the tests. I could reproduce this problem and now looking into providing a fix for it.@sandorkertesz: @malmans2, a hotfix release 0.13.3 is now available on PyPI  fixing this issue.

Solution patch:
diff --git a/docs/release_notes/version_0.13_updates.rst b/docs/release_notes/version_0.13_updates.rst
index e7debcf..c7efb95 100644
--- a/docs/release_notes/version_0.13_updates.rst
+++ b/docs/release_notes/version_0.13_updates.rst
@@ -1,6 +1,14 @@
 Version 0.13 Updates
 /////////////////////////
 
+Version 0.13.3
+===============
+
+Fixes
+++++++++++
+
+- Fixed issue when could not convert ZIP containing a single NetCDF file to Xarray (:pr:`650`)
+
 
 Version 0.13.2
 ===============
diff --git a/src/earthkit/data/readers/directory.py b/src/earthkit/data/readers/directory.py
index 13084f1..1a280bc 100644
--- a/src/earthkit/data/readers/directory.py
+++ b/src/earthkit/data/readers/directory.py
@@ -15,7 +15,6 @@ import shutil
 from earthkit.data import from_source
 
 from . import Reader
-from . import reader as find_reader
 
 LOG = logging.getLogger(__name__)
 
@@ -59,8 +58,6 @@ class DirectoryReader(Reader):
                     self._content.append(full)
 
     def mutate(self):
-        if len(self._content) == 1:
-            return find_reader(self.source, self._content[0])
         return self
 
     def mutate_source(self):
@@ -69,6 +66,17 @@ class DirectoryReader(Reader):
                 raise ValueError("Cannot stream zarr directories")
             return from_source("zarr", self.path)
 
+        if len(self._content) == 1:
+            return from_source(
+                "file",
+                path=self._content[0],
+                filter=self.filter,
+                merger=self.merger,
+                stream=self.stream,
+                parts=self.parts,
+                **self.source._kwargs,
+            )
+
         return from_source(
             "multi",
             [
diff --git a/src/earthkit/data/testing.py b/src/earthkit/data/testing.py
index 9a0ea4f..1c0ac04 100644
--- a/src/earthkit/data/testing.py
+++ b/src/earthkit/data/testing.py
@@ -220,10 +220,18 @@ if not NO_CUPY:
 def make_tgz(target_dir, target_name, paths):
     import tarfile
 
-    tar = tarfile.open(os.path.join(target_dir, target_name), "w:gz")
-    for p in paths:
-        tar.add(p)
-    tar.close()
+    with tarfile.open(os.path.join(target_dir, target_name), "w:gz") as tar:
+        for p in paths:
+            tar.add(p)
+    # tar.close()
+
+
+def make_zip(target_dir, target_name, paths):
+    import zipfile
+
+    with zipfile.ZipFile(os.path.join(target_dir, target_name), "a") as zipf:
+        for p in paths:
+            zipf.write(p)
 
 
 WRITE_TO_FILE_METHODS = ["target", "save", "write"]


Test patch:
diff --git a/tests/sources/test_file.py b/tests/sources/test_file.py
index 0ec33d8..61f9b86 100644
--- a/tests/sources/test_file.py
+++ b/tests/sources/test_file.py
@@ -11,6 +11,7 @@
 
 import logging
 import os
+import shutil
 
 import pytest
 
@@ -19,6 +20,7 @@ from earthkit.data.core.temporary import temp_directory
 from earthkit.data.testing import WRITE_TO_FILE_METHODS
 from earthkit.data.testing import earthkit_examples_file
 from earthkit.data.testing import make_tgz
+from earthkit.data.testing import make_zip
 from earthkit.data.testing import preserve_cwd
 from earthkit.data.testing import write_to_file
 
@@ -325,6 +327,76 @@ def test_file_multi_directory_with_tar(write_method):
             assert ds.metadata(("param", "level")) == ref
 
 
+def test_file_grib_tar_with_single_file():
+    with temp_directory() as tmpdir:
+        path = os.path.join(tmpdir, "a.grib")
+        shutil.copy(earthkit_examples_file("test.grib"), path)
+        make_tgz(tmpdir, "test.tar.gz", [path])
+        t_path = os.path.join(tmpdir, "test.tar.gz")
+
+        ds = from_source("file", t_path)
+        assert len(ds) == 2, len(ds)
+        assert os.path.basename(ds.path) == "a.grib"
+
+
+def test_file_grib_zip_with_single_file():
+    with temp_directory() as tmpdir:
+        path = os.path.join(tmpdir, "a.grib")
+        shutil.copy(earthkit_examples_file("test.grib"), path)
+        make_zip(tmpdir, "test.zip", [path])
+        t_path = os.path.join(tmpdir, "test.zip")
+
+        ds = from_source("file", t_path)
+        assert len(ds) == 2, len(ds)
+        assert os.path.basename(ds.path) == "a.grib"
+
+
+def test_file_netcdf_tar_with_single_file():
+    with temp_directory() as tmpdir:
+        path = os.path.join(tmpdir, "a.nc")
+        shutil.copy(earthkit_examples_file("test.nc"), path)
+        make_tgz(tmpdir, "test.tar.gz", [path])
+        t_path = os.path.join(tmpdir, "test.tar.gz")
+
+        ds = from_source("file", t_path)
+        assert len(ds) == 2, len(ds)
+        assert os.path.basename(ds.path) == "a.nc"
+
+
+def test_file_netcdf_zip_with_single_file_1():
+    with temp_directory() as tmpdir:
+        path = os.path.join(tmpdir, "a.nc")
+        shutil.copy(earthkit_examples_file("test.nc"), path)
+        make_zip(tmpdir, "test.zip", [path])
+        ar_path = os.path.join(tmpdir, "test.zip")
+
+        ds = from_source("file", ar_path)
+        assert len(ds) == 2, len(ds)
+        assert os.path.basename(ds.path) == "a.nc"
+
+
+def test_file_netcdf_zip_with_single_file_2():
+    # This NetCDF is not read into a fieldlist
+    import xarray as xr
+
+    N = 3
+    ds_in = xr.DataArray(range(N), coords={"dim_0": range(N)}, name="foo").to_dataset()
+    # print(ds_in)
+
+    with temp_directory() as tmpdir:
+
+        path = os.path.join(tmpdir, "a.nc")
+        ds_in["foo"].to_netcdf(path)
+
+        make_zip(tmpdir, "test.zip", [path])
+        z_path = os.path.join(tmpdir, "test.zip")
+
+        ds = from_source("file", z_path)
+
+        # print(f"{ds=}")
+        assert os.path.basename(ds.path) == "a.nc"
+
+
 if __name__ == "__main__":
     from earthkit.data.testing import main