Repository: ecmwf/earthkit-data (python)

Issue:
TypeError processing DatetimeJulian coordinates
### What happened?

The following snippet shows an example of earthkit crashing when handling DatetimeJulian coordinates

### What are the steps to reproduce the bug?

```python
import earthkit.data
dataset = "satellite-humidity-profiles"
request = {
    "variable": "all",
    "product_type": "radio_occultation_data",
    "year": 2007,
    "month": "01",
}
earthkit.data.from_source("cds", dataset, request)  # TypeError
```

### Version

0.7.0

### Platform (OS and architecture)

Linux eqc-quality-tools.eqc.compute.cci1.ecmwf.int 5.14.0-362.8.1.el9_3.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Nov 8 17:36:32 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

### Relevant log output

```shell
2024-06-03 12:21:53,670 INFO Welcome to the CDS
2024-06-03 12:21:53,671 INFO Sending request to https://cds.climate.copernicus.eu/api/v2/resources/satellite-humidity-profiles
2024-06-03 12:21:53,750 INFO Request is completed
2024-06-03 12:21:53,751 INFO Downloading https://download-0005-clone.copernicus-climate.eu/cache-compute-0005/cache/data3/dataset-satellite-humidity-profiles-47d24ba0-e481-4f86-bc3d-5838d138dc3e.zip to /data/wp5/.tmp/tmpnjokjzlp/cds-retriever-52b47ff6a52780826a19957ec1d268f266aef501297594a26ed18599adca2910.cache.tmp (51.9K)
2024-06-03 12:21:53,764 INFO Download rate 3.8M/s
                                     
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[1], line 14
      5 collection_id = "satellite-humidity-profiles"
      7 request = {
      8     "variable": "all",
      9     "product_type": "radio_occultation_data",
     10     "year": 2007,
     11     "month": "01",
     12 }
---> 14 earthkit.data.from_source("cds", collection_id, request)

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/sources/__init__.py:155, in from_source(name, lazily, *args, **kwargs)
    153 while src is not prev:
    154     prev = src
--> 155     src = src.mutate()
    156 return src

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/sources/file.py:77, in FileSource.mutate(self)
     73     return FileIndexedSource(self.path, filter=filter, merger=self.merger, **kw)
     75 # Give a chance to directories and zip files
     76 # to return a multi-source
---> 77 source = self._reader.mutate_source()
     78 if source not in (None, self):
     79     source._parent = self

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/sources/file.py:96, in FileSource._reader(self)
     93 @property
     94 def _reader(self):
     95     if self._reader_ is None:
---> 96         self._reader_ = reader(
     97             self, self.path, content_type=self.content_type, parts=self.parts
     98         )
     99     return self._reader_

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/__init__.py:184, in reader(source, path, **kwargs)
    180     magic = f.read(n_bytes)
    182 LOG.debug("Looking for a reader for %s (%s)", path, magic)
--> 184 return _find_reader(
    185     "reader",
    186     source,
    187     path,
    188     magic=magic,
    189     **kwargs,
    190 )

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/__init__.py:132, in _find_reader(method_name, source, path_or_data, **kwargs)
    130         reader = r(source, path_or_data, deeper_check=deeper_check, **kwargs)
    131         if reader is not None:
--> 132             return reader.mutate()
    134 return _unknown(method_name, source, path_or_data, **kwargs)

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/zip.py:74, in ZIPReader.mutate(self)
     71 if self._mutate:
     72     return self._mutate
---> 74 return super().mutate()

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/archive.py:51, in ArchiveReader.mutate(self)
     49 def mutate(self):
     50     if os.path.isdir(self.path):
---> 51         return find_reader(self.source, self.path).mutate()
     53     return self

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/__init__.py:169, in reader(source, path, **kwargs)
    166 if os.path.isdir(path):
    167     from .directory import DirectoryReader
--> 169     return DirectoryReader(source, path).mutate()
    170 LOG.debug("Reader for %s", path)
    172 if not os.path.exists(path):

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/directory.py:64, in DirectoryReader.mutate(self)
     62 def mutate(self):
     63     if len(self._content) == 1:
---> 64         return find_reader(self.source, self._content[0])
     65     return self

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/__init__.py:184, in reader(source, path, **kwargs)
    180     magic = f.read(n_bytes)
    182 LOG.debug("Looking for a reader for %s (%s)", path, magic)
--> 184 return _find_reader(
    185     "reader",
    186     source,
    187     path,
    188     magic=magic,
    189     **kwargs,
    190 )

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/__init__.py:130, in _find_reader(method_name, source, path_or_data, **kwargs)
    126 for deeper_check in (False, True):
    127     # We do two passes, the second one
    128     # allow the plugin to look deeper in the buffer
    129     for name, r in _readers(method_name).items():
--> 130         reader = r(source, path_or_data, deeper_check=deeper_check, **kwargs)
    131         if reader is not None:
    132             return reader.mutate()

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/netcdf/__init__.py:88, in reader(source, path, magic, deeper_check, **kwargs)
     86 if _match_magic(magic, deeper_check):
     87     fs = NetCDFFieldListReader(source, path)
---> 88     if fs.has_fields():
     89         return fs
     90     else:

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/netcdf/fieldlist.py:167, in XArrayFieldListCore.has_fields(self)
    165 def has_fields(self):
    166     if self._fields is None:
--> 167         return get_fields_from_ds(
    168             DataSet(self.xr_dataset),
    169             self.array_backend,
    170             field_type=self.FIELD_TYPE,
    171             check_only=True,
    172         )
    173     else:
    174         return len(self._fields)

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/netcdf/fieldlist.py:127, in get_fields_from_ds(ds, array_backend, field_type, check_only)
    124         use = True
    126     if not use:
--> 127         coordinates.append(OtherCoordinate(c, coord in info))
    129 if not (has_lat and has_lon):
    130     # self.log.info("NetCDFReader: skip %s (Not a 2 field)", name)
    131     continue

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/netcdf/coords.py:65, in Coordinate.__init__(self, variable, info)
     63     self.values = [self.convert(variable.values)]
     64 else:
---> 65     self.values = [self.convert(t) for t in variable.values.flatten()]

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/netcdf/coords.py:65, in <listcomp>(.0)
     63     self.values = [self.convert(variable.values)]
     64 else:
---> 65     self.values = [self.convert(t) for t in variable.values.flatten()]

File /data/common/miniforge3/envs/wp5/lib/python3.11/site-packages/earthkit/data/readers/netcdf/coords.py:28, in as_level(self, level)
     26 if isinstance(level, str):
     27     return level
---> 28 n = float(level)
     29 if int(n) == n:
     30     return int(n)

TypeError: float() argument must be a string or a real number, not 'cftime._cftime.DatetimeJulian'
```


### Accompanying data

_No response_

### Organisation

B-Open / CADS-EQC

Pull request:
Fix netcdf with time coordinate containing cftime
Fixes #399 

Patch:
diff --git a/src/earthkit/data/readers/netcdf/coords.py b/src/earthkit/data/readers/netcdf/coords.py
index 3c3528f..2c8f44b 100644
--- a/src/earthkit/data/readers/netcdf/coords.py
+++ b/src/earthkit/data/readers/netcdf/coords.py
@@ -19,7 +19,14 @@ GEOGRAPHIC_COORDS = {
 
 
 def as_datetime(self, time):
-    return datetime.datetime.strptime(str(time)[:19], "%Y-%m-%dT%H:%M:%S")
+    if isinstance(time, datetime.datetime):
+        return time
+    elif hasattr(time, "isoformat"):
+        return datetime.datetime.fromisoformat(time.isoformat())
+    elif hasattr(time, "to_datetime"):
+        return time.to_datetime()
+    else:
+        return datetime.datetime.strptime(str(time)[:19], "%Y-%m-%dT%H:%M:%S")
 
 
 def as_level(self, level):
diff --git a/src/earthkit/data/readers/netcdf/fieldlist.py b/src/earthkit/data/readers/netcdf/fieldlist.py
index ca62a91..280062e 100644
--- a/src/earthkit/data/readers/netcdf/fieldlist.py
+++ b/src/earthkit/data/readers/netcdf/fieldlist.py
@@ -100,7 +100,12 @@ def get_fields_from_ds(
                 use = True
 
             # Of course, not every one sets the standard_name
-            if standard_name in ["time", "forecast_reference_time"] or long_name in ["time"] or axis == "T":
+            if (
+                standard_name in ["time", "forecast_reference_time"]
+                or long_name in ["time"]
+                or coord_name.lower() in ["time"]
+                or axis == "T"
+            ):
                 # we might not be able to convert time to datetime
                 try:
                     coordinates.append(TimeCoordinate(c, coord in info))
@@ -200,7 +205,8 @@ class XArrayFieldListCore(FieldList):
         )
 
     def to_netcdf(self, *args, **kwargs):
-        """Save the data to a netCDF file.
+        """
+        Save the data to a netCDF file.
 
         Parameters
         ----------


Test patch:
diff --git a/tests/netcdf/test_netcdf_fieldlist.py b/tests/netcdf/test_netcdf_fieldlist.py
index 3615a7a..317aa27 100644
--- a/tests/netcdf/test_netcdf_fieldlist.py
+++ b/tests/netcdf/test_netcdf_fieldlist.py
@@ -12,6 +12,7 @@
 import numpy as np
 
 import earthkit.data
+from earthkit.data.testing import earthkit_remote_test_data_file
 
 
 def test_netcdf_fieldlist_string_coord():
@@ -60,6 +61,17 @@ def test_netcdf_fieldlist_bounds():
     assert len(ds) == 2
 
 
+def test_netcdf_fieldlist_ctime():
+    ds = earthkit.data.from_source(
+        "url",
+        earthkit_remote_test_data_file("test-data", "zgrid_rhgmet_metop_200701_R_2305_0010.nc"),
+    )
+
+    assert len(ds) == 1506
+    assert ds[0].metadata("valid_datetime") == "2007-01-16T00:00:00"
+    assert ds[5].metadata("valid_datetime") == "2007-01-16T00:00:00"
+
+
 if __name__ == "__main__":
     from earthkit.data.testing import main