Repository: ecmwf/earthkit-data
Pull request #635 (2025-03-04 15:05:58)

Use field metadata in file-pattern target


Patch:
diff --git a/docs/release_notes/deprecations.rst b/docs/release_notes/deprecations.rst
index a9bfec6..de47f36 100644
--- a/docs/release_notes/deprecations.rst
+++ b/docs/release_notes/deprecations.rst
@@ -76,7 +76,7 @@ This functionality is now replaced by the :ref:`targets <data-targets>`.
    * - New code
    * -
 
-        .. literalinclude:: include/migrated_data_write.py
+       .. literalinclude:: include/migrated_data_write.py
 
 
 
@@ -105,6 +105,38 @@ This functionality is now replaced by the :ref:`targets <data-targets>`.
         .. literalinclude:: include/migrated_new_grib_output.py
 
 
+The ``split_output=True`` option of  :func:`new_grib_output` is not supported by the :ref:`file <targets-file>` target but implemented by the :ref:`file-pattern <targets-file-pattern>` target.
+
+.. list-table::
+   :header-rows: 0
+
+   * - Deprecated code
+   * -
+
+        .. literalinclude:: include/deprec_new_grib_output_split.py
+
+   * - New code
+   * -
+
+        .. literalinclude:: include/migrated_new_grib_output_split.py
+
+
+.. warning::
+
+    When using :func:`new_grib_output` the ``{param}`` pattern substitutes the value of the ``"param"`` ecCodes key from the GRIB header. However, with the :ref:`targets <data-targets>` the ``{param}`` pattern substitutes the value of the ``"shortName"`` key. This is to match the behaviour of ``Field.metadata("param")``, which always returns the value of the ``"shortName"``. If you still want to use the value of the ``"param"`` ecCodes key you need to use the ``{mars.param}`` pattern instead.
+
+
+    .. code-block:: python
+
+        # Deprecated code
+        new_grib_output("file", "output_{param}.grib", split_output=True)
+        ...
+
+        # New code
+        to_target("file-pattern", "output_{mars.param}.grib")
+
+
+
 .. _deprecated-griboutput:
 
 :py:class:`GribOutput` is deprecated
diff --git a/docs/release_notes/include/deprec_new_grib_output_split.py b/docs/release_notes/include/deprec_new_grib_output_split.py
new file mode 100644
index 0000000..7c03ff9
--- /dev/null
+++ b/docs/release_notes/include/deprec_new_grib_output_split.py
@@ -0,0 +1,9 @@
+import earthkit.data as ekd
+
+ds = ekd.from_source("sample", "test.grib")
+
+# case 1
+o = ekd.new_grib_output("res_{shortName}.grib", split_output=True)
+for field in ds:
+    o.write(None, template=field)
+o.close()
diff --git a/docs/release_notes/include/migrated_new_grib_output_split.py b/docs/release_notes/include/migrated_new_grib_output_split.py
new file mode 100644
index 0000000..13d663a
--- /dev/null
+++ b/docs/release_notes/include/migrated_new_grib_output_split.py
@@ -0,0 +1,12 @@
+import earthkit.data as ekd
+
+ds = ekd.from_source("sample", "test.grib")
+
+# case 1 - equivalent solutions
+t = ekd.create_target("file-pattern", "res_{shortName}.grib")
+for field in ds:
+    t.write(field)
+
+t.close()
+
+ds.to_target("file-pattern", "res_{shortName}.grib")
diff --git a/src/earthkit/data/encoders/grib.py b/src/earthkit/data/encoders/grib.py
index 9afb9d1..74ef47a 100644
--- a/src/earthkit/data/encoders/grib.py
+++ b/src/earthkit/data/encoders/grib.py
@@ -40,7 +40,7 @@ class GribEncodedData(EncodedData):
 
     def metadata(self, key=None):
         if key:
-            return self.handle.get(key)
+            return self.to_field().metadata(key, default=None)
         else:
             raise NotImplementedError
 
diff --git a/src/earthkit/data/targets/file_pattern.py b/src/earthkit/data/targets/file_pattern.py
index d027206..72dfa61 100644
--- a/src/earthkit/data/targets/file_pattern.py
+++ b/src/earthkit/data/targets/file_pattern.py
@@ -48,6 +48,16 @@ class FilePatternTarget(SimpleTarget):
 
         self.split_output = re.findall(r"\{(.*?)\}", self.filename)
 
+        # change format pattern to use positional arguments. This allows using keys
+        # containing a dot, e.g. mars.param
+        for i, s in enumerate(self.split_output):
+            if ":" in s:
+                pos_key = f"{i}:" + s.split(":")[1]
+            else:
+                pos_key = f"{i}"
+
+            self.filename = self.filename.replace(f"{s}", pos_key)
+
     def close(self):
         """Close the target and closing all the files.
 
@@ -73,8 +83,8 @@ class FilePatternTarget(SimpleTarget):
     def _f(self, data):
         self._raise_if_closed()
 
-        keys = {k.split(":")[0]: data.metadata(k.split(":")[0]) for k in self.split_output}
-        path = self.filename.format(**keys)
+        keys = [data.metadata(k.split(":")[0]) for k in self.split_output]
+        path = self.filename.format(*keys)
 
         if path not in self._files:
             flag = "wb" if not self.append else "ab"


Test patch:
diff --git a/tests/targets/test_target_file_pattern.py b/tests/targets/test_target_file_pattern.py
index 7a50821..fa65e6b 100644
--- a/tests/targets/test_target_file_pattern.py
+++ b/tests/targets/test_target_file_pattern.py
@@ -52,6 +52,8 @@ def test_target_file_pattern_grib_core(kwargs, direct_call):
     "pattern,expected_value",
     [
         ("{shortName}", {"t": 2, "u": 2, "v": 2}),
+        ("{param}", {"t": 2, "u": 2, "v": 2}),
+        ("{mars.param}", {"130.128": 2, "131.128": 2, "132.128": 2}),
         ("{shortName}_{level}", {"t_1000": 1, "t_850": 1, "u_1000": 1, "u_850": 1, "v_1000": 1, "v_850": 1}),
         ("{date}_{time}_{step}", {"20180801_1200_0": 6}),
         ("{date}_{time}_{step:03}", {"20180801_1200_000": 6}),