Repository: DHI/mikeio
Pull request #218 (2021-08-17 17:31:14)

Dfs2 append + equidistant 😮 


Patch:
diff --git a/examples/create_dfs2_with_many_timesteps.py b/examples/create_dfs2_with_many_timesteps.py
new file mode 100644
index 00000000..bb786005
--- /dev/null
+++ b/examples/create_dfs2_with_many_timesteps.py
@@ -0,0 +1,39 @@
+from datetime import datetime
+from mikecore.eum import eumUnit
+from mikeio.eum import EUMType, ItemInfo
+import numpy as np
+
+from mikeio import Dfs2
+
+dfs = Dfs2()
+
+# boogus data, in reality you would get this from some other file, like another dfs2 or netcdf...
+data = [
+    np.full(shape=(1, 200, 800), fill_value=0.0),
+    np.random.random(size=[1, 200, 800]),
+]
+
+nt = 1000
+
+with dfs.write(
+    "long_big_file.dfs2",
+    data,
+    start_time=datetime(2021, 1, 1),
+    dt=3600,
+    items=[
+        ItemInfo(EUMType.Water_Level),
+        ItemInfo(EUMType.Temperature),
+    ],
+    coordinate=["UTM-33", 12.0, 55.0, 0.0],
+    dx=100,
+    dy=200,
+    keep_open=True,
+) as f:
+
+    for i in range(1, nt):
+        # boogus data, in reality you would get this from some other file, like another dfs2 or netcdf...
+        data = [
+            np.full(shape=(1, 200, 800), fill_value=float(i)),
+            np.random.random(size=[1, 200, 800]),
+        ]
+        f.append(data)
diff --git a/mikeio/dataset.py b/mikeio/dataset.py
index 0f9e8ba8..3e5dcdae 100644
--- a/mikeio/dataset.py
+++ b/mikeio/dataset.py
@@ -139,7 +139,7 @@ class Dataset(TimeSeries):
             )
 
         self.data = data
-        self.time = pd.DatetimeIndex(time, freq="infer")
+        self.time = pd.DatetimeIndex(time)
 
         self._items = item_infos
 
@@ -148,6 +148,8 @@ class Dataset(TimeSeries):
         out = ["<mikeio.DataSet>"]
         out.append(f"Dimensions: {self.shape}")
         out.append(f"Time: {self.time[0]} - {self.time[-1]}")
+        if not self.is_equidistant:
+            out.append("-- Non-equidistant calendar axis --")
         if self.n_items > 10:
             out.append(f"Number of items: {self.n_items}")
         else:
@@ -704,8 +706,8 @@ class Dataset(TimeSeries):
         """Is Dataset equidistant in time?"""
         if len(self.time) < 3:
             return True
-
-        return self.time.freq is not None
+        return len(self.time.to_series().diff().dropna().unique()) == 1
+        # return self.time.freq is not None
 
     @property
     def start_time(self):
diff --git a/mikeio/dfs.py b/mikeio/dfs.py
index 90802a0f..cd60dacc 100644
--- a/mikeio/dfs.py
+++ b/mikeio/dfs.py
@@ -115,7 +115,16 @@ class _Dfs123(TimeSeries):
         dfs.Close()
 
     def _write(
-        self, filename, data, start_time, dt, datetimes, items, coordinate, title
+        self,
+        filename,
+        data,
+        start_time,
+        dt,
+        datetimes,
+        items,
+        coordinate,
+        title,
+        keep_open=False,
     ):
 
         if isinstance(data, Dataset) and not data.is_equidistant:
@@ -151,6 +160,7 @@ class _Dfs123(TimeSeries):
             self._start_time = start_time
 
         dfs = self._setup_header(filename)
+        self._dfs = dfs
 
         deletevalue = dfs.FileInfo.DeleteValueFloat  # -1.0000000031710769e-30
 
@@ -169,9 +179,59 @@ class _Dfs123(TimeSeries):
                     d = np.flipud(d)
                     darray = d.reshape(d.size, 1)[:, 0]
 
-                dfs.WriteItemTimeStepNext(0, darray.astype(np.float32))
+                if self._is_equidistant:
+                    dfs.WriteItemTimeStepNext(0, darray.astype(np.float32))
+                else:
+                    t = datetimes[i]
+                    relt = (t - self._start_time).total_seconds()
+                    dfs.WriteItemTimeStepNext(relt, darray.astype(np.float32))
 
-        dfs.Close()
+        if not keep_open:
+            dfs.Close()
+        else:
+            return self
+
+    def append(self, data):
+        """Append to a dfs file opened with `write(...,keep_open=True)`
+
+        Parameters
+        -----------
+        data: list[np.array]
+        """
+
+        deletevalue = self._dfs.FileInfo.DeleteValueFloat  # -1.0000000031710769e-30
+
+        for i in trange(self._n_timesteps, disable=not self.show_progress):
+            for item in range(self._n_items):
+
+                d = data[item][i]
+                d = d.copy()  # to avoid modifying the input
+                d[np.isnan(d)] = deletevalue
+
+                if self._ndim == 1:
+                    darray = d
+
+                if self._ndim == 2:
+                    d = d.reshape(self.shape[1:])
+                    d = np.flipud(d)
+                    darray = d.reshape(d.size, 1)[:, 0]
+
+                if self._is_equidistant:
+                    self._dfs.WriteItemTimeStepNext(0, darray.astype(np.float32))
+                else:
+                    raise NotImplementedError(
+                        "Append is not yet available for non-equidistant files"
+                    )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self._dfs.Close()
+
+    def close(self):
+        "Finalize write for a dfs file opened with `write(...,keep_open=True)`"
+        self._dfs.Close()
 
     def _write_handle_common_arguments(
         self, title, data, items, coordinate, start_time, dt
diff --git a/mikeio/dfs1.py b/mikeio/dfs1.py
index 643ec1fd..cdff6edf 100644
--- a/mikeio/dfs1.py
+++ b/mikeio/dfs1.py
@@ -109,6 +109,5 @@ class Dfs1(_Dfs123):
 
     @property
     def dx(self):
-        """Step size in x direction
-        """
+        """Step size in x direction"""
         return self._dx
diff --git a/mikeio/dfs2.py b/mikeio/dfs2.py
index 383b7f64..3c61e08e 100644
--- a/mikeio/dfs2.py
+++ b/mikeio/dfs2.py
@@ -121,6 +121,7 @@ class Dfs2(_Dfs123):
         dy=None,
         coordinate=None,
         title=None,
+        keep_open=False,
     ):
         """
         Create a dfs2 file
@@ -149,6 +150,8 @@ class Dfs2(_Dfs123):
             e.g. ['LONG/LAT', 12.4387, 55.2257, 327]
         title: str, optional
             title of the dfs2 file. Default is blank.
+        keep_open: bool, optional
+            Keep file open for appending
         """
 
         self._builder = DfsBuilder.Create(title, "mikeio", 0)
@@ -162,7 +165,20 @@ class Dfs2(_Dfs123):
         if dy:
             self._dy = dy
 
-        self._write(filename, data, start_time, dt, datetimes, items, coordinate, title)
+        self._write(
+            filename,
+            data,
+            start_time,
+            dt,
+            datetimes,
+            items,
+            coordinate,
+            title,
+            keep_open,
+        )
+
+        if keep_open:
+            return self
 
     def _set_spatial_axis(self):
         self._builder.SetSpatialAxis(


Test patch:
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
index ba72471a..a9f643ea 100644
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -877,3 +877,20 @@ def test_init():
     assert ds.start_time == datetime(2018, 1, 1, 0, 0, 0)
     assert ds.n_elements == n_elements
     assert ds.items[0].name == "Foo"
+
+
+def test_non_equidistant():
+    nt = 4
+    d = np.random.uniform(size=nt)
+
+    ds = Dataset(
+        data=[d],
+        time=[
+            datetime(2000, 1, 1),
+            datetime(2001, 1, 1),
+            datetime(2002, 1, 1),
+            datetime(2003, 1, 1),
+        ],
+    )
+
+    assert not ds.is_equidistant
diff --git a/tests/test_dfs2.py b/tests/test_dfs2.py
index b3eb8088..484186b7 100644
--- a/tests/test_dfs2.py
+++ b/tests/test_dfs2.py
@@ -481,3 +481,55 @@ def test_write_non_equidistant_data(tmpdir):
     ds3 = dfs2.read()
 
     assert not ds3.is_equidistant
+
+
+def test_incremental_write_from_dfs2(tmpdir):
+    "Useful for writing datasets with many timesteps to avoid problems with out of memory"
+
+    sourcefilename = "tests/testdata/eq.dfs2"
+    outfilename = os.path.join(tmpdir.dirname, "appended.dfs2")
+    dfs = Dfs2(sourcefilename)
+
+    nt = dfs.n_timesteps
+
+    ds = dfs.read(time_steps=[0])
+
+    dfs_to_write = Dfs2()
+    dfs_to_write.write(outfilename, ds, dt=dfs.timestep, keep_open=True)
+
+    for i in range(1, nt):
+        ds = dfs.read(time_steps=[i])
+        dfs_to_write.append(ds)
+
+    dfs_to_write.close()
+
+    newdfs = Dfs2(outfilename)
+    assert dfs.start_time == newdfs.start_time
+    assert dfs.timestep == newdfs.timestep
+    assert dfs.end_time == newdfs.end_time
+
+
+def test_incremental_write_from_dfs2_context_manager(tmpdir):
+    "Useful for writing datasets with many timesteps to avoid problems with out of memory"
+
+    sourcefilename = "tests/testdata/eq.dfs2"
+    outfilename = os.path.join(tmpdir.dirname, "appended.dfs2")
+    dfs = Dfs2(sourcefilename)
+
+    nt = dfs.n_timesteps
+
+    ds = dfs.read(time_steps=[0])
+
+    dfs_to_write = Dfs2()
+    with dfs_to_write.write(outfilename, ds, dt=dfs.timestep, keep_open=True) as f:
+
+        for i in range(1, nt):
+            ds = dfs.read(time_steps=[i])
+            f.append(ds)
+
+        # dfs_to_write.close() # called automagically by context manager
+
+    newdfs = Dfs2(outfilename)
+    assert dfs.start_time == newdfs.start_time
+    assert dfs.timestep == newdfs.timestep
+    assert dfs.end_time == newdfs.end_time
\ No newline at end of file
diff --git a/tests/test_dfsu.py b/tests/test_dfsu.py
index 5dd2ad88..a6fce221 100644
--- a/tests/test_dfsu.py
+++ b/tests/test_dfsu.py
@@ -693,6 +693,7 @@ def test_write_from_dfsu(tmpdir):
 
 
 def test_incremental_write_from_dfsu(tmpdir):
+    "Useful for writing datasets with many timesteps to avoid problems with out of memory"
 
     sourcefilename = os.path.join("tests", "testdata", "HD2D.dfsu")
     outfilename = os.path.join(tmpdir.dirname, "simple.dfsu")