Repository: DHI/mikeio
Pull request #346 (2022-05-18 09:16:57)

Error when creating a dataset with many items


Patch:
diff --git a/mikeio/dataarray.py b/mikeio/dataarray.py
index 92ea8ef8..3a9e38d8 100644
--- a/mikeio/dataarray.py
+++ b/mikeio/dataarray.py
@@ -1485,7 +1485,7 @@ class DataArray(DataUtilsMixin, TimeSeries):
                 qd.name = newname
                 res.append(qd)
 
-            return Dataset(res)
+            return Dataset(data=res, validate=False)
 
     # ============= MATH operations ===========
 
diff --git a/mikeio/dataset.py b/mikeio/dataset.py
index c9863cbc..750ae193 100644
--- a/mikeio/dataset.py
+++ b/mikeio/dataset.py
@@ -88,12 +88,13 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
         geometry: _Geometry = None,
         zn=None,
         dims=None,
+        validate=True,
     ):
         if not self._is_DataArrays(data):
             data = self._create_dataarrays(
                 data=data, time=time, items=items, geometry=geometry, zn=zn, dims=dims
             )
-        return self._init_from_DataArrays(data, validate=True)
+        return self._init_from_DataArrays(data, validate=validate)
 
     @staticmethod
     def _is_DataArrays(data):
@@ -140,7 +141,8 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
 
         if (len(self) > 1) and validate:
             first = self[0]
-            for da in self[1:]:
+            for i in range(1, len(self)):
+                da = self[i]
                 first._is_compatible(da, raise_error=True)
 
         self._check_all_different_ids(self._data_vars.values())
@@ -437,7 +439,7 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
         """
         res = {name: da.squeeze() for name, da in self._data_vars.items()}
 
-        return Dataset(res)
+        return Dataset(data=res, validate=False)
 
     # TODO: delete this?
     @staticmethod
@@ -631,7 +633,7 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
             data_vars = {}
             for v in key:
                 data_vars[v] = self._data_vars[v]
-            return Dataset(data_vars)
+            return Dataset(data=data_vars, validate=False)
 
         raise TypeError(f"indexing with a {type(key)} is not (yet) supported")
 
@@ -733,7 +735,7 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
         1985-08-06 07:00:00 - 1985-08-06 12:00:00
         """
         res = [da.isel(idx=idx, axis=axis, **kwargs) for da in self]
-        return Dataset(res)
+        return Dataset(data=res, validate=False)
 
     def sel(
         self,
@@ -747,7 +749,7 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
         ds.sel(area=[1., 12., 2., 15.])
         """
         res = [da.sel(**kwargs) for da in self]
-        return Dataset(res)
+        return Dataset(data=res, validate=False)
 
     def interp(
         self,
@@ -776,9 +778,9 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
                 das = [da.interp(x=x, y=y, interpolant=interpolant) for da in self]
             else:
                 das = [da.interp(x=x, y=y) for da in self]
-            ds = Dataset(das)
+            ds = Dataset(das, validate=False)
         else:
-            ds = Dataset([da for da in self])
+            ds = Dataset([da for da in self], validate=False)
 
         # interp in time
         if time is not None:
@@ -915,7 +917,7 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
 
         interpolant = self.geometry.get_2d_interpolant(xy, **kwargs)
         das = [da.interp_like(geom, interpolant=interpolant) for da in self]
-        ds = Dataset(das)
+        ds = Dataset(das, validate=False)
 
         if hasattr(other, "time"):
             ds = ds.interp_time(other.time)
@@ -1121,7 +1123,7 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
             for name, da in self._data_vars.items()
         }
 
-        return Dataset(res)
+        return Dataset(data=res, validate=False)
 
     def quantile(self, q, *, axis="time", **kwargs) -> "Dataset":
         """Compute the q-th quantile of the data along the specified axis.
@@ -1193,7 +1195,7 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
                     qd.name = newname
                     res.append(qd)
 
-        return Dataset(res)
+        return Dataset(data=res, validate=False)
 
     def max(self, axis="time") -> "Dataset":
         """Max value along an axis
@@ -1404,7 +1406,12 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
         items = deepcopy(self.items)
         time = self.time.copy()
         return Dataset(
-            data, time=time, items=items, geometry=self.geometry, zn=self._zn
+            data,
+            time=time,
+            items=items,
+            geometry=self.geometry,
+            zn=self._zn,
+            validate=False,
         )
 
     def _multiply_value(self, value) -> "Dataset":
@@ -1415,7 +1422,12 @@ class Dataset(DataUtilsMixin, TimeSeries, collections.abc.MutableMapping):
         items = deepcopy(self.items)
         time = self.time.copy()
         return Dataset(
-            data, time=time, items=items, geometry=self.geometry, zn=self._zn
+            data,
+            time=time,
+            items=items,
+            geometry=self.geometry,
+            zn=self._zn,
+            validate=False,
         )
 
     # ===============================================
diff --git a/mikeio/dfs.py b/mikeio/dfs.py
index e96155f9..816f34c1 100644
--- a/mikeio/dfs.py
+++ b/mikeio/dfs.py
@@ -113,7 +113,7 @@ class _Dfs123(TimeSeries):
         items = _get_item_info(self._dfs.ItemInfo, item_numbers)
 
         self._dfs.Close()
-        return Dataset(data_list, time, items, geometry=self.geometry)
+        return Dataset(data_list, time, items, geometry=self.geometry, validate=False)
 
     def _read_header(self):
         dfs = self._dfs
diff --git a/mikeio/dfs0.py b/mikeio/dfs0.py
index d4bc258f..e1d12bf4 100644
--- a/mikeio/dfs0.py
+++ b/mikeio/dfs0.py
@@ -252,7 +252,7 @@ class Dfs0(TimeSeries):
             for item in self._dfs.ItemInfo
         ]
 
-        return Dataset(data, time, items)
+        return Dataset(data, time, items, validate=False)
 
     @staticmethod
     def _to_dfs_datatype(dtype):
diff --git a/mikeio/dfs2.py b/mikeio/dfs2.py
index 55043690..09be518c 100644
--- a/mikeio/dfs2.py
+++ b/mikeio/dfs2.py
@@ -298,7 +298,9 @@ class Dfs2(_Dfs123):
         self._dfs.Close()
 
         time = pd.to_datetime(t_seconds, unit="s", origin=self.start_time)
-        return Dataset(data_list, time=time, items=items, geometry=geometry)
+        return Dataset(
+            data_list, time=time, items=items, geometry=geometry, validate=False
+        )
 
     def find_nearest_elements(
         self,
diff --git a/mikeio/dfs3.py b/mikeio/dfs3.py
index 9e3b9da2..1314a1f7 100644
--- a/mikeio/dfs3.py
+++ b/mikeio/dfs3.py
@@ -272,7 +272,9 @@ class Dfs3(_Dfs123):
 
         time = pd.to_datetime(t_seconds, unit="s", origin=self.start_time)
         items = _get_item_info(dfs.ItemInfo, item_numbers)
-        return Dataset(data_list, time=time, items=items, geometry=geometry)
+        return Dataset(
+            data_list, time=time, items=items, geometry=geometry, validate=False
+        )
 
     def write(
         self,
diff --git a/mikeio/dfsu.py b/mikeio/dfsu.py
index 82e02a48..8bb01832 100644
--- a/mikeio/dfsu.py
+++ b/mikeio/dfsu.py
@@ -851,7 +851,9 @@ class _Dfsu(_UnstructuredFile, EquidistantTimeSeries):
             dims = tuple([d for d in dims if d != "element"])
             data_list = [np.squeeze(d) for d in data_list]
 
-        return Dataset(data_list, time, items, geometry=geometry, dims=dims)
+        return Dataset(
+            data_list, time, items, geometry=geometry, dims=dims, validate=False
+        )
 
     def _validate_elements_and_geometry_sel(self, elements, **kwargs):
         used_kwargs = []
diff --git a/mikeio/dfsu_layered.py b/mikeio/dfsu_layered.py
index b72d4eef..8ba10a06 100644
--- a/mikeio/dfsu_layered.py
+++ b/mikeio/dfsu_layered.py
@@ -264,9 +264,12 @@ class DfsuLayered(_Dfsu):
                 geometry=geometry,
                 zn=data_list[0],
                 dims=dims,
+                validate=False,
             )
         else:
-            return Dataset(data_list, time, items, geometry=geometry, dims=dims)
+            return Dataset(
+                data_list, time, items, geometry=geometry, dims=dims, validate=False
+            )
 
     def _parse_geometry_sel(self, area, layers, x, y, z):
         elements = None
diff --git a/mikeio/dfsu_spectral.py b/mikeio/dfsu_spectral.py
index acbce0be..6c04bc36 100644
--- a/mikeio/dfsu_spectral.py
+++ b/mikeio/dfsu_spectral.py
@@ -201,7 +201,9 @@ class DfsuSpectral(_Dfsu):
         dfs.Close()
 
         time = pd.to_datetime(t_seconds, unit="s", origin=self.start_time)
-        return Dataset(data_list, time, items, geometry=geometry, dims=dims)
+        return Dataset(
+            data_list, time, items, geometry=geometry, dims=dims, validate=False
+        )
 
     def _parse_elements_nodes(self, elements, nodes):
         if self._type == DfsuFileType.DfsuSpectral0D:
diff --git a/mikeio/interpolation.py b/mikeio/interpolation.py
index 199b07c5..cd22bafa 100644
--- a/mikeio/interpolation.py
+++ b/mikeio/interpolation.py
@@ -87,7 +87,7 @@ def interp2d(data, elem_ids, weights=None, shape=None):
                 idatitem = idatitem.reshape((nt, *shape))
             interp_data_vars[key] = DataArray(data=idatitem, time=da.time, item=da.item)
 
-        new_ds = Dataset(interp_data_vars)
+        new_ds = Dataset(interp_data_vars, validate=False)
         return new_ds
 
     is_single_item = False


Test patch:
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
index 6cb224fb..2ea96911 100644
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -1407,3 +1407,20 @@ def test_time_selection():
 
     assert dss_t.shape == (24,)
     assert len(dss_tix) == 1
+
+
+def test_create_dataset_with_many_items():
+    n_items = 800
+    nt = 2
+    time = pd.date_range("2000", freq="H", periods=nt)
+
+    das = []
+
+    for i in range(n_items):
+        x = np.random.random(nt)
+        da = mikeio.DataArray(data=x, time=time, item=mikeio.ItemInfo(f"Item {i+1}"))
+        das.append(da)
+
+    ds = mikeio.Dataset(das)
+
+    assert ds.n_items == n_items
diff --git a/tests/test_dfs0.py b/tests/test_dfs0.py
index 573cb32b..6d2b274f 100644
--- a/tests/test_dfs0.py
+++ b/tests/test_dfs0.py
@@ -622,3 +622,10 @@ def test_read_write_single_step_to_dataframe(tmp_path):
     assert df.shape[0] == 1
     assert df.iloc[0, 0] == pytest.approx(1.81134)
     assert np.isnan(df.iloc[0, 3])
+
+
+def test_read_dfs0_with_many_items():
+
+    ds = mikeio.read("tests/testdata/many_items.dfs0")
+
+    assert ds.n_items == 800