Repository: wright-group/WrightTools

Issue:
lazy cache for channel max, min
when `max()` is called:
```python
if 'max' not in self.attrs.keys():
    self.attrs['max'] =  max(self.chunkwise(np.nanmax).values())
return self.attrs['max']
```
same for min...

when `__setitem__` is called:
```python
del(self.attrs['max'])
del(self.attrs['min'])
```

only calculated once for each new array

only calculated when needed

relies on #354


Discussion:

Resolving patch:
diff --git a/WrightTools/_dataset.py b/WrightTools/_dataset.py
--- a/WrightTools/_dataset.py
+++ b/WrightTools/_dataset.py
@@ -5,6 +5,10 @@
 
 
 import posixpath
+import collections
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
 
 import h5py
 
@@ -39,6 +43,11 @@ def __repr__(self):
         return '<WrightTools.{0} \'{1}\' at {2}>'.format(self.class_name, self.natural_name,
                                                          self.fullpath)
 
+    def __setitem__(self, index, value):
+        del self.attrs['max']
+        del self.attrs['min']
+        return super().__setitem__(index, value)
+
     @property
     def fullpath(self):
         """Full path: file and internal structure."""
@@ -78,3 +87,68 @@ def units(self, value):
                 self.attrs.pop('units')
         else:
             self.attrs['units'] = value.encode()
+
+    def chunkwise(self, func, *args, **kwargs):
+        """Execute a function for each chunk in the dataset.
+
+        Order of excecution is not guaranteed.
+
+        Parameters
+        ----------
+        func : function
+            Function to execute. First two arguments must be dataset,
+            slices.
+        args (optional)
+            Additional (unchanging) arguments passed to func.
+        kwargs (optional)
+            Additional (unchanging) keyword arguments passed to func.
+
+        Returns
+        -------
+        collections OrderedDict
+            Dictionary of index: function output. Index is to lowest corner
+            of each chunk.
+        """
+        out = collections.OrderedDict()
+        with ThreadPoolExecutor() as executor:
+            # submit
+            for s in self.slices():
+                key = tuple(sss.start for sss in s)
+                out[key] = executor.submit(func, self, s, *args, **kwargs)
+            # collect
+            for k, v in out.items():
+                out[k] = v.result()
+        return out
+
+    def max(self):
+        """Maximum, ignorning nans."""
+        if 'max' not in self.attrs.keys():
+            def f(dataset, s):
+                return np.nanmax(dataset[s])
+            self.attrs['max'] = max(self.chunkwise(f).values())
+        return self.attrs['max']
+
+    def min(self):
+        """Minimum, ignoring nans."""
+        if 'min' not in self.attrs.keys():
+            def f(dataset, s):
+                return np.nanmin(dataset[s])
+            self.attrs['min'] = min(self.chunkwise(f).values())
+        return self.attrs['min']
+
+    def slices(self):
+        """Returns a generator yielding tuple of slice objects.
+
+        Order is not guaranteed.
+        """
+        if self.chunks is None:
+            yield tuple(slice(None, s) for s in self.shape)
+        else:
+            ceilings = tuple(-(-s // c) for s, c in zip(self.shape, self.chunks))
+            for idx in np.ndindex(ceilings):  # could also use itertools.product
+                out = []
+                for i, c, s in zip(idx, self.chunks, self.shape):
+                    start = i * c
+                    stop = min(start + c, s + 1)
+                    out.append(slice(start, stop, 1))
+                yield tuple(out)
diff --git a/WrightTools/data/_channel.py b/WrightTools/data/_channel.py
--- a/WrightTools/data/_channel.py
+++ b/WrightTools/data/_channel.py
@@ -147,14 +147,6 @@ def mag(self):
         """Channel magnitude (maximum deviation from null)."""
         return self.major_extent
 
-    def max(self):
-        """Maximum, ignorning nans."""
-        return np.nanmax(self[:])
-
-    def min(self):
-        """Minimum, ignoring nans."""
-        return np.nanmin(self[:])
-
     def normalize(self, axis=None):
         """Normalize a Channel, set `null` to 0 and the max to 1."""
         # process axis argument
diff --git a/WrightTools/data/_data.py b/WrightTools/data/_data.py
--- a/WrightTools/data/_data.py
+++ b/WrightTools/data/_data.py
@@ -471,7 +471,8 @@ def create_channel(self, name, values=None, units=None, **kwargs):
             #if not shape == self.shape:
             #    raise Exception  # TODO: better exception
         # create dataset
-        dataset_id = self.require_dataset(name=name, data=values, shape=shape, dtype=dtype).id
+        dataset_id = self.require_dataset(name=name, data=values, shape=shape, dtype=dtype,
+                                          chunks=True).id
         channel = Channel(self, dataset_id, units=units, **kwargs)
         # finish
         self.channels.append(channel)


Test patch:
diff --git a/tests/dataset/max.py b/tests/dataset/max.py
new file mode 100644
--- /dev/null
+++ b/tests/dataset/max.py
@@ -0,0 +1,41 @@
+"""Test dataset min method."""
+
+
+# --- import --------------------------------------------------------------------------------------
+
+
+import numpy as np
+
+import WrightTools as wt
+from WrightTools import datasets
+
+
+# --- test ----------------------------------------------------------------------------------------
+
+
+def test_COLORS_v2p2_WL_wigner():
+    p = wt.datasets.COLORS.v2p2_WL_wigner
+    data = wt.data.from_COLORS(p)
+    assert np.isclose(data['wm'].min(), 416.6946)
+    assert np.isclose(data['d1'].min(), -500.12859918527766)
+    assert np.isclose(data.ai0.min(), -1.3343566958049833)
+    data.close()
+
+
+def test_JASCO_PbSe_batch_4_2012_02_21():
+    p = wt.datasets.JASCO.PbSe_batch_4_2012_02_21
+    data = wt.data.from_JASCO(p)
+    assert np.isclose(data.signal.min(), 0.02875)
+    assert np.isclose(data['energy'].min(), 750.0)
+    data.close()
+
+
+def test_PyCMDS_wm_w2_w1_000():
+    p = wt.datasets.PyCMDS.wm_w2_w1_000
+    data = wt.data.from_PyCMDS(p)
+    assert np.isclose(data['wm'].min(), 483.0833)
+    assert np.isclose(data['w2'].min(), 1550.0)
+    assert np.isclose(data['w1'].min(), 6250.0)
+    assert np.isclose(data.signal_diff.min(), 0.000773)
+    assert np.isclose(data.signal_mean.min(), 0.000563)
+    data.close()
diff --git a/tests/dataset/min.py b/tests/dataset/min.py
new file mode 100644
--- /dev/null
+++ b/tests/dataset/min.py
@@ -0,0 +1,41 @@
+"""Test dataset max method."""
+
+
+# --- import --------------------------------------------------------------------------------------
+
+
+import numpy as np
+
+import WrightTools as wt
+from WrightTools import datasets
+
+
+# --- test ----------------------------------------------------------------------------------------
+
+
+def test_COLORS_v2p2_WL_wigner():
+    p = wt.datasets.COLORS.v2p2_WL_wigner
+    data = wt.data.from_COLORS(p)
+    assert np.isclose(data['wm'].max(), 833.3189)
+    assert np.isclose(data['d1'].max(), 499.89967217282987)
+    assert np.isclose(data.ai0.max(), 21.23047333257083)
+    data.close()
+
+
+def test_JASCO_PbSe_batch_4_2012_02_21():
+    p = wt.datasets.JASCO.PbSe_batch_4_2012_02_21
+    data = wt.data.from_JASCO(p)
+    assert np.isclose(data.signal.max(), 0.78552)
+    assert np.isclose(data['energy'].max(), 2000.0)
+    data.close()
+
+
+def test_PyCMDS_wm_w2_w1_000():
+    p = wt.datasets.PyCMDS.wm_w2_w1_000
+    data = wt.data.from_PyCMDS(p)
+    assert np.isclose(data['wm'].max(), 526.3259)
+    assert np.isclose(data['w2'].max(), 1600.0)
+    assert np.isclose(data['w1'].max(), 6451.612903)
+    assert np.isclose(data.signal_diff.max(), 0.264612)
+    assert np.isclose(data.signal_mean.max(), 0.07345)
+    data.close()