Repository: wright-group/WrightTools

Issue:
wt.data.join all nan slices
Using the function wt.data.join creates a data object combining multiple files, but it creates all nan slices at the boundary between the parent objects.


Discussion:

Resolving patch:
diff --git a/WrightTools/data.py b/WrightTools/data.py
--- a/WrightTools/data.py
+++ b/WrightTools/data.py
@@ -267,7 +267,7 @@ def normalize(self, axis=None):
                 axis = tuple((int(i) for i in axis))
             else:  # presumably a simple number
                 axis = int(axis)
-        # subtract off znull
+        # subtract off null
         self.values -= self.null
         self.null = 0.
         # create dummy array
@@ -2161,7 +2161,7 @@ def from_COLORS(
 
     # create data object --------------------------------------------------------------------------
 
-    data = Data(list(scanned), list(channels.values()), list(constant), znull)
+    data = Data(list(scanned), list(channels.values()), list(constant))
 
     if color_steps_as == 'energy':
         try:
@@ -2233,7 +2233,7 @@ def from_JASCO(filepath, name=None, kind='absorbance', verbose=True):
     return data
 
 
-def from_KENT(filepaths, znull=None, name=None, ignore=['wm'], use_norm=False,
+def from_KENT(filepaths, null=None, name=None, ignore=['wm'], use_norm=False,
               delay_tolerance=0.1, frequency_tolerance=0.5, verbose=True):
     """
     filepaths may be string or list
@@ -2331,7 +2331,7 @@ def from_KENT(filepaths, znull=None, name=None, ignore=['wm'], use_norm=False,
             if debug:
                 print(key)
     # create data object --------------------------------------------------------------------------
-    data = Data(list(scanned), list(channels.values()), list(constant), znull)
+    data = Data(list(scanned), list(channels.values()), list(constant))
     for axis in data.axes:
         axis.get_label()
     for axis in data.constants:
@@ -2712,7 +2712,7 @@ def from_Tensor27(filepath, name=None, verbose=True):
     return data
 
 
-def join(datas, method='first', verbose=True):
+def join(datas, method='first', verbose=True, **kwargs):
     """ Join a list of data objects together.
 
     For now datas must have identical dimensionalities (order and identity).
@@ -2723,11 +2723,17 @@ def join(datas, method='first', verbose=True):
         The list of data objects to join together.
     method : {'first', 'sum', 'max', 'min', 'mean'} (optional)
         The method for how overlapping points get treated. Default is first,
-        meaning that the data object that appears first in data will take
+        meaning that the data object that appears first in datas will take
         precedence.
     verbose : bool (optional)
         Toggle talkback. Default is True.
 
+    kwargs
+    ------
+    axis objects
+        The axes of the new data object. If not supplied, the points of the
+        new axis will be guessed from the given datas.
+
     Returns
     -------
     data
@@ -2744,7 +2750,11 @@ def join(datas, method='first', verbose=True):
     axis_units = []
     axis_objects = []
     for data in datas:
-        for axis in data.axes:
+        for i, axis in enumerate(data.axes):
+            if axis.name in kwargs.keys():
+                axis.convert(kwargs[axis.name].units)
+            if axis.points[0] > axis.points[-1]:
+                data.flip(i)
             if axis.name not in axis_names:
                 axis_names.append(axis.name)
                 axis_units.append(axis.units)
@@ -2759,6 +2769,9 @@ def join(datas, method='first', verbose=True):
     # get axis points
     axis_points = []  # list of 1D arrays
     for axis_name in axis_names:
+        if axis_name in kwargs.keys():
+            axis_points.append(kwargs[axis_name].points)
+            continue
         all_points = np.array([])
         step_sizes = []
         for data in datas:
@@ -2767,21 +2780,22 @@ def join(datas, method='first', verbose=True):
                     all_points = np.concatenate([all_points, axis.points])
                     this_axis_min = np.nanmin(axis.points)
                     this_axis_max = np.nanmax(axis.points)
-                    this_axis_number = float(axis.points.size)
+                    this_axis_number = float(axis.points.size)-1
                     step_size = (this_axis_max - this_axis_min) / this_axis_number
                     step_sizes.append(step_size)
         axis_min = np.nanmin(all_points)
         axis_max = np.nanmax(all_points)
         axis_step_size = min(step_sizes)
         axis_n_points = np.ceil((axis_max - axis_min) / axis_step_size)
-        points = np.linspace(axis_min, axis_max, axis_n_points)
+        points = np.linspace(axis_min, axis_max, axis_n_points+1)
         axis_points.append(points)
     # map datas to new points
     for axis_index, axis_name in enumerate(axis_names):
         for data in datas:
             for axis in data.axes:
                 if axis.name == axis_name:
-                    data.map_axis(axis_name, axis_points[axis_index])
+                    if not np.array_equiv(axis.points, axis_points[axis_index]):
+                        data.map_axis(axis_name, axis_points[axis_index])
     # make new channel objects
     channel_objects = []
     n_channels = min([len(d.channels) for d in datas])
@@ -2808,7 +2822,7 @@ def join(datas, method='first', verbose=True):
             print('method', method, 'not recognized in join')
             return
         zis[np.isnan(full).all(axis=0)] = np.nan  # if all datas NaN, zis NaN
-        channel = Channel(zis, 'V', znull=0.,
+        channel = Channel(zis, 'V', null=0.,
                           signed=datas[0].channels[channel_index].signed,
                           name=datas[0].channels[channel_index].name)
         channel_objects.append(channel)
@@ -2827,7 +2841,7 @@ def join(datas, method='first', verbose=True):
             percent_nan = np.around(100. * (np.isnan(channel.values).sum() /
                                             float(channel.values.size)), decimals=2)
             print('    {0} : {1} to {2} ({3}% NaN)'.format(
-                channel.name, channel.zmin, channel.zmax, percent_nan))
+                channel.name, channel.min, channel.max, percent_nan))
     return out
 
 
diff --git a/WrightTools/datasets/__init__.py b/WrightTools/datasets/__init__.py
--- a/WrightTools/datasets/__init__.py
+++ b/WrightTools/datasets/__init__.py
@@ -48,6 +48,9 @@ def from_directory(self, dirname, prefix=''):
 KENT.from_directory(os.path.join(here, 'KENT', 'PbSe 2D delay A'))
 KENT.from_directory(os.path.join(here, 'KENT', 'PbSe 2D delay B'))
 
+PyCMDS = DatasetContainer()
+PyCMDS.from_files('PyCMDS')
+
 
 # --- pretty namespace ----------------------------------------------------------------------------
 


Test patch:
diff --git a/tests/test_import.py b/tests/import.py
similarity index 100%
rename from tests/test_import.py
rename to tests/import.py
diff --git a/tests/join.py b/tests/join.py
new file mode 100644
--- /dev/null
+++ b/tests/join.py
@@ -0,0 +1,13 @@
+import numpy as np
+
+import WrightTools as wt
+from WrightTools import datasets
+
+def test_wm_w2_w1():
+    p = datasets.PyCMDS.wm_w2_w1_000
+    a = wt.data.from_PyCMDS(p)
+    p = datasets.PyCMDS.wm_w2_w1_001
+    b = wt.data.from_PyCMDS(p)
+    joined = wt.data.join([a, b])
+    assert joined.shape == (63, 11, 11)
+    assert not np.isnan(joined.channels[0].values).any()