Repository: wright-group/WrightTools

Issue:
from_PyCMDS breaks for 1D and TuneTest data. 
Currently working in synced with origin `'3.0.0-development'`. I have encountered two instances when `data.from_PyCMDS` breaks. 

The first instance is when I call `wt.data.from_PyCMDS` on [this](https://drive.google.com/open?id=0BzJTClorMBuwb3dVMnhwZGx6REU) file I get the following error. `TypeError: unhashable type: 'slice'` on [this](https://github.com/wright-group/WrightTools/blob/development/WrightTools/data/_pycmds.py#L146) line of code: ` meshgrid = tuple([axes[0][:]])`.

I haven't understood the core problem. But I will note:
- Given that `meshgrid` is only used inside of the `if any(interpolate_toggles):`  it should probably be put inside of that `if` block and not called before it. 

The second instance is when importing [this](https://drive.google.com/open?id=0BzJTClorMBuwUllvaG5Iek9OX1U) TuneTest file which has array data. I get the following error `AttributeError: 'dict' object has no attribute 'name'` on [this](https://github.com/wright-group/WrightTools/blob/development/WrightTools/data/_pycmds.py#L121) line of code: `centers = np.array(headers[axis.name + ' centers'])`


Discussion:
`axis.name` should probably be `axis.expression`

That was one of the things that was changed with the transition to variables. `name` is an attribute of variables, not axes. However, I believe we want the full expression here. for all PyCMDS generated data files, this should be identical, as there should only be one variable in this axis, I believe.
I agree that meshgrid can be moved, though I'm unclear as to why it failed, and it still may fail for 1-D data where it is desired

Resolving patch:
diff --git a/WrightTools/data/_pycmds.py b/WrightTools/data/_pycmds.py
--- a/WrightTools/data/_pycmds.py
+++ b/WrightTools/data/_pycmds.py
@@ -66,158 +66,67 @@ def from_PyCMDS(filepath, name=None, parent=None, verbose=True):
         data = Data(**kwargs)
     # array
     arr = np.genfromtxt(filepath).T
-    # get axes
+    # get axes and scanned variables
     axes = []
     for name, identity, units in zip(headers['axis names'],
                                      headers['axis identities'],
                                      headers['axis units']):
+        # points and centers
         points = np.array(headers[name + ' points'])
-        # label seed (temporary implementation)
-        try:
-            index = headers['name'].index(name)
-            label = headers['label'][index]
-            label_seed = [label]
-        except ValueError:
-            label_seed = ['']
-        # create axis
-        kwargs = {'identity': identity}
-        if 'D' in identity:
-            kwargs['centers'] = headers[name + ' centers']
-        axis = {'points': points, 'units': units, 'name': name, 'label_seed': label_seed, **kwargs}
+        if name + ' centers' in headers.keys():
+            centers = headers[name + ' centers']
+        else:
+            centers = None
+        # create
+        axis = {'points': points, 'units': units, 'name': name, 'identity': identity,
+                'centers': centers}
         axes.append(axis)
-    # get indicies arrays
-    indicies = arr[:len(axes)].T
-    indicies = indicies.astype('int64')
-    # get interpolation toggles
-    if 'axis interpolate' in headers.keys():
-        interpolate_toggles = headers['axis interpolate']
-    else:
-        # old data files may not have interpolate toggles in headers
-        # assume no interpolation, unless the axis is the array detector
-        interpolate_toggles = [True if name == 'wa' else False for name in headers['axis names']]
-    # get assorted remaining things
     shape = tuple([a['points'].size for a in axes])
-    tols = [wt_kit.closest_pair(a['points'], give='distance') / 2. for a in axes]
-    # prepare points for interpolation
-    points_dict = collections.OrderedDict()
-    for i, axis in enumerate(axes):
-        # TODO: math and proper full recognition...
-        axis_col_name = [name for name in headers['name'][::-1] if name in axis['identity']][0]
-        axis_index = headers['name'].index(axis_col_name)
-        # shape array acording to recorded coordinates
-        points = np.full(shape, np.nan)
-        for j, idx in enumerate(indicies):
-            lis = arr[axis_index]
-            points[tuple(idx)] = lis[j]
-        # convert array
-        points = wt_units.converter(points, headers['units'][axis_index], axis['units'])
-        # take case of scan about center
-        if axis['identity'][0] == 'D':
-            # transpose so this axis is first
-            transpose_order = list(range(len(axes)))
-            transpose_order.insert(0, transpose_order.pop(i))
-            points = points.transpose(transpose_order)
-            # subtract out centers
-            centers = np.array(headers[axis.name + ' centers'])
-            points -= centers
-            # transpose out
-            transpose_order = list(range(len(axes)))
-            transpose_order.insert(i, transpose_order.pop(0))
-            points = points.transpose(transpose_order)
-        points = points.flatten()
-        points_dict[axis['name']] = points
-        # check, coerce non-interpolated axes
-        if not interpolate_toggles[i]:
-            for j, idx in enumerate(indicies):
-                actual = points[j]
-                expected = axis['points'][idx[i]]
-                if abs(actual - expected) > tols[i]:
-                    warnings.warn('at least one point exceded tolerance ' +
-                                  'in axis {}'.format(axis.name))
-                points[j] = expected
-    all_points = tuple(points_dict.values())
-    # prepare values for interpolation
-    values_dict = collections.OrderedDict()
-    for i, kind in enumerate(headers['kind']):
-        if kind == 'channel':
-            values_dict[headers['name'][i]] = arr[i]
-    # create grid to interpolate onto
-    if len(axes) == 1:
-        meshgrid = tuple([axes[0][:]])
-    else:
-        meshgrid = tuple(np.meshgrid(*[a['points'] for a in axes], indexing='ij'))
-    if any(interpolate_toggles):
-        # create channels through linear interpolation
-        channels = []
-        for i in range(len(arr)):
-            if headers['kind'][i] == 'channel':
-                # unpack
-                units = headers['units'][i]
-                signed = headers['channel signed'][len(channels)]
-                name = headers['name'][i]
-                label = headers['label'][i]
-                # interpolate
-                values = values_dict[name]
-                zi = griddata(all_points, values, meshgrid, rescale=True,
-                              method='linear', fill_value=np.nan)
-                # assemble
-                channel = {'values': zi, 'units': units, 'signed': signed, 'name': name,
-                           'label': label}
-                channels.append(channel)
-    else:
-        # if none of the axes are interpolated onto,
-        # simply fill zis based on recorded axis index
-        num_channels = headers['kind'].count('channel')
-        channel_indicies = [i for i, kind in enumerate(headers['kind']) if kind == 'channel']
-        zis = [np.full(shape, np.nan) for _ in range(num_channels)]
-        # iterate through each row of the array, filling zis
-        for i in range(len(arr[0])):
-            idx = tuple(indicies[i])  # yes, this MUST be a tuple >:(
-            for zi_index, arr_index in enumerate(channel_indicies):
-                zis[zi_index][idx] = arr[arr_index, i]
-        # assemble channels
-        channels = []
-        for zi_index, arr_index in zip(range(len(zis)), channel_indicies):
-            zi = zis[zi_index]
-            units = headers['units'][arr_index]
-            signed = headers['channel signed'][zi_index]
-            name = headers['name'][arr_index]
-            label = headers['label'][arr_index]
-            channel = {'values': zi, 'units': units, 'signed': signed, 'name': name,
-                       'label': label}
-            channels.append(channel)
-    # add labtime variable
-    index = headers['name'].index('time')
-    values = np.reshape(arr[index], shape)
-    data.create_variable('labtime', values=values)
-    # add all other variables
+    # get assorted remaining things
+    # variables and channels
     for index, kind, name in zip(range(len(arr)), headers['kind'], headers['name']):
+        if name == 'time':
+            values = np.reshape(arr[index], shape)
+            data.create_variable(name='labtime', values=values)
         if kind == 'hardware':
+            # sadly, recorded tolerances are not reliable
+            # so a bit of hard-coded hacking is needed
+            # if this ends up being too fragile, we might have to use the points arrays
+            # ---Blaise 2018-01-09
             values = np.reshape(arr[index], shape)
-            for dim in range(values.ndim):
-                collapse = True
-                s = list(values.shape)
-                s.pop(dim)
-                for idx in np.ndindex(tuple(s)):
-                    idx = list(idx)
-                    idx.insert(dim, slice(None))
-                    test = values[tuple(idx)]
-                    if not (test == test[0]).all():
-                        collapse = False
+            if 'w' in name and name.startswith(tuple(data.variable_names)):
+                inherited_shape = data[name.split('_')[0]].shape
+                for i, s in enumerate(inherited_shape):
+                    if s == 1:
+                        values = np.mean(values, axis=i)
+                        values = np.expand_dims(values, i)
+            else:
+                tolerance = headers['tolerance'][index]
+                for i in range(len(shape)):
+                    if tolerance is None:
                         break
-                if collapse:
-                    idx = [slice(None)] * values.ndim
-                    idx[dim] = 0
-                    values = values[tuple(idx)]
-                    values = np.expand_dims(values, dim)
+                    if 'd' in name:
+                        tolerance = 3.
+                    if 'zero' in name:
+                        tolerance = 1e-10
+                    mean = np.mean(values, axis=i)
+                    mean = np.expand_dims(mean, i)
+                    if np.allclose(mean, values, atol=tolerance):
+                        values = mean
             units = headers['units'][index]
             label = headers['label'][index]
             data.create_variable(name, values=values, units=units, label=label)
-    # add channels
-    for channel in channels:
-        data.create_channel(**channel)
-    # add axes
-    data.transform([a['identity'] for a in axes])
+        if kind == 'channel':
+            values = np.reshape(arr[index], shape)
+            data.create_channel(name=name, values=values, shape=values.shape)
+    # axes
+    for a in axes:
+        expression = a['identity']
+        if expression.startswith('D'):
+            expression = expression[1:]
+        expression.replace('=D', '=')
+        a['expression'] = expression
+    data.transform([a['expression'] for a in axes])
     # return
     if verbose:
         print('data created at {0}'.format(data.fullpath))
diff --git a/WrightTools/datasets/__init__.py b/WrightTools/datasets/__init__.py
--- a/WrightTools/datasets/__init__.py
+++ b/WrightTools/datasets/__init__.py
@@ -88,4 +88,4 @@ def _from_directory(self, dirname, prefix=''):
 # --- pretty namespace ----------------------------------------------------------------------------
 
 
-__all__ = ['COLORS', 'JASCO', 'KENT', 'BrunoldrRaman', 'Cary50', 'Tensor27']
+__all__ = ['COLORS', 'JASCO', 'KENT', 'PyCMDS', 'BrunoldrRaman', 'Cary50', 'Tensor27']


Test patch:
diff --git a/examples/tune_test.py b/examples/tune_test.py
new file mode 100644
--- /dev/null
+++ b/examples/tune_test.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""
+Tune test
+=========
+
+An example of transform on a tune test.
+"""
+
+import matplotlib.pyplot as plt
+
+import WrightTools as wt
+from WrightTools import datasets
+
+p = datasets.PyCMDS.w1_wa_000
+data = wt.data.from_PyCMDS(p)
+
+fig, gs = wt.artists.create_figure(width='double', cols=[1, 1, 'cbar'])
+
+# as taken
+ax = plt.subplot(gs[0, 0])
+ax.pcolor(data.w1__e__wm.full, data.wa.full, data.array_signal)
+wt.artists.set_ax_labels(xlabel=data.w1__e__wm.label, ylabel=data.wa.label)
+ax.set_title('as taken', fontsize=20)
+
+# transformed
+ax = plt.subplot(gs[0, 1])
+data.transform(['w1', 'wa-w1'])
+ax.pcolor(data.w1.full, data.wa__m__w1.full, data.array_signal)
+wt.artists.set_ax_labels(xlabel=data.w1.label, ylabel=data.wa__m__w1.label)
+ax.set_title('transformed', fontsize=20)
+
+# colorbar
+cax = plt.subplot(gs[0, -1])
+wt.artists.plot_colorbar(cax, label='amplitude')
diff --git a/tests/data/from_PyCMDS.py b/tests/data/from_PyCMDS.py
--- a/tests/data/from_PyCMDS.py
+++ b/tests/data/from_PyCMDS.py
@@ -11,11 +11,27 @@
 # --- test ----------------------------------------------------------------------------------------
 
 
+def test_w1_000():
+    p = datasets.PyCMDS.w1_000
+    data = wt.data.from_PyCMDS(p)
+    assert data.shape == (51,)
+    assert data.axis_expressions == ['w1']
+    data.close()
+
+
+def test_w1_wa_000():
+    p = datasets.PyCMDS.w1_wa_000
+    data = wt.data.from_PyCMDS(p)
+    assert data.shape == (25, 256)
+    assert data.axis_expressions == ['w1=wm', 'wa']
+    data.close()
+
+
 def test_w2_w1_000():
     p = datasets.PyCMDS.w2_w1_000
     data = wt.data.from_PyCMDS(p)
     assert data.shape == (81, 81)
-    assert data.axis_names == ['w2', 'w1']
+    assert data.axis_expressions == ['w2', 'w1']
     data.close()
 
 
@@ -23,7 +39,7 @@ def test_wm_w2_w1_000():
     p = datasets.PyCMDS.wm_w2_w1_000
     data = wt.data.from_PyCMDS(p)
     assert data.shape == (35, 11, 11)
-    assert data.axis_names == ['wm', 'w2', 'w1']
+    assert data.axis_expressions == ['wm', 'w2', 'w1']
     data.close()
 
 
@@ -31,5 +47,16 @@ def test_wm_w2_w1_001():
     p = datasets.PyCMDS.wm_w2_w1_001
     data = wt.data.from_PyCMDS(p)
     assert data.shape == (29, 11, 11)
-    assert data.axis_names == ['wm', 'w2', 'w1']
+    assert data.axis_expressions == ['wm', 'w2', 'w1']
     data.close()
+
+
+# --- run -----------------------------------------------------------------------------------------
+
+
+if __name__ == '__main__':
+    test_w1_000()
+    test_w1_wa_000()
+    test_w2_w1_000()
+    test_wm_w2_w1_000()
+    test_wm_w2_w1_001()