Repository: microsoft/Qcodes

Issue:
hdf5format breaks when reading incomplete datasets
### Steps to reproduce
```
data = qcodes.DataSet(location = r'D:\measurements\test', formatter=qcodes.data.hdf5_format.HDF5Format())
myArray = qcodes.DataArray(array_id = 'test_array_set', shape = (10,))
myArray.init_data()
myArray.ndarray[0] = 1
data.add_array(myArray)
data.write()
data.read()
```

### Expected behaviour
data should be read from disk

### Actual behaviour
reading breaks with
``` ValueError: cannot reshape array of size 1 into shape (10,)```

@peendebak 


Discussion:

Resolving patch:
diff --git a/qcodes/data/hdf5_format.py b/qcodes/data/hdf5_format.py
--- a/qcodes/data/hdf5_format.py
+++ b/qcodes/data/hdf5_format.py
@@ -14,7 +14,7 @@ class HDF5Format(Formatter):
     HDF5 formatter for saving qcodes datasets.
 
     Capable of storing (write) and recovering (read) qcodes datasets.
-           
+
     """
 
     _format_tag = 'hdf5'
@@ -22,7 +22,7 @@ class HDF5Format(Formatter):
     def close_file(self, data_set):
         """
         Closes the hdf5 file open in the dataset.
-        
+
         Args:
             data_set (DataSet): DataSet object
         """
@@ -99,6 +99,9 @@ def read(self, data_set, location=None):
             #     set_arrays = ()
             vals = dat_arr.value[:, 0]
             if 'shape' in dat_arr.attrs.keys():
+                # extend with NaN if needed
+                esize = np.prod(dat_arr.attrs['shape'])
+                vals = np.append(vals, [np.nan] * (esize - vals.size))
                 vals = vals.reshape(dat_arr.attrs['shape'])
             if array_id not in data_set.arrays.keys():  # create new array
                 d_array = DataArray(
@@ -207,7 +210,12 @@ def write(self, data_set, io_manager=None, location=None,
             datasetshape = dset.shape
             old_dlen = datasetshape[0]
             x = data_set.arrays[array_id]
-            new_dlen = len(x[~np.isnan(x)])
+            try:
+                # get latest NaN element
+                new_dlen = (~np.isnan(x)).flatten().nonzero()[0][-1] + 1
+            except IndexError:
+                new_dlen = old_dlen
+
             new_datasetshape = (new_dlen,
                                 datasetshape[1])
             dset.resize(new_datasetshape)


Test patch:
diff --git a/qcodes/tests/test_hdf5formatter.py b/qcodes/tests/test_hdf5formatter.py
--- a/qcodes/tests/test_hdf5formatter.py
+++ b/qcodes/tests/test_hdf5formatter.py
@@ -4,6 +4,7 @@
 import h5py
 from shutil import copy
 
+import qcodes.data
 from qcodes.station import Station
 from qcodes.loops import Loop
 from qcodes.data.location import FormatLocation
@@ -149,6 +150,15 @@ def test_loop_writing(self):
         self.formatter.close_file(data1)
         self.formatter.close_file(data2)
 
+    def test_partial_dataset(self):
+        data = qcodes.data.data_set.new_data(formatter=self.formatter)
+        data_array = qcodes.data.data_array.DataArray(array_id = 'test_partial_dataset', shape = (10,))
+        data_array.init_data()
+        data_array.ndarray[0] = 1
+        data.add_array(data_array)
+        data.write()
+        data.read()
+
     def test_loop_writing_2D(self):
         # pass
         station = Station()