Repository: microsoft/Qcodes

Issue:
data synchronization crash
When my datasets get bigger (e.g. 400x40 points) somewhere halfway in the measurement, the synchronization becomes very slow. There is no error message, but the synchronization basically stops.
It is not possible to stop the measurement as also 

```
qc.halt_bg()
```

does not get executed. The only way to get back to work is to quit the kernel which is impractical to do.



Discussion:

Resolving patch:
diff --git a/qcodes/data/data_array.py b/qcodes/data/data_array.py
--- a/qcodes/data/data_array.py
+++ b/qcodes/data/data_array.py
@@ -5,7 +5,7 @@
 
 
 class DataArray(DelegateAttributes):
-    '''
+    """
     A container for one parameter in a measurement loop
 
     If this is a measured parameter, This object doesn't contain
@@ -24,7 +24,7 @@ class DataArray(DelegateAttributes):
 
     Once the array is initialized, a DataArray acts a lot like a numpy array,
     because we delegate attributes through to the numpy array
-    '''
+    """
     def __init__(self, parameter=None, name=None, label=None, array_id=None,
                  set_arrays=(), size=None, action_indices=(),
                  preset_data=None):
@@ -67,7 +67,7 @@ def data_set(self, new_data_set):
         self._data_set = new_data_set
 
     def nest(self, size, action_index=None, set_array=None):
-        '''
+        """
         nest this array inside a new outer loop
 
         size: length of the new loop
@@ -76,7 +76,7 @@ def nest(self, size, action_index=None, set_array=None):
             if this DataArray *is* a setpoint array, you should omit both
             action_index and set_array, and it will reference itself as the
             set_array
-        '''
+        """
         if self.ndarray is not None and not self._preset:
             raise RuntimeError('Only preset arrays can be nested after data '
                                'is initialized! {}'.format(self))
@@ -105,11 +105,11 @@ def nest(self, size, action_index=None, set_array=None):
         return self
 
     def init_data(self, data=None):
-        '''
+        """
         create a data array (if one doesn't exist)
         if data is provided, this array is marked as a preset
         meaning it can still be nested around this data.
-        '''
+        """
         if data is not None:
             if not isinstance(data, np.ndarray):
                 if isinstance(data, collections.Iterator):
@@ -142,9 +142,9 @@ def _set_index_bounds(self):
         self._max_indices = [d - 1 for d in self.size]
 
     def clear(self):
-        '''
+        """
         Fill the (already existing) data array with nan
-        '''
+        """
         # only floats can hold nan values. I guess we could
         # also raise an error in this case? But generally float is
         # what people want anyway.
@@ -153,13 +153,13 @@ def clear(self):
         self.ndarray.fill(float('nan'))
 
     def __setitem__(self, loop_indices, value):
-        '''
+        """
         set data values. Follows numpy syntax, allowing indices of lower
         dimensionality than the array, if value makes up the extra dimension(s)
 
         Also updates the record of modifications to the array. If you don't
         want this overhead, you can access self.ndarray directly.
-        '''
+        """
         if isinstance(loop_indices, collections.Iterable):
             min_indices = list(loop_indices)
             max_indices = list(loop_indices)
@@ -186,10 +186,10 @@ def __getitem__(self, loop_indices):
     delegate_attr_objects = ['ndarray']
 
     def __len__(self):
-        '''
+        """
         must be explicitly delegated, because len() will look for this
         attribute to already exist
-        '''
+        """
         return len(self.ndarray)
 
     def _flat_index(self, indices, index_fill):
@@ -204,10 +204,10 @@ def _update_modified_range(self, low, high):
             self.modified_range = (low, high)
 
     def mark_saved(self, last_saved_index):
-        '''
+        """
         after saving data, mark outstanding modifications up to
         last_saved_index as saved
-        '''
+        """
         if self.modified_range:
             if last_saved_index >= self.modified_range[1]:
                 self.modified_range = None
@@ -218,15 +218,47 @@ def mark_saved(self, last_saved_index):
         self.last_saved_index = last_saved_index
 
     def clear_save(self):
-        '''
+        """
         make this array look unsaved, so we can force overwrite
         or rewrite, like if we're moving or copying the DataSet
-        '''
+        """
         if self.last_saved_index is not None:
             self._update_modified_range(0, self.last_saved_index)
 
         self.last_saved_index = None
 
+    def get_synced_index(self):
+        if not hasattr(self, 'synced_index'):
+            self.init_data()
+            self.synced_index = -1
+
+        return self.synced_index
+
+    def get_changes(self, synced_index):
+        latest_index = self.last_saved_index
+        if latest_index is None:
+            latest_index = -1
+        if self.modified_range:
+            latest_index = max(latest_index, self.modified_range[1])
+
+        vals = [
+            self.ndarray[np.unravel_index(i, self.ndarray.shape)]
+            for i in range(synced_index + 1, latest_index + 1)
+        ]
+
+        if vals:
+            return {
+                'start': synced_index + 1,
+                'stop': latest_index,
+                'vals': vals
+            }
+
+    def apply_changes(self, start, stop, vals):
+        for i, val in enumerate(vals):
+            index = np.unravel_index(i + start, self.ndarray.shape)
+            self.ndarray[index] = val
+        self.synced_index = stop
+
     def __repr__(self):
         array_id_or_none = ' {}'.format(self.array_id) if self.array_id else ''
         return '{}[{}]:{}\n{}'.format(self.__class__.__name__,
diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py
--- a/qcodes/data/data_set.py
+++ b/qcodes/data/data_set.py
@@ -19,7 +19,7 @@ class DataMode(Enum):
 
 def new_data(location=None, name=None, overwrite=False, io=None,
              data_manager=None, mode=DataMode.LOCAL, **kwargs):
-    '''
+    """
     Create a new DataSet. Arguments are the same as DataSet constructor, plus:
 
     overwrite: Are we allowed to overwrite an existing location? default False
@@ -35,7 +35,7 @@ def new_data(location=None, name=None, overwrite=False, io=None,
 
     name: an optional string to be passed to location_provider to augment
         the automatic location with something meaningful
-    '''
+    """
     if io is None:
         io = DataSet.default_io
 
@@ -58,7 +58,7 @@ def new_data(location=None, name=None, overwrite=False, io=None,
 
 
 def load_data(location=None, data_manager=None, formatter=None, io=None):
-    '''
+    """
     Load an existing DataSet. Arguments are a subset of the DataSet
     constructor:
 
@@ -76,7 +76,7 @@ def load_data(location=None, data_manager=None, formatter=None, io=None):
 
     formatter: as in DataSet
     io: as in DataSet
-    '''
+    """
     if data_manager is None:
         data_manager = get_data_manager(only_existing=True)
 
@@ -112,7 +112,7 @@ def _get_live_data(data_manager):
 
 
 class TimestampLocation:
-    '''
+    """
     This is the default `DataSet.location_provider`.
     A `location_provider` object should be a callable taking two parameters:
     - an io manager `io` used to search for existing data using
@@ -126,7 +126,7 @@ class TimestampLocation:
     format string, which can include slashes (forward and backward are
     equivalent) to create folder structure.
     Default format string is '%Y-%m-%d/%H-%M-%S'
-    '''
+    """
     def __init__(self, fmt='%Y-%m-%d/%H-%M-%S'):
         self.fmt = fmt
 
@@ -148,7 +148,7 @@ def __call__(self, io, name=None):
 
 
 class DataSet(DelegateAttributes):
-    '''
+    """
     A container for one complete measurement loop
     May contain many individual arrays with potentially different
     sizes and dimensionalities.
@@ -189,7 +189,7 @@ class DataSet(DelegateAttributes):
         if mode=LOCAL, otherwise the DataManager handles this (and generally
         writes more often because it's not tying up the main process to do so).
         use None to disable writing from calls to self.store
-    '''
+    """
 
     # ie data_array.arrays['vsd'] === data_array.vsd
     delegate_attr_dicts = ['arrays']
@@ -262,10 +262,10 @@ def _init_push_to_server(self, data_manager):
         self.data_manager = data_manager
 
     def init_on_server(self):
-        '''
+        """
         Configure this DataSet as the DataServer copy
         Should be run only by the DataServer itself.
-        '''
+        """
         if not self.arrays:
             raise RuntimeError('A server-side DataSet needs DataArrays.')
 
@@ -283,18 +283,18 @@ def _init_live(self, data_manager):
 
     @property
     def is_live_mode(self):
-        '''
+        """
         indicate whether this DataSet thinks it is live in the DataServer
         without actually talking to the DataServer or syncing with it
-        '''
+        """
         return self.mode in SERVER_MODES and self.data_manager and True
 
     @property
     def is_on_server(self):
-        '''
+        """
         Check whether this DataSet is being mirrored in the DataServer
         If it thought it was but isn't, convert it to mode=LOCAL
-        '''
+        """
         if not self.is_live_mode or self.location is False:
             return False
 
@@ -303,12 +303,12 @@ def is_on_server(self):
             return self.location == live_location
 
     def sync(self):
-        '''
+        """
         synchronize this data set with a possibly newer version either
         in storage or on the DataServer, depending on its mode
 
         returns: boolean, is this DataSet live on the server
-        '''
+        """
         # TODO: sync implies bidirectional... and it could be!
         # we should keep track of last sync timestamp and last modification
         # so we can tell whether this one, the other one, or both copies have
@@ -330,14 +330,15 @@ def sync(self):
 
         with self.data_manager.query_lock:
             if self.is_on_server:
-                # TODO: can we reduce the amount of data to send?
-                # seems like in the most general case this would need to
-                # remember each client DataSet on the server, and what has
-                # changed since that particular client last synced
-                # (at least first and last pt)
-                live_data = self.data_manager.ask('get_data').arrays
-                for array_id in self.arrays:
-                    self.arrays[array_id].ndarray = live_data[array_id].ndarray
+                synced_indices = {
+                    array_id: array.get_synced_index()
+                    for array_id, array in self.arrays.items()
+                }
+
+                changes = self.data_manager.ask('get_changes', synced_indices)
+
+                for array_id, array_changes in changes.items():
+                    self.arrays[array_id].apply_changes(**array_changes)
 
                 measuring = self.data_manager.ask('get_measuring')
                 if not measuring:
@@ -354,8 +355,18 @@ def sync(self):
                 self.read()
                 return False
 
+    def get_changes(self, synced_index):
+        changes = {}
+
+        for array_id, synced_index in synced_index.items():
+            array_changes = self.arrays[array_id].get_changes(synced_index)
+            if array_changes:
+                changes[array_id] = array_changes
+
+        return changes
+
     def add_array(self, data_array):
-        '''
+        """
         add one DataArray to this DataSet
 
         note: DO NOT just set data_set.arrays[id] = data_array
@@ -363,7 +374,7 @@ def add_array(self, data_array):
         reference back to this DataSet. It would also allow you to
         load the array in with different id than it holds itself.
 
-        '''
+        """
         # TODO: mask self.arrays so you *can't* set it directly
 
         if data_array.array_id in self.arrays:
@@ -375,10 +386,10 @@ def add_array(self, data_array):
         data_array.data_set = self
 
     def _clean_array_ids(self, arrays):
-        '''
+        """
         replace action_indices tuple with compact string array_ids
         stripping off as much extraneous info as possible
-        '''
+        """
         action_indices = [array.action_indices for array in arrays]
         array_names = set(array.name for array in arrays)
         for name in array_names:
@@ -421,14 +432,14 @@ def _clean_param_ids(self, arrays, name):
             array.array_id = name + ''.join('_' + str(i) for i in ai)
 
     def store(self, loop_indices, ids_values):
-        '''
+        """
         Set some collection of data points
 
         loop_indices: the indices within whatever loops we are inside
         values: a dict of action_index:value or array_id:value
             where value may be an arbitrarily nested list, to record
             many values at once into one array
-        '''
+        """
         if self.mode == DataMode.PUSH_TO_SERVER:
             self.data_manager.write('store_data', loop_indices, ids_values)
         else:
@@ -440,18 +451,18 @@ def store(self, loop_indices, ids_values):
                 self.last_write = time.time()
 
     def read(self):
-        '''
+        """
         Read the whole DataSet from storage, overwriting the local data
-        '''
+        """
         if self.location is False:
             return
         self.formatter.read(self)
 
     def write(self):
-        '''
+        """
         Write the whole (or only changed parts) DataSet to storage,
         overwriting the existing storage if any.
-        '''
+        """
         if self.mode != DataMode.LOCAL:
             raise RuntimeError('This object is connected to a DataServer, '
                                'which handles writing automatically.')
@@ -461,9 +472,9 @@ def write(self):
         self.formatter.write(self)
 
     def finalize(self):
-        '''
+        """
         Mark the DataSet as complete
-        '''
+        """
         if self.mode == DataMode.PUSH_TO_SERVER:
             self.data_manager.ask('end_data')
         elif self.mode == DataMode.LOCAL:
diff --git a/qcodes/data/manager.py b/qcodes/data/manager.py
--- a/qcodes/data/manager.py
+++ b/qcodes/data/manager.py
@@ -6,10 +6,10 @@
 
 
 def get_data_manager(only_existing=False):
-    '''
+    """
     create or retrieve the storage manager
     makes sure we don't accidentally create multiple DataManager processes
-    '''
+    """
     dm = DataManager.default
     if dm and dm._server.is_alive():
         return dm
@@ -19,10 +19,10 @@ def get_data_manager(only_existing=False):
 
 
 class NoData:
-    '''
+    """
     A placeholder object for DataServer to hold
     when there is no loop running.
-    '''
+    """
     location = None
 
     def store(self, *args, **kwargs):
@@ -34,23 +34,23 @@ def write(self, *args, **kwargs):
 
 class DataManager(ServerManager):
     default = None
-    '''
+    """
     creates a separate process (DataServer) that holds running measurement
     and monitor data, and manages writing these to disk or other storage
 
     DataServer communicates with other processes through messages
     Written using multiprocessing Queue's, but should be easily
     extensible to other messaging systems
-    '''
+    """
     def __init__(self):
         type(self).default = self
         super().__init__(name='DataServer', server_class=DataServer)
 
     def restart(self, force=False):
-        '''
+        """
         Restart the DataServer
         Use force=True to abort a running measurement.
-        '''
+        """
         if (not force) and self.ask('get_data', 'location'):
             raise RuntimeError('A measurement is running. Use '
                                'restart(force=True) to override.')
@@ -58,7 +58,7 @@ def restart(self, force=False):
 
 
 class DataServer:
-    '''
+    """
     Running in its own process, receives, holds, and returns current `Loop` and
     monitor data, and writes it to disk (or other storage)
 
@@ -69,7 +69,7 @@ class DataServer:
     they are nearly identical objects, but are configured differently so that
     the loop `DataSet` doesn't hold any data itself, it only passes that data
     on to the `DataServer`
-    '''
+    """
     default_storage_period = 1  # seconds between data storage calls
     queries_per_store = 5
     default_monitor_period = 60  # seconds between monitoring storage calls
@@ -140,17 +140,17 @@ def _post_error(self, e):
     ######################################################################
 
     def handle_halt(self):
-        '''
+        """
         Quit this DataServer
-        '''
+        """
         self._running = False
         self._reply(True)
 
     def handle_new_data(self, data_set):
-        '''
+        """
         Load a new (normally empty) DataSet into the DataServer, and
         prepare it to start receiving and storing data
-        '''
+        """
         if self._measuring:
             raise RuntimeError('Already executing a measurement')
 
@@ -160,29 +160,35 @@ def handle_new_data(self, data_set):
         self._reply(True)
 
     def handle_end_data(self):
-        '''
+        """
         Mark this DataSet as complete and write its final changes to storage
-        '''
+        """
         self._data.write()
         self._measuring = False
         self._reply(True)
 
     def handle_store_data(self, *args):
-        '''
+        """
         Put some data into the DataSet
         This is the only query that does not return a value, so the measurement
         loop does not need to wait for a reply.
-        '''
+        """
         self._data.store(*args)
 
     def handle_get_measuring(self):
-        '''
+        """
         Is a measurement loop presently running?
-        '''
+        """
         self._reply(self._measuring)
 
     def handle_get_data(self, attr=None):
-        '''
+        """
         Return the active DataSet or some attribute of it
-        '''
+        """
         self._reply(getattr(self._data, attr) if attr else self._data)
+
+    def handle_get_changes(self, synced_indices):
+        """
+        Return all new data after the last sync
+        """
+        self._reply(self._data.get_changes(synced_indices))


Test patch: