Repository: microsoft/Qcodes

Issue:
Tests for dataset context manager subscriptions are flaky
@QCoDeS/core know what I'm talking about. It's annoying as hell and might point to some scary underlying problem. We should really try to figure out what is going on.


Discussion:
http://hypothesis.readthedocs.io/en/latest/reproducing.html
As stated in PR 1131, we "fixed" the test_subscriptions test by calling flush_to_database explicitly instead of waiting a period of time before calling "add_result". Even if this fixes the "test_subscriptions" test, there is still some underlying problem because of which data is not flushed to the database at expected times. This still needs to be investigated. 
In this travis run https://travis-ci.org/QCoDeS/Qcodes/builds/390728436?utm_source=github_status&utm_medium=notification `test_subscriptions_getting_all_points` failed again even with this fix
And here `test_subscriptions` failed: https://travis-ci.org/QCoDeS/Qcodes/builds/391788144?utm_source=github_status&utm_medium=notification

-Darn!
Thank you for collecting the failed runs. I will look into that soon...
as @jenshnielsen suggested, perhaps, the problem is in the resolution of `monotonic()` calls. It might help to substitute them with [performance counter](https://docs.python.org/3/library/time.html#time.perf_counter).
@astafan8 I think it's worth trying. Do you want to give it a go?
sure, i'm on it :) I am still the one whom this issue is assigned.

Resolving patch:
diff --git a/qcodes/dataset/data_set.py b/qcodes/dataset/data_set.py
--- a/qcodes/dataset/data_set.py
+++ b/qcodes/dataset/data_set.py
@@ -60,66 +60,79 @@ class CompletedError(RuntimeError):
     pass
 
 
-class Subscriber(Thread):
+class _Subscriber(Thread):
     """
     Class to add a subscriber to a DataSet. The subscriber gets called
     every time an insert is made to the results_table.
 
-    The Subscriber is not meant to be instantiated directly, but rather used
+    The _Subscriber is not meant to be instantiated directly, but rather used
     via the 'subscribe' method of the DataSet.
 
-    NOTE: A subscriber should be added *after* all parameters have added.
+    NOTE: A subscriber should be added *after* all parameters have been added.
+
+    NOTE: Special care shall be taken when using the *state* object: it is the user's
+    responsibility to operate with it in a thread-safe way.
     """
-    def __init__(self, dataSet, sub_id: str,
+    def __init__(self,
+                 dataSet,
+                 id_: str,
                  callback: Callable[..., None],
-                 state: Optional[Any] = None, min_wait: int = 100,
-                 min_count: int = 1,
-                 callback_kwargs: Optional[Dict[str, Any]]=None) -> None:
-        self.sub_id = sub_id
-        # whether or not this is actually thread safe I am not sure :P
+                 state: Optional[Any] = None,
+                 loop_sleep_time: int = 0,  # in milliseconds
+                 min_queue_length: int = 1,
+                 callback_kwargs: Optional[Dict[str, Any]]=None
+                 ) -> None:
+        super().__init__()
+
+        self._id = id_
+
         self.dataSet = dataSet
         self.table_name = dataSet.table_name
-        conn = dataSet.conn
-        self.log = logging.getLogger(f"Subscriber {self.sub_id}")
+        self._data_set_len = len(dataSet)
 
         self.state = state
-        self.min_wait = min_wait
-        self.min_count = min_count
-        self._send_queue: int = 0
+
+        self.data_queue: Queue = Queue()
+        self._queue_length: int = 0
+        self._stop_signal: bool = False
+        self._loop_sleep_time = loop_sleep_time / 1000  # convert milliseconds to seconds
+        self.min_queue_length = min_queue_length
+
         if callback_kwargs is None:
             self.callback = callback
         else:
             self.callback = functools.partial(callback, **callback_kwargs)
-        self._stop_signal: bool = False
 
-        parameters = dataSet.get_parameters()
-        param_sql = ",".join([f"NEW.{p.name}" for p in parameters])
-        self.callbackid = f"callback{self.sub_id}"
+        self.callback_id = f"callback{self._id}"
 
-        conn.create_function(self.callbackid, -1, self.cache)
-        sql = f"""
-        CREATE TRIGGER sub{self.sub_id}
+        conn = dataSet.conn
+
+        conn.create_function(self.callback_id, -1, self._cache_data_to_queue)
+
+        parameters = dataSet.get_parameters()
+        sql_param_list = ",".join([f"NEW.{p.name}" for p in parameters])
+        sql_create_trigger_for_callback = f"""
+        CREATE TRIGGER sub{self._id}
             AFTER INSERT ON '{self.table_name}'
         BEGIN
-            SELECT {self.callbackid}({param_sql});
+            SELECT {self.callback_id}({sql_param_list});
         END;"""
-        atomic_transaction(conn, sql)
-        self.data: Queue = Queue()
-        self._data_set_len = len(dataSet)
-        super().__init__()
+        atomic_transaction(conn, sql_create_trigger_for_callback)
+
+        self.log = logging.getLogger(f"_Subscriber {self._id}")
 
-    def cache(self, *args) -> None:
-        self.log.debug(f"Args:{args} put into queue for {self.callbackid}")
-        self.data.put(args)
+    def _cache_data_to_queue(self, *args) -> None:
+        self.log.debug(f"Args:{args} put into queue for {self.callback_id}")
+        self.data_queue.put(args)
         self._data_set_len += 1
-        self._send_queue += 1
+        self._queue_length += 1
 
     def run(self) -> None:
         self.log.debug("Starting subscriber")
         self._loop()
 
     @staticmethod
-    def _exhaust_queue(queue) -> List:
+    def _exhaust_queue(queue: Queue) -> List:
         result_list = []
         while True:
             try:
@@ -128,40 +141,38 @@ def _exhaust_queue(queue) -> List:
                 break
         return result_list
 
-    def _send(self) -> List:
-        result_list = self._exhaust_queue(self.data)
+    def _call_callback_on_queue_data(self) -> None:
+        result_list = self._exhaust_queue(self.data_queue)
         self.callback(result_list, self._data_set_len, self.state)
         self.log.debug(f"{self.callback} called with "
                        f"result_list: {result_list}.")
-        # TODO (WilliamHPNielsen): why does this method return smth?
-        return result_list
 
     def _loop(self) -> None:
         while True:
             if self._stop_signal:
                 self._clean_up()
                 break
-            if self._send_queue >= self.min_count:
-                self._send()
-                self._send_queue = 0
 
-            # if nothing happens we let the word go foward
-            time.sleep(self.min_wait / 1000)
+            if self._queue_length >= self.min_queue_length:
+                self._call_callback_on_queue_data()
+                self._queue_length = 0
+
+            time.sleep(self._loop_sleep_time)
+
             if self.dataSet.completed:
-                self._send()
+                self._call_callback_on_queue_data()
                 break
 
     def done_callback(self) -> None:
         self.log.debug("Done callback")
-        self._send()
+        self._call_callback_on_queue_data()
 
-    def schedule_stop(self):
+    def schedule_stop(self) -> None:
         if not self._stop_signal:
             self.log.debug("Scheduling stop")
             self._stop_signal = True
 
     def _clean_up(self) -> None:
-        # TODO: just a temp implemation (remove?)
         self.log.debug("Stopped subscriber")
 
 
@@ -188,7 +199,7 @@ def __init__(self, path_to_db: str, run_id: Optional[int]=None,
 
         self.run_id = run_id
         self._debug = False
-        self.subscribers: Dict[str, Subscriber] = {}
+        self.subscribers: Dict[str, _Subscriber] = {}
         if run_id:
             self._completed = completed(self.conn, self.run_id)
 
@@ -564,17 +575,19 @@ def get_setpoints(self, param_name: str) -> List[List[Any]]:
         return setpoints
 
     # NEED to pass Any for some reason
-    def subscribe(self, callback: Callable[[Any, int, Optional[Any]], None],
-                  min_wait: int = 0, min_count: int = 1,
+    def subscribe(self,
+                  callback: Callable[[Any, int, Optional[Any]], None],
+                  min_wait: int = 0,
+                  min_count: int = 1,
                   state: Optional[Any] = None,
-                  callback_kwargs: Optional[Dict[str, Any]] = None,
-                  subscriber_class=Subscriber) -> str:
-        sub_id = uuid.uuid4().hex
-        sub = Subscriber(self, sub_id, callback, state, min_wait, min_count,
-                         callback_kwargs)
-        self.subscribers[sub_id] = sub
-        sub.start()
-        return sub.sub_id
+                  callback_kwargs: Optional[Dict[str, Any]] = None
+                  ) -> str:
+        subscriber_id = uuid.uuid4().hex
+        subscriber = _Subscriber(self, subscriber_id, callback, state, min_wait, min_count,
+                                 callback_kwargs)
+        self.subscribers[subscriber_id] = subscriber
+        subscriber.start()
+        return subscriber_id
 
     def unsubscribe(self, uuid: str) -> None:
         """
@@ -588,7 +601,7 @@ def unsubscribe(self, uuid: str) -> None:
             del self.subscribers[uuid]
 
     def _remove_trigger(self, name):
-        transaction(self.conn, f"DROP TRIGGER IF EXISTS name;")
+        transaction(self.conn, f"DROP TRIGGER IF EXISTS {name};")
 
     def unsubscribe_all(self):
         """
diff --git a/qcodes/dataset/measurements.py b/qcodes/dataset/measurements.py
--- a/qcodes/dataset/measurements.py
+++ b/qcodes/dataset/measurements.py
@@ -292,12 +292,12 @@ def __exit__(self, exception_type, exception_value, traceback) -> None:
         for func, args in self.exitactions:
             func(*args)
 
-        self.ds.unsubscribe_all()
-
         # and finally mark the dataset as closed, thus
         # finishing the measurement
         self.ds.mark_complete()
 
+        self.ds.unsubscribe_all()
+
 
 class Measurement:
     """


Test patch:
diff --git a/qcodes/tests/common.py b/qcodes/tests/common.py
--- a/qcodes/tests/common.py
+++ b/qcodes/tests/common.py
@@ -1,3 +1,8 @@
+from typing import Callable, Type
+from functools import wraps
+from time import sleep
+
+
 def strip_qc(d, keys=('instrument', '__class__')):
     # depending on how you run the tests, __module__ can either
     # have qcodes on the front or not. Just strip it off.
@@ -5,3 +10,60 @@ def strip_qc(d, keys=('instrument', '__class__')):
         if key in d:
             d[key] = d[key].replace('qcodes.tests.', 'tests.')
     return d
+
+
+def retry_until_does_not_throw(
+        exception_class_to_expect: Type[Exception]=AssertionError,
+        tries: int=5,
+        delay: float=0.1
+) -> Callable:
+    """
+    Call the decorated function given number of times with given delay between
+    the calls until it does not throw an exception of a given class.
+
+    If the function throws an exception of a different class, it gets propagated
+    outside (i.e. the function is not called anymore).
+
+    Usage:
+        >>  x = False  # let's assume that another thread has access to "x",
+                       # and it is going to change "x" to "True" very soon
+        >>  @retry_until_does_not_throw() ...
+            def assert_x_is_true(): ...
+                assert x, "x is still False..." ...
+        >>  assert_x_is_true()  # depending on the settings of
+                                # "retry_until_does_not_throw", it will keep
+                                # calling the function (with breaks in between)
+                                # until either it does not throw or
+                                # the number of tries is exceeded.
+
+    Args:
+        exception_class_to_expect
+            Only in case of this exception the function will be called again
+        tries
+            Number of times to retry calling the function before giving up
+        delay
+            Delay between retries of the function call, in seconds
+
+    Returns:
+        A callable that runs the decorated function until it does not throw
+        a given exception
+    """
+    def retry_until_passes_decorator(func: Callable):
+
+        @wraps(func)
+        def func_retry(*args, **kwargs):
+            tries_left = tries - 1
+            while tries_left > 0:
+                try:
+                    return func(*args, **kwargs)
+                except exception_class_to_expect:
+                    tries_left -= 1
+                    sleep(delay)
+            # the very last attempt to call the function is outside
+            # the "try-except" clause, so that the exception can propagate
+            # up the call stack
+            return func(*args, **kwargs)
+
+        return func_retry
+
+    return retry_until_passes_decorator
diff --git a/qcodes/tests/dataset/test_measurement_context_manager.py b/qcodes/tests/dataset/test_measurement_context_manager.py
--- a/qcodes/tests/dataset/test_measurement_context_manager.py
+++ b/qcodes/tests/dataset/test_measurement_context_manager.py
@@ -8,12 +8,14 @@
 import hypothesis.strategies as hst
 import numpy as np
 
+from qcodes.tests.common import retry_until_does_not_throw
+
 import qcodes as qc
 from qcodes.dataset.measurements import Measurement
 from qcodes.dataset.experiment_container import new_experiment
 from qcodes.tests.instrument_mocks import DummyInstrument, DummyChannelInstrument
 from qcodes.dataset.param_spec import ParamSpec
-from qcodes.dataset.sqlite_base import connect, init_db
+from qcodes.dataset.sqlite_base import atomic_transaction
 from qcodes.instrument.parameter import ArrayParameter
 from qcodes.dataset.legacy_import import import_dat_file
 from qcodes.dataset.data_set import load_by_id
@@ -347,71 +349,151 @@ def action(lst, word):
 
 def test_subscriptions(experiment, DAC, DMM):
     """
-    Test that subscribers are called at the moment that data is flushed to database
-
-    Note that for the purpose of this test, flush_data_to_database method is called explicitly instead of waiting for
-    the data to be flushed automatically after the write_period passes after a add_result call.
+    Test that subscribers are called at the moment the data is flushed to database
 
-    Args:
-        experiment (qcodes.dataset.experiment_container.Experiment) : qcodes experiment object
-        DAC (qcodes.instrument.base.Instrument) : dummy instrument object
-        DMM (qcodes.instrument.base.Instrument) : another dummy instrument object
+    Note that for the purpose of this test, flush_data_to_database method is
+    called explicitly instead of waiting for the data to be flushed
+    automatically after the write_period passes after a add_result call.
     """
 
-    def subscriber1(results, length, state):
+    def collect_all_results(results, length, state):
         """
-        A dict of all results
+        Updates the *state* to contain all the *results* acquired
+        during the experiment run
         """
+        # Due to the fact that by default subscribers only hold 1 data value
+        # in their internal queue, this assignment should work (i.e. not
+        # overwrite values in the "state" object) assuming that at the start
+        # of the experiment both the dataset and the *state* objects have
+        # the same length.
         state[length] = results
 
-    def subscriber2(results, length, state):
+    def collect_values_larger_than_7(results, length, state):
         """
-        A list of all parameter values larger than 7
+        Appends to the *state* only the values from *results*
+        that are larger than 7
         """
-        for res in results:
-            state += [pres for pres in res if pres > 7]
+        for result_tuple in results:
+            state += [value for value in result_tuple if value > 7]
 
     meas = Measurement(exp=experiment)
     meas.register_parameter(DAC.ch1)
     meas.register_parameter(DMM.v1, setpoints=(DAC.ch1,))
 
-    res_dict = {}
-    lt7s = []
+    # key is the number of the result tuple,
+    # value is the result tuple itself
+    all_results_dict = {}
+    values_larger_than_7 = []
 
-    meas.add_subscriber(subscriber1, state=res_dict)
+    meas.add_subscriber(collect_all_results, state=all_results_dict)
     assert len(meas.subscribers) == 1
-    meas.add_subscriber(subscriber2, state=lt7s)
+    meas.add_subscriber(collect_values_larger_than_7, state=values_larger_than_7)
     assert len(meas.subscribers) == 2
 
     meas.write_period = 0.2
 
-    expected_list = []
-
     with meas.run() as datasaver:
 
+        # Assert that the measurement, runner, and datasaver
+        # have added subscribers to the dataset
         assert len(datasaver._dataset.subscribers) == 2
-        assert res_dict == {}
-        assert lt7s == []
 
-        as_and_bs = list(zip(range(5), range(3, 8)))
+        assert all_results_dict == {}
+        assert values_larger_than_7 == []
+
+        dac_vals_and_dmm_vals = list(zip(range(5), range(3, 8)))
+        values_larger_than_7__expected = []
 
         for num in range(5):
+            (dac_val, dmm_val) = dac_vals_and_dmm_vals[num]
+            values_larger_than_7__expected += \
+                [val for val in (dac_val, dmm_val) if val > 7]
 
-            (a, b) = as_and_bs[num]
-            expected_list += [c for c in (a, b) if c > 7]
+            datasaver.add_result((DAC.ch1, dac_val), (DMM.v1, dmm_val))
 
-            datasaver.add_result((DAC.ch1, a), (DMM.v1, b))
+            # Ensure that data is flushed to the database despite the write
+            # period, so that the database triggers are executed, which in turn
+            # add data to the queues within the subscribers
             datasaver.flush_data_to_database()
 
-            assert lt7s == expected_list
-            assert list(res_dict.keys()) == [n for n in range(1, num+2)]
-
+            # In order to make this test deterministic, we need to ensure that
+            # just enough time has passed between the moment the data is flushed
+            # to database and the "state" object (that is passed to subscriber
+            # constructor) has been updated by the corresponding subscriber's
+            # callback function. At the moment, there is no robust way to ensure
+            # this. The reason is that the subscribers have internal queue which
+            # is populated via a trigger call from the SQL database, hence from
+            # this "main" thread it is difficult to say whether the queue is
+            # empty because the subscriber callbacks have already been executed
+            # or because the triggers of the SQL database has not been executed
+            # yet.
+            #
+            # In order to overcome this problem, a special decorator is used
+            # to wrap the assertions. This is going to ensure that some time
+            # is given to the Subscriber threads to finish exhausting the queue.
+            @retry_until_does_not_throw(
+                exception_class_to_expect=AssertionError, delay=0.5, tries=10)
+            def assert_states_updated_from_callbacks():
+                assert values_larger_than_7 == values_larger_than_7__expected
+                assert list(all_results_dict.keys()) == \
+                    [result_index for result_index in range(1, num + 1 + 1)]
+            assert_states_updated_from_callbacks()
+
+    # Ensure that after exiting the "run()" context,
+    # all subscribers get unsubscribed from the dataset
     assert len(datasaver._dataset.subscribers) == 0
 
+    # Ensure that the triggers for each subscriber
+    # have been removed from the database
+    get_triggers_sql = "SELECT * FROM sqlite_master WHERE TYPE = 'trigger';"
+    triggers = atomic_transaction(
+        datasaver._dataset.conn, get_triggers_sql).fetchall()
+    assert len(triggers) == 0
+
+
+def test_subscribers_called_at_exiting_context_if_queue_is_not_empty(experiment, DAC):
+    """
+    Upon quitting the "run()" context, verify that in case the queue is
+    not empty, the subscriber's callback is still called on that data.
+    This situation is created by setting the minimum length of the queue
+    to a number that is larger than the number of value written to the dataset.
+    """
+    def collect_x_vals(results, length, state):
+        """
+        Collects first elements of results tuples in *state*
+        """
+        index_of_x = 0
+        state += [res[index_of_x] for res in results]
+
+    meas = Measurement(exp=experiment)
+    meas.register_parameter(DAC.ch1)
+
+    collected_x_vals = []
+
+    meas.add_subscriber(collect_x_vals, state=collected_x_vals)
+
+    given_x_vals = [0, 1, 2, 3]
+
+    with meas.run() as datasaver:
+        # Set the minimum queue size of the subscriber to more that
+        # the total number of values being added to the dataset;
+        # this way the subscriber callback is not called before
+        # we exit the "run()" context.
+        subscriber = list(datasaver.dataset.subscribers.values())[0]
+        subscriber.min_queue_length = int(len(given_x_vals) + 1)
+
+        for x in given_x_vals:
+            datasaver.add_result((DAC.ch1, x))
+            # Verify that the subscriber callback is not called yet
+            assert collected_x_vals == []
+
+    # Verify that the subscriber callback is finally called
+    assert collected_x_vals == given_x_vals
+
 
 @settings(deadline=None, max_examples=25)
 @given(N=hst.integers(min_value=2000, max_value=3000))
-def test_subscriptions_getting_all_points(experiment, DAC, DMM, N):
+def test_subscribers_called_for_all_data_points(experiment, DAC, DMM, N):
 
     def sub_get_x_vals(results, length, state):
         """