Repository: microsoft/Qcodes

Issue:
Websockets connection not closed leading to hang on monitor restart
The monitor seems to hang due to an improperly closed websockets connection which does not seem to allow the monitor thread to do any more work.

### Steps to reproduce
1. `m = qc.Monitor()`
```
DEBUG:qcodes.monitor.monitor:Start monitoring thread
DEBUG:qcodes.monitor.monitor:Running Websocket server
DEBUG:asyncio:Using selector: SelectSelector
DEBUG:qcodes.monitor.monitor:Start monitoring server
```
2. Navigate to `localhost:3000`
```
DEBUG:websockets.protocol:server - state = CONNECTING
DEBUG:websockets.protocol:server - event = connection_made(<_SelectorSocketTransport fd=1408 read=idle write=<idle, bufsize=0>>)
DEBUG:websockets.protocol:server - state = OPEN
DEBUG:qcodes.monitor.monitor:sending.. to <websockets.server.WebSocketServerProtocol object at 0x000001C5AB01DF98>
DEBUG:websockets.protocol:server > Frame(fin=True, opcode=1, data=b'{"ts": 1543811051.201102, "parameters": []}', rsv1=False, rsv2=False, rsv3=False)
etc...
```
3. Close web browser
```
DEBUG:websockets.protocol:server < Frame(fin=True, opcode=8, data=b'\x03\xe9', rsv1=False, rsv2=False, rsv3=False)
DEBUG:websockets.protocol:server - state = CLOSING
DEBUG:websockets.protocol:server > Frame(fin=True, opcode=8, data=b'\x03\xe9', rsv1=False, rsv2=False, rsv3=False)
DEBUG:websockets.protocol:server x half-closing TCP connection
DEBUG:websockets.protocol:server - event = eof_received()
DEBUG:websockets.protocol:server - event = connection_lost(None)
DEBUG:websockets.protocol:server - state = CLOSED
DEBUG:websockets.protocol:server x code = 1001, reason = [no reason]
DEBUG:qcodes.monitor.monitor:sending.. to <websockets.server.WebSocketServerProtocol object at 0x000002C8A11EEF28>
DEBUG:qcodes.monitor.monitor:WebSocket connection is closed: code = 1001 (going away), no reason
^^^ The last two messages repeat indefinitely
```
5. It's now not possible to join to the monitor:
```
m.join()
DEBUG:qcodes.monitor.monitor:waiting for loop to stop and close
DEBUG:qcodes.monitor.monitor:waiting for loop to stop and close
DEBUG:qcodes.monitor.monitor:waiting for loop to stop and close
^^^ This message continues until KeyboardInterrupt
```

### System
Windows 10
QCodes 2d8fd0b8e0fa12d7921a96003318598ad347dd05
Websockets 7.0


Discussion:

Resolving patch:
diff --git a/qcodes/instrument/parameter.py b/qcodes/instrument/parameter.py
--- a/qcodes/instrument/parameter.py
+++ b/qcodes/instrument/parameter.py
@@ -1364,6 +1364,13 @@ def get(self):
             else:
                 return state['value']
 
+    def get_timestamp(self) -> datetime:
+        """
+        Return the age of the latest parameter value.
+        """
+        state = self.parameter._latest
+        return state["ts"]
+
     def __call__(self):
         return self.get()
 
diff --git a/qcodes/monitor/monitor.py b/qcodes/monitor/monitor.py
--- a/qcodes/monitor/monitor.py
+++ b/qcodes/monitor/monitor.py
@@ -6,29 +6,41 @@
 #
 # Distributed under terms of the MIT license.
 """
-Monitor a set of parameter in a background thread
-stream opuput over websocket
+Monitor a set of parameters in a background thread
+stream output over websocket
+
+To start monitor, run this file, or if qcodes is installed as a module:
+```
+% python -m qcodes.monitor.monitor
+```
+
+Add parameters to monitor in your measurement by creating a new monitor with a list
+of parameters to monitor:
+```
+monitor = qcodes.Monitor(param1, param2, param3, ...)
+```
 """
 
-import asyncio
 import logging
 import os
 import time
 import json
-import http.server
-import socketserver
-import webbrowser
-import datetime
-from copy import deepcopy
 from contextlib import suppress
-
-from threading import Thread
 from typing import Dict, Any
+from collections import defaultdict
+
+import asyncio
 from asyncio import CancelledError
-import functools
+from threading import Thread, Event
 
+import http.server
+import socketserver
+import webbrowser
 import websockets
 
+from qcodes.instrument.parameter import Parameter
+
+WEBSOCKET_PORT = 5678
 SERVER_PORT = 3000
 
 log = logging.getLogger(__name__)
@@ -39,65 +51,70 @@ def _get_metadata(*parameters) -> Dict[str, Any]:
     Return a dict that contains the parameter metadata grouped by the
     instrument it belongs to.
     """
-    ts = time.time()
-    # group meta data by instrument if any
-    metas = {} # type: Dict
+    metadata_timestamp = time.time()
+    # group metadata by instrument
+    metas = defaultdict(list) # type: dict
     for parameter in parameters:
-        _meta = getattr(parameter, "_latest", None)
-        if _meta:
-            meta = deepcopy(_meta)
+        # Get the latest value from the parameter, respecting the max_val_age parameter
+        meta = {}
+        meta["value"] = str(parameter.get_latest())
+        if parameter.get_latest.get_timestamp() is not None:
+            meta["ts"] = parameter.get_latest.get_timestamp().timestamp()
         else:
-            raise ValueError("Input is not a parameter; Refusing to proceed")
-        # convert to string
-        meta['value'] = str(meta['value'])
-        if isinstance(meta["ts"], datetime.datetime):
-            meta["ts"] = time.mktime(meta["ts"].timetuple())
+            meta["ts"] = None
         meta["name"] = parameter.label or parameter.name
         meta["unit"] = parameter.unit
-        # find the base instrument in case this is a channel parameter
-        baseinst = parameter._instrument
-        while hasattr(baseinst, '_parent'):
-            baseinst = baseinst._parent
-        accumulator = metas.get(str(baseinst), [])
-        accumulator.append(meta)
-        metas[str(baseinst)] = accumulator
+
+        # find the base instrument that this parameter belongs to
+        baseinst = parameter.root_instrument
+        if baseinst is None:
+            metas["Unbound Parameter"].append(meta)
+        else:
+            metas[str(baseinst)].append(meta)
+
+    # Create list of parameters, grouped by instrument
     parameters_out = []
     for instrument in metas:
         temp = {"instrument": instrument, "parameters": metas[instrument]}
         parameters_out.append(temp)
-    state = {"ts": ts, "parameters": parameters_out}
+
+    state = {"ts": metadata_timestamp, "parameters": parameters_out}
     return state
 
 
 def _handler(parameters, interval: int):
-
-    async def serverFunc(websocket, path):
+    """
+    Return the websockets server handler
+    """
+    async def server_func(websocket, _):
+        """
+        Create a websockets handler that sends parameter values to a listener
+        every "interval" seconds
+        """
         while True:
             try:
+                # Update the parameter values
                 try:
                     meta = _get_metadata(*parameters)
-                except ValueError as e:
-                    log.exception(e)
+                except ValueError:
+                    log.exception("Error getting parameters")
                     break
-                log.debug(f"sending.. to {websocket}")
-                try:
-                    await websocket.send(json.dumps(meta))
-                # mute browser disconnects
-                except websockets.exceptions.ConnectionClosed as e:
-                    log.debug(e)
+                log.debug("sending.. to %r", websocket)
+                await websocket.send(json.dumps(meta))
+                # Wait for interval seconds and then send again
                 await asyncio.sleep(interval)
-            except CancelledError:
-                log.debug("Got CancelledError")
+            except (CancelledError, websockets.exceptions.ConnectionClosed):
+                log.debug("Got CancelledError or ConnectionClosed", exc_info=True)
                 break
+        log.debug("Closing websockets connection")
 
-        log.debug("Stopping Websocket handler")
-
-    return serverFunc
-
+    return server_func
 
 class Monitor(Thread):
+    """
+    QCodes Monitor - WebSockets server to monitor qcodes parameters
+    """
     running = None
-    server = None
 
     def __init__(self, *parameters, interval=1):
         """
@@ -107,12 +124,31 @@ def __init__(self, *parameters, interval=1):
             *parameters: Parameters to monitor
             interval: How often one wants to refresh the values
         """
-        # let the thread start
-        time.sleep(0.01)
         super().__init__()
+
+        # Check that all values are valid parameters
+        for parameter in parameters:
+            if not isinstance(parameter, Parameter):
+                raise TypeError(f"We can only monitor QCodes Parameters, not {type(parameter)}")
+
         self.loop = None
+        self.server = None
         self._parameters = parameters
-        self._monitor(*parameters, interval=interval)
+        self.loop_is_closed = Event()
+        self.server_is_started = Event()
+        self.handler = _handler(parameters, interval=interval)
+
+        log.debug("Start monitoring thread")
+        if Monitor.running:
+            # stop the old server
+            log.debug("Stopping and restarting server")
+            Monitor.running.stop()
+        self.start()
+
+        # Wait until the loop is running
+        self.server_is_started.wait(timeout=5)
+        if not self.server_is_started.is_set():
+            raise RuntimeError("Failed to start server")
         Monitor.running = self
 
     def run(self):
@@ -121,32 +157,33 @@ def run(self):
         """
         log.debug("Running Websocket server")
         self.loop = asyncio.new_event_loop()
-        self.loop_is_closed = False
         asyncio.set_event_loop(self.loop)
         try:
-            server_start = websockets.serve(self.handler, '127.0.0.1', 5678)
+            server_start = websockets.serve(self.handler, '127.0.0.1',
+                                            WEBSOCKET_PORT, close_timeout=1)
             self.server = self.loop.run_until_complete(server_start)
+            self.server_is_started.set()
             self.loop.run_forever()
-        except OSError as e:
+        except OSError:
             # The code above may throw an OSError
             # if the socket cannot be bound
-            log.exception(e)
+            log.exception("Server could not be started")
         finally:
             log.debug("loop stopped")
-            log.debug("Pending tasks at close: {}".format(
-                asyncio.Task.all_tasks(self.loop)))
+            log.debug("Pending tasks at close: %r",
+                      asyncio.Task.all_tasks(self.loop))
             self.loop.close()
-            while not self.loop.is_closed():
-                log.debug("waiting for loop to stop and close")
-                time.sleep(0.01)
-            self.loop_is_closed = True
             log.debug("loop closed")
+            self.loop_is_closed.set()
 
     def update_all(self):
-        for p in self._parameters:
+        """
+        Update all parameters in the monitor
+        """
+        for parameter in self._parameters:
             # call get if it can be called without arguments
             with suppress(TypeError):
-                p.get()
+                parameter.get()
 
     def stop(self) -> None:
         """
@@ -157,12 +194,13 @@ def stop(self) -> None:
         Monitor.running = None
 
     async def __stop_server(self):
-        log.debug("asking server to close")
+        log.debug("asking server %r to close", self.server)
         self.server.close()
         log.debug("waiting for server to close")
         await self.loop.create_task(self.server.wait_closed())
         log.debug("stopping loop")
-        log.debug("Pending tasks at stop: {}".format(asyncio.Task.all_tasks(self.loop)))
+        log.debug("Pending tasks at stop: %r",
+                  asyncio.Task.all_tasks(self.loop))
         self.loop.stop()
 
     def join(self, timeout=None) -> None:
@@ -171,15 +209,20 @@ def join(self, timeout=None) -> None:
         joining avoiding a potential deadlock.
         """
         log.debug("Shutting down server")
+        if not self.is_alive():
+            # we run this check before trying to run to prevent a cryptic
+            # error message
+            log.debug("monitor is dead")
+            return
         try:
             asyncio.run_coroutine_threadsafe(self.__stop_server(), self.loop)
-        except RuntimeError as e:
+        except RuntimeError:
             # the above may throw a runtime error if the loop is already
             # stopped in which case there is nothing more to do
             log.exception("Could not close loop")
-        while not self.loop_is_closed:
-            log.debug("waiting for loop to stop and close")
-            time.sleep(0.01)
+        self.loop_is_closed.wait(timeout=5)
+        if not self.loop_is_closed.is_set():
+            raise RuntimeError("Failed to join loop")
         log.debug("Loop reported closed")
         super().join(timeout=timeout)
         log.debug("Monitor Thread has joined")
@@ -200,47 +243,17 @@ def show():
         """
         webbrowser.open("http://localhost:{}".format(SERVER_PORT))
 
-    def _monitor(self, *parameters, interval=1):
-        self.handler = _handler(parameters, interval=interval)
-        # TODO (giulioungaretti) read from config
-
-        log.debug("Start monitoring thread")
-
-        if Monitor.running:
-            # stop the old server
-            log.debug("Stopping and restarting server")
-            Monitor.running.stop()
-
-        self.start()
-
-        # let the thread start
-        time.sleep(0.01)
-        log.debug("Start monitoring server")
-
-
-class Server():
-
-    def __init__(self, port=3000):
-        self.port = port
-        self.handler = http.server.SimpleHTTPRequestHandler
-        self.httpd = socketserver.TCPServer(("", self.port), self.handler)
-        self.static_dir = os.path.join(os.path.dirname(__file__), 'dist')
-
-    def run(self):
-        os.chdir(self.static_dir)
-        log.debug("serving directory %s", self.static_dir)
-        log.info("Open browser at http://localhost::{}".format(self.port))
-        self.httpd.serve_forever()
-
-    def stop(self):
-        self.httpd.shutdown()
-
-
 if __name__ == "__main__":
-    server = Server(SERVER_PORT)
-    print("Open browser at http://localhost:{}".format(server.port))
+    # If this file is run, create a simple webserver that serves a simple website
+    # that can be used to view monitored parameters.
+    STATIC_DIR = os.path.join(os.path.dirname(__file__), 'dist')
+    os.chdir(STATIC_DIR)
     try:
-        webbrowser.open("http://localhost:{}".format(server.port))
-        server.run()
+        log.info("Starting HTTP Server at http://localhost:%i", SERVER_PORT)
+        with socketserver.TCPServer(("", SERVER_PORT),
+                                    http.server.SimpleHTTPRequestHandler) as httpd:
+            log.debug("serving directory %s", STATIC_DIR)
+            webbrowser.open("http://localhost:{}".format(SERVER_PORT))
+            httpd.serve_forever()
     except KeyboardInterrupt:
-        exit()
+        log.info("Shutting Down HTTP Server")


Test patch:
diff --git a/qcodes/tests/test_monitor.py b/qcodes/tests/test_monitor.py
new file mode 100644
--- /dev/null
+++ b/qcodes/tests/test_monitor.py
@@ -0,0 +1,146 @@
+"""
+Test suite for monitor
+"""
+from unittest import TestCase
+
+import asyncio
+import json
+import random
+import websockets
+
+from qcodes.monitor import monitor
+from qcodes.instrument.base import Parameter
+from qcodes.tests.instrument_mocks import DummyInstrument
+
+monitor.WEBSOCKET_PORT = random.randint(50000, 60000)
+
+class TestMonitor(TestCase):
+    """
+    Test cases for the qcodes monitor
+    """
+    def test_setup_teardown(self):
+        """
+        Check that monitor starts up and closes correctly
+        """
+        m = monitor.Monitor()
+        self.assertTrue(m.is_alive())
+        self.assertTrue(m.loop.is_running())
+        self.assertEqual(monitor.Monitor.running, m)
+        m.stop()
+        self.assertFalse(m.loop.is_running())
+        self.assertTrue(m.loop.is_closed())
+        self.assertFalse(m.is_alive())
+        self.assertIsNone(monitor.Monitor.running)
+
+    def test_monitor_replace(self):
+        """
+        Check that monitors get correctly replaced
+        """
+        m = monitor.Monitor()
+        self.assertEqual(monitor.Monitor.running, m)
+        m2 = monitor.Monitor()
+        self.assertEqual(monitor.Monitor.running, m2)
+        self.assertTrue(m.loop.is_closed())
+        self.assertFalse(m.is_alive())
+        self.assertTrue(m2.is_alive())
+        m2.stop()
+
+    def test_double_join(self):
+        """
+        Check that a double join doesn't cause a hang
+        """
+        m = monitor.Monitor()
+        self.assertEqual(monitor.Monitor.running, m)
+        m.stop()
+        m.stop()
+
+
+    def test_connection(self):
+        """
+        Test that we can connect to a monitor instance
+        """
+        m = monitor.Monitor()
+
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        @asyncio.coroutine
+        def async_test_connection():
+            websocket = yield from websockets.connect(f"ws://localhost:{monitor.WEBSOCKET_PORT}")
+            yield from websocket.close()
+        loop.run_until_complete(async_test_connection())
+
+        m.stop()
+
+class TestMonitorWithInstr(TestCase):
+    """
+    Test monitor values from instruments
+    """
+    def setUp(self):
+        """
+        Create a dummy instrument for use in monitor tests, and hook it into
+        a monitor
+        """
+        self.instr = DummyInstrument("MonitorDummy")
+        self.param = Parameter("DummyParam",
+                               unit="V",
+                               get_cmd=None,
+                               set_cmd=None)
+        self.param(1)
+        self.monitor_parameters = tuple(self.instr.parameters.values())[1:]
+        self.monitor = monitor.Monitor(*self.monitor_parameters, self.param, interval=0.1)
+
+    def tearDown(self):
+        """
+        Close the dummy instrument
+        """
+        self.monitor.stop()
+        self.instr.close()
+
+    def test_parameter(self):
+        """
+        Test instrument updates
+        """
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        @asyncio.coroutine
+        def async_test_monitor():
+            websocket = yield from websockets.connect(f"ws://localhost:{monitor.WEBSOCKET_PORT}")
+
+            # Recieve data from monitor
+            data = yield from websocket.recv()
+            data = json.loads(data)
+            # Check fields
+            self.assertIn("ts", data)
+            self.assertIn("parameters", data)
+            # Check one instrument and "No Instrument" is being sent
+            self.assertEqual(len(data["parameters"]), 2)
+            self.assertEqual(data["parameters"][1]["instrument"], "Unbound Parameter")
+            metadata = data["parameters"][0]
+            self.assertEqual(metadata["instrument"], str(self.instr))
+
+            # Check parameter values
+            old_timestamps = {}
+            for local, mon in zip(self.monitor_parameters, metadata["parameters"]):
+                self.assertEqual(str(local.get_latest()), mon["value"])
+                self.assertEqual(local.label, mon["name"])
+                old_timestamps[local.label] = float(mon["ts"])
+                local(random.random())
+
+            # Check parameter updates
+            data = yield from websocket.recv()
+            data = yield from websocket.recv()
+            data = json.loads(data)
+            metadata = data["parameters"][0]
+            for local, mon in zip(self.monitor_parameters, metadata["parameters"]):
+                self.assertEqual(str(local.get_latest()), mon["value"])
+                self.assertEqual(local.label, mon["name"])
+                self.assertGreater(float(mon["ts"]), old_timestamps[local.label])
+
+            # Check unbound parameter
+            metadata = data["parameters"][1]["parameters"]
+            self.assertEqual(len(metadata), 1)
+            self.assertEqual(self.param.label, metadata[0]["name"])
+
+        loop.run_until_complete(async_test_monitor())
diff --git a/qcodes/tests/test_parameter.py b/qcodes/tests/test_parameter.py
--- a/qcodes/tests/test_parameter.py
+++ b/qcodes/tests/test_parameter.py
@@ -6,6 +6,7 @@
 from unittest import TestCase
 from typing import Tuple
 import pytest
+from datetime import datetime
 
 import numpy as np
 from hypothesis import given, event, settings
@@ -181,6 +182,22 @@ def test_snapshot_value(self):
         snap = p_no_snapshot.snapshot()
         self.assertNotIn('value', snap)
 
+    def test_get_latest(self):
+        # Create a gettable parameter
+        local_parameter = Parameter('test_param', set_cmd=None, get_cmd=None)
+        before_set = datetime.now()
+        local_parameter.set(1)
+        after_set = datetime.now()
+
+        # Check we return last set value, with the correct timestamp
+        self.assertEqual(local_parameter.get_latest(), 1)
+        self.assertTrue(before_set <= local_parameter.get_latest.get_timestamp() <= after_set)
+
+        # Check that updating the value updates the timestamp
+        local_parameter.set(2)
+        self.assertEqual(local_parameter.get_latest(), 2)
+        self.assertGreaterEqual(local_parameter.get_latest.get_timestamp(), after_set)
+
     def test_has_set_get(self):
         # Create parameter that has no set_cmd, and get_cmd returns last value
         gettable_parameter = Parameter('one', set_cmd=False, get_cmd=None)