Repository: dask/distributed

Issue:
Occasional spurious stack trace 
We get the following spurious output, probably because of the combination of our kernel and our virtual environment:

	Traceback (most recent call last):
	  File "lib/python3.5/site-packages/tornado/ioloop.py", line 1026, in _run
		return self.callback()
	  File "lib/python3.5/site-packages/distributed/system_monitor.py", line 38, in update
		ioc = psutil.net_io_counters()
	  File "lib/python3.5/site-packages/psutil/__init__.py", line 2087, in net_io_counters
		rawdict = _psplatform.net_io_counters()
	  File "lib/python3.5/site-packages/psutil/_pslinux.py", line 934, in net_io_counters
		with open_text("%s/net/dev" % get_procfs_path()) as f:
	  File "lib/python3.5/site-packages/psutil/_pslinux.py", line 204, in open_text
		return open(fname, "rt", **kwargs)
	NotADirectoryError: [Errno 20] Not a directory: '/proc/net/dev'

Seems similar to this: https://github.com/DataDog/dd-agent/issues/1175

Wondering if it's easy to quiet down / ignore.


Discussion:
Yes, that's certainly doable.  We could either disable the system monitor entirely or just avoid collecting that data with a try-except block.  
cc @jcrist , if you have time and interest

Resolving patch:
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -14,18 +14,25 @@ def __init__(self, n=10000):
         self.time = deque(maxlen=n)
         self.cpu = deque(maxlen=n)
         self.memory = deque(maxlen=n)
-        self.read_bytes = deque(maxlen=n)
-        self.write_bytes = deque(maxlen=n)
-
-        self.last_time = time()
         self.count = 0
 
-        self._last_io_counters = psutil.net_io_counters()
         self.quantities = {'cpu': self.cpu,
                            'memory': self.memory,
-                           'time': self.time,
-                           'read_bytes': self.read_bytes,
-                           'write_bytes': self.write_bytes}
+                           'time': self.time}
+
+        try:
+            ioc = psutil.net_io_counters()
+        except Exception:
+            self._collect_net_io_counters = False
+        else:
+            self.last_time = time()
+            self.read_bytes = deque(maxlen=n)
+            self.write_bytes = deque(maxlen=n)
+            self.quantities['read_bytes'] = self.read_bytes
+            self.quantities['write_bytes'] = self.write_bytes
+            self._last_io_counters = ioc
+            self._collect_net_io_counters = True
+
         if not WINDOWS:
             self.num_fds = deque(maxlen=n)
             self.quantities['num_fds'] = self.num_fds
@@ -33,28 +40,30 @@ def __init__(self, n=10000):
     def update(self):
         cpu = self.proc.cpu_percent()
         memory = self.proc.memory_info().rss
-
         now = time()
-        ioc = psutil.net_io_counters()
-        last = self._last_io_counters
-        duration = now - self.last_time
-        read_bytes = (ioc.bytes_recv - last.bytes_recv) / (duration or 0.5)
-        write_bytes = (ioc.bytes_sent - last.bytes_sent) / (duration or 0.5)
-        self._last_io_counters = ioc
-        self.last_time = now
 
         self.cpu.append(cpu)
         self.memory.append(memory)
         self.time.append(now)
-
-        self.read_bytes.append(read_bytes)
-        self.write_bytes.append(write_bytes)
-
         self.count += 1
 
-        result = {'cpu': cpu, 'memory': memory, 'time': now,
-                  'count': self.count, 'read_bytes': read_bytes,
-                  'write_bytes': write_bytes}
+        result = {'cpu': cpu,
+                  'memory': memory,
+                  'time': now,
+                  'count': self.count}
+
+        if self._collect_net_io_counters:
+            ioc = psutil.net_io_counters()
+            last = self._last_io_counters
+            duration = now - self.last_time
+            read_bytes = (ioc.bytes_recv - last.bytes_recv) / (duration or 0.5)
+            write_bytes = (ioc.bytes_sent - last.bytes_sent) / (duration or 0.5)
+            self.last_time = now
+            self._last_io_counters = ioc
+            self.read_bytes.append(read_bytes)
+            self.write_bytes.append(write_bytes)
+            result['read_bytes'] = read_bytes
+            result['write_bytes'] = write_bytes
 
         if not WINDOWS:
             num_fds = self.proc.num_fds()


Test patch: