Repository: dask/distributed

Issue:
_reevaluate_occupancy_worker() resets processing estimate
This is an intermittent test failure:
https://travis-ci.org/dask/distributed/jobs/235929242#L1472-L1521

Basically, `transition_waiting_processing` updates `processing` with the expected task duration plus the estimated communication cost, but `_reevaluate_occupancy_worker` can run just after and reset the `processing` estimate to its former value from `task_duration`.

I admit I don't understand why `_reevaluate_occupancy_worker` does this. Why does it mutate `processing` at all? `processing` is already maintained in real time in various state transitions...


Discussion:
Actually, even `occupancy` is already updated in other transitions.
Original PR here: https://github.com/dask/distributed/pull/1038

As estimated task durations change the values in processing do not change.  However this change was intentionally experimental.  There may be more work to do here.

Resolving patch:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1229,7 +1229,7 @@ def send_task_to_worker(self, worker, key):
             msg = {'op': 'compute-task',
                    'key': key,
                    'priority': self.priority[key],
-                   'duration': self.task_duration.get(key_split(key), 0.5)}
+                   'duration': self.get_task_duration(key)}
             if key in self.resource_restrictions:
                 msg['resource_restrictions'] = self.resource_restrictions[key]
 
@@ -1980,6 +1980,27 @@ def get_nbytes(self, comm=None, keys=None, summary=True):
 
             return result
 
+    def get_comm_cost(self, key, worker):
+        """
+        Get the estimated communication cost (in s.) to compute key
+        on the given worker.
+        """
+        return (sum(self.nbytes[d] for d in
+                    self.dependencies[key] - self.has_what[worker])
+                / BANDWIDTH)
+
+    def get_task_duration(self, key, default=0.5):
+        """
+        Get the estimated computation cost of the given key
+        (not including any communication cost).
+        """
+        ks = key_split(key)
+        try:
+            return self.task_duration[ks]
+        except KeyError:
+            self.unknown_durations[ks].add(key)
+            return default
+
     def run_function(self, stream, function, args=(), kwargs={}):
         """ Run a function within this process
 
@@ -2143,14 +2164,8 @@ def transition_waiting_processing(self, key):
 
             ks = key_split(key)
 
-            duration = self.task_duration.get(ks)
-            if duration is None:
-                self.unknown_durations[ks].add(key)
-                duration = 0.5
-
-            comm = (sum(self.nbytes[dep]
-                       for dep in self.dependencies[key] - self.has_what[worker])
-                    / BANDWIDTH)
+            duration = self.get_task_duration(ks)
+            comm = self.get_comm_cost(key, worker)
 
             self.processing[worker][key] = duration + comm
             self.rprocessing[key] = worker
@@ -2291,9 +2306,7 @@ def transition_processing_memory(self, key, nbytes=None, type=None,
                         if k in self.rprocessing:
                             w = self.rprocessing[k]
                             old = self.processing[w][k]
-                            comm = (sum(self.nbytes[d] for d in
-                                         self.dependencies[k] - self.has_what[w])
-                                    / BANDWIDTH)
+                            comm = self.get_comm_cost(k, w)
                             self.processing[w][k] = avg_duration + comm
                             self.occupancy[w] += avg_duration + comm - old
                             self.total_occupancy += avg_duration + comm - old
@@ -3076,19 +3089,21 @@ def reevaluate_occupancy(self):
         lets us avoid this fringe optimization when we have better things to
         think about.
         """
+        DELAY = 0.1
         with log_errors():
             import psutil
             proc = psutil.Process()
             last = time()
+
             while self.status != 'closed':
-                yield gen.sleep(0.100)
+                yield gen.sleep(DELAY)
                 while not self.rprocessing:
-                    yield gen.sleep(0.100)
-                    last = time()
+                    yield gen.sleep(DELAY)
+                last = time()
 
                 for w, processing in list(self.processing.items()):
                     while proc.cpu_percent() > 50:
-                        yield gen.sleep(0.100)
+                        yield gen.sleep(DELAY)
                         last = time()
 
                     if w not in self.workers or not processing:
@@ -3112,15 +3127,12 @@ def _reevaluate_occupancy_worker(self, worker):
         new = 0
         nbytes = 0
         for key in processing:
-            duration = self.task_duration.get(key_split(key), 0.5)
-            processing[key] = duration
-            new += duration
-            for dep in self.dependencies[key]:
-                if dep not in self.has_what[w]:
-                    nbytes += self.nbytes.get(key, 0)
+            duration = self.get_task_duration(key)
+            comm = self.get_comm_cost(key, worker)
+            processing[key] = duration + comm
+            new += duration + comm
 
-        comm = nbytes / BANDWIDTH
-        self.occupancy[w] = max(new, comm)  # These overlap. Take maximum
+        self.occupancy[w] = new
         self.total_occupancy += new - old
         self.check_idle_saturated(w)
 


Test patch:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -878,7 +878,14 @@ def test_include_communication_in_occupancy(c, s, a, b):
     while z.key not in s.rprocessing:
         yield gen.sleep(0.01)
 
-    assert s.processing[b.address][z.key] > 1
+    try:
+        assert s.processing[b.address][z.key] > 1
+    except Exception:
+        print("processing:", s.processing)
+        print("rprocessing:", s.rprocessing)
+        print("task_duration:", s.task_duration)
+        print("nbytes:", s.nbytes)
+        raise
 
 
 @gen_cluster(client=True)