Repository: spotify/luigi

Issue:
Scheduler failing during priority update
My scheduler is failing several times a week during priority update. It starts after midnight when the new daily jobs come in and usually prevents scheduling for 6 hours, although I just kill the scheduler and all workers and flush everything if it lasts longer than that. Clearing state fixes it. The worker fails during scheduling with the traceback

```
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/worker.py", line 299, in add
    for next in self._add(current):
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/worker.py", line 369, in _add
    family=task.task_family)
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/rpc.py", line 110, in add_task
    'params': params,
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/rpc.py", line 86, in _request
    last_exception
RPCError: Errors (3 attempts) when connecting to remote scheduler 'hdwu01.houzz.com'
```

and the scheduler shows traceback 

```
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/tornado/web.py", line 1218, in _when_complete
    callback()
  File "/usr/local/lib/python2.7/dist-packages/tornado/web.py", line 1239, in _execute_method
    self._when_complete(method(*self.path_args, **self.path_kwargs),
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/server.py", line 68, in get
    result = getattr(self._api, method)(**arguments)
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/rpc.py", line 177, in add_task
    resources, priority, family, params)
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/scheduler.py", line 418, in add_task
    self._update_priority(task, priority, worker)
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/scheduler.py", line 365, in _update_priority
    self._update_priority(t, prio, worker)
  File "/usr/local/lib/python2.7/dist-packages/luigi-1.0.19-py2.7.egg/luigi/scheduler.py", line 364, in _update_priority
    if prio > t.priority:
AttributeError: 'NoneType' object has no attribute 'priority'
```

at the same time.



Discussion:

Resolving patch:
diff --git a/luigi/scheduler.py b/luigi/scheduler.py
--- a/luigi/scheduler.py
+++ b/luigi/scheduler.py
@@ -276,8 +276,8 @@ def _update_priority(self, task, prio, worker):
         """
         task.priority = prio = max(prio, task.priority)
         for dep in task.deps or []:
-            t = self._state.get_task(dep) # This should always exist, see add_task
-            if prio > t.priority:
+            t = self._state.get_task(dep)
+            if t is not None and prio > t.priority:
                 self._update_priority(t, prio, worker)
 
     def add_task(self, worker, task_id, status=PENDING, runnable=True,


Test patch:
diff --git a/test/central_planner_test.py b/test/central_planner_test.py
--- a/test/central_planner_test.py
+++ b/test/central_planner_test.py
@@ -204,6 +204,25 @@ def test_scheduler_with_priority_and_competing_resources(self):
         self.sch.add_task(worker='Y', task_id='D', priority=0)
         self.assertEqual(self.sch.get_work(worker='Y')['task_id'], 'D')
 
+    def test_priority_update_with_pruning(self):
+        self.setTime(0)
+        self.sch.add_task(task_id='A', worker='X')
+
+        self.setTime(50)  # after worker disconnects
+        self.sch.prune()
+        self.sch.add_task(task_id='B', deps=['A'], worker='X')
+
+        self.setTime(2000)  # after remove for task A
+        self.sch.prune()
+
+        # Here task A that B depends on is missing
+        self.sch.add_task(WORKER, task_id='C', deps=['B'], priority=100)
+        self.sch.add_task(WORKER, task_id='B', deps=['A'])
+        self.sch.add_task(WORKER, task_id='A')
+        self.sch.add_task(WORKER, task_id='D', priority=10)
+
+        self.check_task_order('ABCD')
+
     def test_update_resources(self):
         self.sch.add_task(WORKER, task_id='A', deps=['B'])
         self.sch.add_task(WORKER, task_id='B', resources={'r': 2})