Repository: spotify/luigi

Issue:
tests: Various fixes for external task tests
I found a lot of errors and irrelevant code when I started to
investigate why these tests were failing for me in spotify/luigi#1182.
This patch basically splits an existing test into two tests and also
clarifies how keep_alive and retry_external_tasks differ.



Discussion:

Resolving patch:
diff --git a/luigi/worker.py b/luigi/worker.py
--- a/luigi/worker.py
+++ b/luigi/worker.py
@@ -270,8 +270,8 @@ class worker(Config):
                                   description='worker-count-uniques means that we will keep a '
                                   'worker alive only if it has a unique pending task, as '
                                   'well as having keep-alive true')
-    wait_interval = IntParameter(default=1,
-                                 config_path=dict(section='core', name='worker-wait-interval'))
+    wait_interval = FloatParameter(default=1.0,
+                                   config_path=dict(section='core', name='worker-wait-interval'))
     max_reschedules = IntParameter(default=1,
                                    config_path=dict(section='core', name='worker-max-reschedules'))
     timeout = IntParameter(default=0,
@@ -334,6 +334,9 @@ def __init__(self, scheduler=None, worker_id=None, worker_processes=1, assistant
 
         self._config = worker(**kwargs)
 
+        # multiprocessing.Queue.get() is undefined for timeout=0
+        assert self._config.wait_interval >= 0.00001, "[worker] wait_interval must be positive"
+
         self._id = worker_id
         self._scheduler = scheduler
         self._assistant = assistant
@@ -676,7 +679,7 @@ def _handle_next_task(self):
             try:
                 task_id, status, expl, missing, new_requirements, exception = (
                     self._task_result_queue.get(
-                        timeout=float(self._config.wait_interval)))
+                        timeout=self._config.wait_interval))
             except Queue.Empty:
                 return
 
@@ -731,7 +734,7 @@ def _handle_next_task(self):
     def _sleeper(self):
         # TODO is exponential backoff necessary?
         while True:
-            wait_interval = self._config.wait_interval + random.randint(1, 5)
+            wait_interval = self._config.wait_interval + random.uniform(1, 5)
             logger.debug('Sleeping for %d seconds', wait_interval)
             time.sleep(wait_interval)
             yield


Test patch:
diff --git a/test/worker_external_task_test.py b/test/worker_external_task_test.py
--- a/test/worker_external_task_test.py
+++ b/test/worker_external_task_test.py
@@ -74,20 +74,12 @@ def run(self):
 
 class WorkerExternalTaskTest(unittest.TestCase):
 
-    def setUp(self):
-        self.scheduler = CentralPlannerScheduler(retry_delay=0.01,
-                                                 remove_delay=3,
-                                                 worker_disconnect_delay=3,
-                                                 disable_persist=3,
-                                                 disable_window=5,
-                                                 disable_failures=2,
-                                                 prune_on_get_work=True)
-
     def _assert_complete(self, tasks):
         for t in tasks:
             self.assert_(t.complete())
 
     def _build(self, tasks):
+        self.scheduler = CentralPlannerScheduler(prune_on_get_work=True)
         w = luigi.worker.Worker(scheduler=self.scheduler, worker_processes=1)
         for t in tasks:
             w.add(t)
@@ -113,26 +105,18 @@ def test_external_dependency_already_complete(self):
         # complete() is called once per failure, twice per success
         assert test_task.dependency.times_called == 2
 
-    @with_config({'core': {'retry-external-tasks': 'true',
-                           'disable-num-failures': '4',
-                           'max-reschedules': '4',
-                           'worker-keep-alive': 'true',
-                           'retry-delay': '0.01'}})
-    def test_external_dependency_completes_later(self):
+    @with_config({'worker': {'retry_external_tasks': 'true'},
+                  'scheduler': {'retry_delay': '0.0'}})
+    def test_external_dependency_gets_rechecked(self):
         """
-        Test that an external dependency that is not `complete` when luigi is invoked, but \
-        becomes `complete` while the workflow is executing is re-evaluated and
-        allows dependencies to run.
+        Test that retry_external_tasks re-checks external tasks
         """
-        assert luigi.configuration.get_config().getboolean('core',
-                                                           'retry-external-tasks',
-                                                           False) is True
+        assert luigi.worker.worker().retry_external_tasks is True
 
         tempdir = tempfile.mkdtemp(prefix='luigi-test-')
 
-        with patch('random.randint', return_value=0.1):
-            test_task = TestTask(tempdir=tempdir, complete_after=3)
-            self._build([test_task])
+        test_task = TestTask(tempdir=tempdir, complete_after=10)
+        self._build([test_task])
 
         assert os.path.exists(test_task.dep_path)
         assert os.path.exists(test_task.output_path)
@@ -141,9 +125,32 @@ def test_external_dependency_completes_later(self):
         os.unlink(test_task.output_path)
         os.rmdir(tempdir)
 
-        # complete() is called once per failure, twice per success
-        assert test_task.dependency.times_called == 4
+        self.assertGreaterEqual(test_task.dependency.times_called, 10)
+
+    @with_config({'worker': {'retry_external_tasks': 'true',
+                             'keep_alive': 'true',
+                             'wait_interval': '0.00001'},
+                  'scheduler': {'retry_delay': '0.01'}})
+    def test_external_dependency_worker_is_patient(self):
+        """
+        Test that worker doesn't "give up" with keep_alive option
+
+        Instead, it should sleep for random.uniform() seconds, then ask
+        scheduler for work.
+        """
+        assert luigi.worker.worker().retry_external_tasks is True
+
+        tempdir = tempfile.mkdtemp(prefix='luigi-test-')
 
+        with patch('random.uniform', return_value=0.001):
+            test_task = TestTask(tempdir=tempdir, complete_after=5)
+            self._build([test_task])
+
+        assert os.path.exists(test_task.dep_path)
+        assert os.path.exists(test_task.output_path)
+
+        os.unlink(test_task.dep_path)
+        os.unlink(test_task.output_path)
+        os.rmdir(tempdir)
 
-if __name__ == '__main__':
-    unittest.main()
+        self.assertGreaterEqual(test_task.dependency.times_called, 5)
diff --git a/test/worker_test.py b/test/worker_test.py
--- a/test/worker_test.py
+++ b/test/worker_test.py
@@ -992,5 +992,11 @@ def test_no_task_limit(self):
         self.assertTrue(t.complete())
 
 
-if __name__ == '__main__':
-    luigi.run()
+class WorkerConfigurationTest(unittest.TestCase):
+
+    def test_asserts_for_worker(self):
+        """
+        Test that Worker() asserts that it's sanely configured
+        """
+        Worker(wait_interval=1)  # This shouldn't raise
+        self.assertRaises(AssertionError, Worker, wait_interval=0)