Repository: spotify/luigi

Issue:
MissingParameterException when generating execution summary
Generating the execution summary may generate a MissingParameterException if a `requires` attempts to access a Config parameter which was specified only on the command line.

Example:

```python
# trouble.py
import luigi

class conf(luigi.Config):
    opt = luigi.Parameter()

class Root(luigi.WrapperTask):
    def requires(self):
        if conf().opt == 'A':
            return A()

class A(luigi.Task):
    def complete(self):
        return False

    def run(self):
        raise Exception()
```

```
$ luigi --module trouble Root --conf-opt=A
===== Luigi Execution Summary =====

Scheduled 2 tasks of which:
* 1 failed:
    - 1 A()
* 1 were left pending, among these:
    * 1 had failed dependencies:
        - 1 Root()

This progress looks :( because there were failed tasks

===== Luigi Execution Summary =====

Traceback (most recent call last):
  File "/opt/etl-virtualenv/bin/luigi", line 11, in <module>
    sys.exit(luigi_run())
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/cmdline.py", line 11, in luigi_run
    run_with_retcodes(argv)
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/retcodes.py", line 83, in run_with_retcodes
    task_sets = luigi.execution_summary._summary_dict(worker)
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/execution_summary.py", line 337, in _summary_dict
    _populate_unknown_statuses(set_tasks)
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/execution_summary.py", line 75, in _populate_unknown_statuses
    _depth_first_search(set_tasks, task, visited)
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/execution_summary.py", line 88, in _depth_first_search
    for task in current_task._requires():
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/task.py", line 607, in _requires
    return flatten(self.requires())  # base impl
  File "/private/tmp/trouble.py", line 8, in requires
    if conf().opt == 'A':
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/task_register.py", line 88, in __call__
    param_values = cls.get_param_values(params, args, kwargs)
  File "/opt/etl-virtualenv/lib/python3.6/site-packages/luigi/task.py", line 412, in get_param_values
    raise parameter.MissingParameterException("%s: requires the '%s' parameter to be set" % (exc_desc, param_name))
luigi.parameter.MissingParameterException: conf[args=(), kwargs={}]: requires the 'opt' parameter to be set
```

This seems to be because the execution summary is generated outside any CmdlineParser context manager in run_with_retcodes. So this should be fairly easy to avoid by extending the entire run_with_retcodes to be within the CmdlineParser cm already there for the `retcode()` config - or if that could cause side effects I am unaware of then a separate context just for the call to _summary_dict.

I can attempt a PR for either approach.

May be related to #1964


Discussion:
Nice catch. I'm confused by the example run you have pasted. It looks like the execution summary is still printed?

I guess it makes sense to use the same CmdlineParser context managerand put the execution summary generation in there. The damage of global variables it seems.

Resolving patch:
diff --git a/luigi/retcodes.py b/luigi/retcodes.py
--- a/luigi/retcodes.py
+++ b/luigi/retcodes.py
@@ -80,9 +80,10 @@ def run_with_retcodes(argv):
         logger.exception("Uncaught exception in luigi")
         sys.exit(retcodes.unhandled_exception)
 
-    task_sets = luigi.execution_summary._summary_dict(worker)
-    root_task = luigi.execution_summary._root_task(worker)
-    non_empty_categories = {k: v for k, v in task_sets.items() if v}.keys()
+    with luigi.cmdline_parser.CmdlineParser.global_instance(argv):
+        task_sets = luigi.execution_summary._summary_dict(worker)
+        root_task = luigi.execution_summary._root_task(worker)
+        non_empty_categories = {k: v for k, v in task_sets.items() if v}.keys()
 
     def has(status):
         assert status in luigi.execution_summary._ORDERED_STATUSES


Test patch:
diff --git a/test/cmdline_test.py b/test/cmdline_test.py
--- a/test/cmdline_test.py
+++ b/test/cmdline_test.py
@@ -90,6 +90,24 @@ def run(self):
         raise ValueError()
 
 
+class RequiredConfig(luigi.Config):
+    required_test_param = luigi.Parameter()
+
+
+class TaskThatRequiresConfig(luigi.WrapperTask):
+    def requires(self):
+        if RequiredConfig().required_test_param == 'A':
+            return SubTaskThatFails()
+
+
+class SubTaskThatFails(luigi.Task):
+    def complete(self):
+        return False
+
+    def run(self):
+        raise Exception()
+
+
 class CmdlineTest(unittest.TestCase):
 
     def setUp(self):
@@ -339,6 +357,20 @@ def test_stack_trace_has_no_inner(self):
         self.assertFalse(stdout.find(b"run() got an unexpected keyword argument 'tracking_url_callback'") != -1)
         self.assertFalse(stdout.find(b'During handling of the above exception, another exception occurred') != -1)
 
+    def test_cmd_line_params_are_available_for_execution_summary(self):
+        """
+        Test that config parameters specified on the command line are available while generating the execution summary.
+        """
+        returncode, stdout, stderr = self._run_cmdline([
+            './bin/luigi', '--module', 'cmdline_test', 'TaskThatRequiresConfig', '--local-scheduler', '--no-lock'
+            '--RequiredConfig-required-test-param', 'A',
+        ])
+        print(stdout)
+        print(stderr)
+
+        self.assertNotEquals(returncode, 1)
+        self.assertFalse(b'required_test_param' in stderr)
+
 
 if __name__ == '__main__':
     # Needed for one of the tests