Repository: spotify/luigi

Issue:
TupleParameter errors in dynamic dependencies
Passing a `TupleParameter` in a dynamic dependency results in a `TypeError` in Python 3.6.5. See the following example for a basic demonstration.

The following code:

```python
import luigi
import luigi.mock
import luigi.util

class FirstTask(luigi.Task):

    # my_tuple = luigi.ListParameter(default=(1, 2, 3))
    my_tuple = luigi.TupleParameter(default=(1, 2, 3))

    @property
    def tuple_string(self):
        return '-'.join(map(str, self.my_tuple))

    def run(self):
        print(self.task_id + ': ' + self.tuple_string)
        with self.output().open('w') as f:
            f.write(self.tuple_string)

    def output(self):
        return luigi.mock.MockTarget(self.task_id)


@luigi.util.inherits(FirstTask)
class SecondTask(luigi.Task):

    def run(self):
        input_target = yield FirstTask(my_tuple=tuple(self.my_tuple))

        with input_target.open('r') as src:
            my_tuple = src.read()
            print(self.task_id + ': ' + my_tuple)

            with self.output().open('w') as dst:
                dst.write(my_tuple)

    def output(self):
        return luigi.mock.MockTarget(self.task_id)

if __name__ == '__main__':
    luigi.build([SecondTask()], local_scheduler=True)
```

... results in this error:

```console
$ python scratch_9.py
DEBUG: Checking if SecondTask(my_tuple=[1, 2, 3]) is complete
INFO: Informed scheduler that task   SecondTask__1__2__3__a7a294ba13   has status   PENDING
INFO: Done scheduling tasks
INFO: Running Worker with 1 processes
DEBUG: Asking scheduler for work...
DEBUG: Pending tasks: 1
INFO: [pid 10635] Worker Worker(salt=723375901, workers=1, host=DGJonathanDuckworth.local, username=jduckworth, pid=10635) running   SecondTask(my_tuple=[1, 2, 3])
INFO: [pid 10635] Worker Worker(salt=723375901, workers=1, host=DGJonathanDuckworth.local, username=jduckworth, pid=10635) new requirements      SecondTask(my_tuple=[1, 2, 3])
DEBUG: 1 running tasks, waiting for next task to finish
INFO: Worker Worker(salt=723375901, workers=1, host=DGJonathanDuckworth.local, username=jduckworth, pid=10635) was stopped. Shutting down Keep-Alive thread
Traceback (most recent call last):
  File "/Users/jduckworth/Library/Preferences/PyCharmCE2018.3/scratches/scratch_9.py", line 40, in <module>
    luigi.build([SecondTask()], local_scheduler=True)
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/interface.py", line 274, in build
    return _schedule_and_run(tasks, worker_scheduler_factory, override_defaults=env_params)['success']
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/interface.py", line 207, in _schedule_and_run
    success &= worker.run()
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/worker.py", line 1162, in run
    self._handle_next_task()
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/worker.py", line 1066, in _handle_next_task
    for module, name, params in new_requirements]
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/worker.py", line 1066, in <listcomp>
    for module, name, params in new_requirements]
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/task_register.py", line 253, in load_task
    return task_cls.from_str_params(params_str)
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/task.py", line 482, in from_str_params
    kwargs[param_name] = param.parse(param_str)
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/parameter.py", line 1107, in parse
    return tuple(tuple(x) for x in json.loads(x, object_pairs_hook=_FrozenOrderedDict))
  File "/Users/jduckworth/.virtualenvs/viper-dev/lib/python3.6/site-packages/luigi/parameter.py", line 1107, in <genexpr>
    return tuple(tuple(x) for x in json.loads(x, object_pairs_hook=_FrozenOrderedDict))
TypeError: 'int' object is not iterable
```

It looks like the problem is that the code [here](https://github.com/spotify/luigi/blob/master/luigi/parameter.py#L1117) catches a `ValueError`, but not a `TypeError`. I haven't tested in Python 2, but I'm assuming a `ValueError` gets thrown in that case and this is just a result of differences in Python versions.

I'm happy to put in a PR if it's helpful.


Discussion:
Hi @duckontheweb Thanks for raising this issue? I'd be happy to review a PR for this issue - just be sure to include a test which fails before the patch and passes after the patch! Thanks!
@dlstadther I'm getting a whole bunch of errors when running `tox -e flake8` even before making any changes. Is there anything I need to do to resolve these before I work on the fix?

Here are the last few lines of the output:

```console
./env/lib/python3.6/site-packages/dateutil/tzwin.py:2:1: F403 'from .tz.win import *' used; unable to detect undefined names
./env/lib/python3.6/site-packages/dateutil/tzwin.py:2:1: F401 '.tz.win.*' imported but unused
./env/lib/python3.6/site-packages/dateutil/rrule.py:24:1: F401 '.tz.tzutc' imported but unused
./env/lib/python3.6/site-packages/dateutil/rrule.py:24:1: F401 '.tz.tzlocal' imported but unused
./env/lib/python3.6/site-packages/dateutil/rrule.py:255:13: E731 do not assign a lambda expression, use a def
./env/lib/python3.6/site-packages/dateutil/rrule.py:257:13: E731 do not assign a lambda expression, use a def
./env/lib/python3.6/site-packages/dateutil/rrule.py:280:9: E741 ambiguous variable name 'l'
./env/lib/python3.6/site-packages/dateutil/rrule.py:432:18: W291 trailing whitespace
./env/lib/python3.6/site-packages/dateutil/rrule.py:764:70: E202 whitespace before '}'
./env/lib/python3.6/site-packages/dateutil/rrule.py:1461:9: E741 ambiguous variable name 'l'
./env/lib/python3.6/site-packages/dateutil/parser/__init__.py:3:1: F401 '._parser.DEFAULTPARSER' imported but unused
./env/lib/python3.6/site-packages/dateutil/parser/__init__.py:3:1: F401 '._parser.DEFAULTTZPARSER' imported but unused
./env/lib/python3.6/site-packages/dateutil/parser/__init__.py:6:1: F401 '._parser.__doc__' imported but unused
./env/lib/python3.6/site-packages/dateutil/parser/__init__.py:35:1: E302 expected 2 blank lines, found 1
./env/lib/python3.6/site-packages/dateutil/parser/__init__.py:43:9: F811 redefinition of unused '__doc__' from line 6
./env/lib/python3.6/site-packages/dateutil/parser/__init__.py:54:1: E402 module level import not at top of file
./env/lib/python3.6/site-packages/dateutil/parser/__init__.py:55:1: E402 module level import not at top of file
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:181:13: E741 ambiguous variable name 'l'
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:232:9: E741 ambiguous variable name 'l'
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:666:50: E231 missing whitespace after ','
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:721:9: E741 ambiguous variable name 'l'
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:1380:9: E741 ambiguous variable name 'l'
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:1380:61: E231 missing whitespace after ','
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:1440:13: E303 too many blank lines (2)
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:1566:78: E231 missing whitespace after ','
./env/lib/python3.6/site-packages/dateutil/parser/_parser.py:1576:1: E302 expected 2 blank lines, found 1
./env/lib/python3.6/site-packages/dateutil/tz/_factories.py:14:1: E302 expected 2 blank lines, found 1
./env/lib/python3.6/site-packages/dateutil/tz/_factories.py:47:17: E128 continuation line under-indented for visual indent
./env/lib/python3.6/site-packages/dateutil/tz/_factories.py:49:1: W391 blank line at end of file
./env/lib/python3.6/site-packages/dateutil/tz/tz.py:325:9: E741 ambiguous variable name 'l'
./env/lib/python3.6/site-packages/dateutil/tz/__init__.py:2:1: F403 'from .tz import *' used; unable to detect undefined names
./env/lib/python3.6/site-packages/dateutil/tz/__init__.py:3:1: F401 '.tz.__doc__' imported but unused
./env/lib/python3.6/site-packages/dateutil/tz/__init__.py:8:7: F405 'tzutc' may be undefined, or defined from star imports: .tz
./env/lib/python3.6/site-packages/dateutil/tz/win.py:110:9: E722 do not use bare 'except'
./env/lib/python3.6/site-packages/dateutil/tz/win.py:126:15: E271 multiple spaces after keyword
./env/lib/python3.6/site-packages/dateutil/tz/win.py:247:59: E128 continuation line under-indented for visual indent
ERROR: InvocationError for command '/Users/jduckworth/managed_code/python/luigi/.tox/flake8/bin/flake8 --exclude=doc,luigi/six.py,.tox' (exited with code 1)
__________________________________________________________________________________________________________________________________ summary ___________________________________________________________________________________________________________________________________
ERROR:   flake8: commands failed
```

Resolving patch:
diff --git a/luigi/parameter.py b/luigi/parameter.py
--- a/luigi/parameter.py
+++ b/luigi/parameter.py
@@ -1114,8 +1114,8 @@ def parse(self, x):
         try:
             # loop required to parse tuple of tuples
             return tuple(tuple(x) for x in json.loads(x, object_pairs_hook=_FrozenOrderedDict))
-        except ValueError:
-            return literal_eval(x)  # if this causes an error, let that error be raised.
+        except (ValueError, TypeError):
+            return tuple(literal_eval(x))  # if this causes an error, let that error be raised.
 
 
 class NumericalParameter(Parameter):


Test patch:
diff --git a/test/parameter_test.py b/test/parameter_test.py
--- a/test/parameter_test.py
+++ b/test/parameter_test.py
@@ -1099,6 +1099,13 @@ def run(self):
         self.assertTrue(self.run_locally(['MainTask']))
 
 
+class TestSerializeTupleParameter(LuigiTestCase):
+    def testSerialize(self):
+        the_tuple = (1, 2, 3)
+
+        self.assertEqual(luigi.TupleParameter().parse(luigi.TupleParameter().serialize(the_tuple)), the_tuple)
+
+
 class NewStyleParameters822Test(LuigiTestCase):
     """
     I bet these tests created at 2015-03-08 are reduntant by now (Oct 2015).