Repository: spotify/luigi

Issue:
Duplicated outputs of luigi.task.flatten_output
**Description**
`luigi.task.flatten_output` does not detect "diamond" problem. If some task is required by more than one wrapper task, its output is duplicated in returned list. 

**Simple reproduction:**
```python
import luigi
from luigi.task import flatten_output
from luigi.util import requires


class TestTask(luigi.ExternalTask):
    def output(self):
        return luigi.LocalTarget("file.txt")
    

@requires(TestTask)
class WrapperOne(luigi.WrapperTask):
    pass
        

@requires(TestTask)
class WrapperTwo(luigi.WrapperTask):
    pass
        

@requires(WrapperOne, WrapperTwo)
class WrapperMaster(luigi.WrapperTask):
    pass
        
        
print([target.path for target in flatten_output(WrapperMaster())])
```

Actual output (`luigi==3.0.3`): `['file.txt', 'file.txt']`
Expected output: `['file.txt']`


Discussion:
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. If closed, you may revisit when your time allows and reopen! Thank you for your contributions.


Resolving patch:
diff --git a/luigi/task.py b/luigi/task.py
--- a/luigi/task.py
+++ b/luigi/task.py
@@ -19,7 +19,7 @@
 It is a central concept of Luigi and represents the state of the workflow.
 See :doc:`/tasks` for an overview.
 """
-
+from collections import deque, OrderedDict
 from contextlib import contextmanager
 import logging
 import traceback
@@ -955,7 +955,7 @@ def getpaths(struct):
 
 def flatten(struct):
     """
-    Creates a flat list of all all items in structured output (dicts, lists, items):
+    Creates a flat list of all items in structured output (dicts, lists, items):
 
     .. code-block:: python
 
@@ -992,14 +992,19 @@ def flatten(struct):
 def flatten_output(task):
     """
     Lists all output targets by recursively walking output-less (wrapper) tasks.
-
-    FIXME order consistently.
     """
-    r = flatten(task.output())
-    if not r:
-        for dep in flatten(task.requires()):
-            r += flatten_output(dep)
-    return r
+
+    output_tasks = OrderedDict()  # OrderedDict used as ordered set
+    tasks_to_process = deque([task])
+    while tasks_to_process:
+        current_task = tasks_to_process.popleft()
+        if flatten(current_task.output()):
+            if current_task not in output_tasks:
+                output_tasks[current_task] = None
+        else:
+            tasks_to_process.extend(flatten(current_task.requires()))
+
+    return flatten(task.output() for task in output_tasks)
 
 
 def _task_wraps(task_class):


Test patch:
diff --git a/test/task_test.py b/test/task_test.py
--- a/test/task_test.py
+++ b/test/task_test.py
@@ -219,6 +219,55 @@ class TaskB(luigi.Task):
                 )
 
 
+class TaskFlattenOutputTest(unittest.TestCase):
+    def test_single_task(self):
+        expected = [luigi.LocalTarget("f1.txt"), luigi.LocalTarget("f2.txt")]
+
+        class TestTask(luigi.ExternalTask):
+            def output(self):
+                return expected
+
+        self.assertListEqual(luigi.task.flatten_output(TestTask()), expected)
+
+    def test_wrapper_task(self):
+        expected = [luigi.LocalTarget("f1.txt"), luigi.LocalTarget("f2.txt")]
+
+        class Test1Task(luigi.ExternalTask):
+            def output(self):
+                return expected[0]
+
+        class Test2Task(luigi.ExternalTask):
+            def output(self):
+                return expected[1]
+
+        @luigi.util.requires(Test1Task, Test2Task)
+        class TestWrapperTask(luigi.WrapperTask):
+            pass
+
+        self.assertListEqual(luigi.task.flatten_output(TestWrapperTask()), expected)
+
+    def test_wrapper_tasks_diamond(self):
+        expected = [luigi.LocalTarget("file.txt")]
+
+        class TestTask(luigi.ExternalTask):
+            def output(self):
+                return expected
+
+        @luigi.util.requires(TestTask)
+        class LeftWrapperTask(luigi.WrapperTask):
+            pass
+
+        @luigi.util.requires(TestTask)
+        class RightWrapperTask(luigi.WrapperTask):
+            pass
+
+        @luigi.util.requires(LeftWrapperTask, RightWrapperTask)
+        class MasterWrapperTask(luigi.WrapperTask):
+            pass
+
+        self.assertListEqual(luigi.task.flatten_output(MasterWrapperTask()), expected)
+
+
 class ExternalizeTaskTest(LuigiTestCase):
 
     def test_externalize_taskclass(self):