Repository: deepset-ai/haystack

Issue:
`MultiLabel` should not change the order of passed in labels
**Describe the bug**
During deduping labels the order of labels passed in is changed. If I passed in non-duplicate items I would expect to get them from `MultiLabel.labels` in the same order as passed in. The order is important as the `EvaluationResult` dataframes rely on this order for some fields (e.g. `gold_answers`, `gold_answers_f1`, etc.)

The changed order is a result of converting the list to a set and back again:
https://github.com/deepset-ai/haystack/blob/e766bb86840392676c650adefb0ff5b4be7d9333/haystack/schema.py#L627

**Error message**
None

**Expected behavior**
Labels passed to `MultiLabel` can be retrieved in the same order by `MultiLabel.labels`.

**To Reproduce**
Pass list of (non-duplicate) labels to `MultiLabel` constructor. Access `MultiLabel.labels`. As set order is random, the order will be random.


Discussion:

Resolving patch:
diff --git a/haystack/schema.py b/haystack/schema.py
--- a/haystack/schema.py
+++ b/haystack/schema.py
@@ -624,7 +624,7 @@ def __init__(self, labels: List[Label], drop_negative_labels=False, drop_no_answ
         :param drop_no_answers: Whether to drop labels that specify the answer is impossible
         """
         # drop duplicate labels and remove negative labels if needed.
-        labels = list(set(labels))
+        labels = list(dict.fromkeys(labels))
         if drop_negative_labels:
             labels = [l for l in labels if is_positive_label(l)]
 


Test patch:
diff --git a/test/others/test_schema.py b/test/others/test_schema.py
--- a/test/others/test_schema.py
+++ b/test/others/test_schema.py
@@ -239,6 +239,128 @@ def test_aggregate_labels_with_labels():
         label = MultiLabel(labels=[label1_with_filter1, label3_with_filter2])
 
 
+def test_multilabel_preserve_order():
+    labels = [
+        Label(
+            id="0",
+            query="question",
+            answer=Answer(answer="answer1", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some", id="123"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+        Label(
+            id="1",
+            query="question",
+            answer=Answer(answer="answer2", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some", id="123"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+        Label(
+            id="2",
+            query="question",
+            answer=Answer(answer="answer3", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some other", id="333"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+        Label(
+            id="3",
+            query="question",
+            answer=Answer(answer="", offsets_in_document=[Span(start=0, end=0)]),
+            document=Document(content="some", id="777"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=True,
+            origin="gold-label",
+        ),
+        Label(
+            id="4",
+            query="question",
+            answer=Answer(answer="answer5", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some", id="123"),
+            is_correct_answer=False,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+    ]
+
+    multilabel = MultiLabel(labels=labels)
+
+    for i in range(0, 5):
+        assert multilabel.labels[i].id == str(i)
+
+
+def test_multilabel_preserve_order_w_duplicates():
+    labels = [
+        Label(
+            id="0",
+            query="question",
+            answer=Answer(answer="answer1", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some", id="123"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+        Label(
+            id="1",
+            query="question",
+            answer=Answer(answer="answer2", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some", id="123"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+        Label(
+            id="2",
+            query="question",
+            answer=Answer(answer="answer3", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some other", id="333"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+        Label(
+            id="0",
+            query="question",
+            answer=Answer(answer="answer1", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some", id="123"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+        Label(
+            id="2",
+            query="question",
+            answer=Answer(answer="answer3", offsets_in_document=[Span(start=12, end=18)]),
+            document=Document(content="some other", id="333"),
+            is_correct_answer=True,
+            is_correct_document=True,
+            no_answer=False,
+            origin="gold-label",
+        ),
+    ]
+
+    multilabel = MultiLabel(labels=labels)
+
+    assert len(multilabel.document_ids) == 3
+
+    for i in range(0, 3):
+        assert multilabel.labels[i].id == str(i)
+
+
 def test_serialize_speech_document():
     speech_doc = SpeechDocument(
         id=12345,