Repository: e-valuation/EvaP

Issue:
Dictionary merge error in course importer
the following code wants to merge error dicts. however, this fails with a `TypeError` (`unsupported operand type(s) for |: 'str' and 'str'`) in `merge_dictionaries_of_sets`.
https://github.com/e-valuation/EvaP/blob/32caa9d799a938f88fa49940b16f4f44bd89d431/evap/staff/importers.py#L413-L415


reproduce by importing a file with two entries: two participants having the same course that differs in its degree (e.g., `BA` and `MA`) and having another error (e.g. grade `ungraded` instead of `no`).


Discussion:
This happens because the `errors` aren't actually shaped like this function assumes, looking at `EvaluationDataFactory`, we can see that the `is_graded` and `course_type` keys actually map to single strings, not a `set` like `errors["degrees"]`.

We could fix this by either only merging the keys that we know are `set`s, or make all value types `set` too (and add a type hint?). At first, I though that since there could be multiple wrong course types or `is_graded`-strings for the same evaluation, we should do the second approach. _But_, since this error only happens if the two rows are `equal[s]_except_for_degrees`, the first is probably more appropriate.

Resolving patch:
diff --git a/evap/staff/importers.py b/evap/staff/importers.py
--- a/evap/staff/importers.py
+++ b/evap/staff/importers.py
@@ -14,7 +14,7 @@
 
 from evap.evaluation.models import Contribution, Course, CourseType, Degree, Evaluation, UserProfile
 from evap.evaluation.tools import clean_email
-from evap.staff.tools import ImportType, create_user_list_html_string_for_message, merge_dictionaries_of_sets
+from evap.staff.tools import ImportType, create_user_list_html_string_for_message
 
 
 def sorted_messages(messages):
@@ -402,18 +402,25 @@ def process_evaluation(self, evaluation_data, sheetname, row):
                 self.evaluations[evaluation_id] = evaluation_data
                 self.names_de.add(evaluation_data.name_de)
         else:
-            if evaluation_data.equals_except_for_degrees(self.evaluations[evaluation_id]):
+            known_data = self.evaluations[evaluation_id]
+            if evaluation_data.equals_except_for_degrees(known_data):
                 self.warnings[ImporterWarning.DEGREE].append(
                     _(
                         'Sheet "{}", row {}: The course\'s "{}" degree differs from it\'s degree in a previous row.'
                         " Both degrees have been set for the course."
                     ).format(sheetname, row + 1, evaluation_data.name_en)
                 )
-                self.evaluations[evaluation_id].degrees |= evaluation_data.degrees
-                self.evaluations[evaluation_id].errors = merge_dictionaries_of_sets(
-                    self.evaluations[evaluation_id].errors, evaluation_data.errors
-                )
-            elif evaluation_data != self.evaluations[evaluation_id]:
+
+                known_data.degrees |= evaluation_data.degrees
+
+                assert evaluation_data.errors.keys() <= {"degrees", "course_type", "is_graded"}
+                assert known_data.errors.get("course_type") == evaluation_data.errors.get("course_type")
+                assert known_data.errors.get("is_graded") == evaluation_data.errors.get("is_graded")
+
+                degree_errors = known_data.errors.get("degrees", set()) | evaluation_data.errors.get("degrees", set())
+                if len(degree_errors) > 0:
+                    known_data.errors["degrees"] = degree_errors
+            elif evaluation_data != known_data:
                 self.errors[ImporterError.COURSE].append(
                     _('Sheet "{}", row {}: The course\'s "{}" data differs from it\'s data in a previous row.').format(
                         sheetname, row + 1, evaluation_data.name_en
@@ -439,6 +446,9 @@ def check_evaluation_data_correctness(self, semester):
                 self.errors[ImporterError.COURSE].append(
                     _("Course {} does already exist in this semester.").format(evaluation_data.name_de)
                 )
+
+            assert evaluation_data.errors.keys() <= {"degrees", "course_type", "is_graded"}
+
             if "degrees" in evaluation_data.errors:
                 missing_degree_names |= evaluation_data.errors["degrees"]
             if "course_type" in evaluation_data.errors:
diff --git a/evap/staff/tools.py b/evap/staff/tools.py
--- a/evap/staff/tools.py
+++ b/evap/staff/tools.py
@@ -1,7 +1,6 @@
 import os
 from datetime import date, datetime, timedelta
 from enum import Enum
-from typing import Any, Dict, Set
 
 from django.conf import settings
 from django.contrib import messages
@@ -369,11 +368,3 @@ def remove_user_from_represented_and_ccing_users(user, ignored_users=None, test_
             cc_user.cc_users.remove(user)
             remove_messages.append(_("Removed {} from the CC users of {}.").format(user.full_name, cc_user.full_name))
     return remove_messages
-
-
-def merge_dictionaries_of_sets(a: Dict[Any, Set], b: Dict[Any, Set]) -> Dict[Any, Set]:
-    return {
-        **a,
-        **b,
-        **({key: (a[key] | b[key]) for key in a if key in b}),
-    }


Test patch:
diff --git a/evap/staff/fixtures/excel_files_test_data.py b/evap/staff/fixtures/excel_files_test_data.py
--- a/evap/staff/fixtures/excel_files_test_data.py
+++ b/evap/staff/fixtures/excel_files_test_data.py
@@ -122,6 +122,14 @@
     ]
 }
 
+test_enrollment_data_error_merge_filedata = {
+    'MA Belegungen': [
+        ['Degree', 'Student last name', 'Student first name', 'Student email address', 'Course kind', 'Course is graded', 'Course name (de)', 'Course name (en)', 'Responsible title', 'Responsible last name', 'Responsible first name', 'Responsible email address'],
+        ['Grandmaster', 'Quid', 'Bastius', 'bastius.quid@external.example.com', 'jaminar', 'probably not', 'Bauen', 'Build', '', 'Sed', 'Diam', '345@external.example.com'],
+        ['Beginner,Bachelor', 'Lorem', 'Ipsum', 'ipsum.lorem@institution.example.com', 'jaminar', 'probably not', 'Bauen', 'Build', '', 'Sed', 'Diam', '345@external.example.com'],
+    ],
+}
+
 test_enrollment_data_import_names_filedata = {
     'MA Belegungen': [
         ['Degree', 'Student last name', 'Student first name', 'Student email address', 'Course kind', 'Course is graded', 'Course name (de)', 'Course name (en)', 'Responsible title', 'Responsible last name', 'Responsible first name', 'Responsible email address'],
diff --git a/evap/staff/tests/test_importers.py b/evap/staff/tests/test_importers.py
--- a/evap/staff/tests/test_importers.py
+++ b/evap/staff/tests/test_importers.py
@@ -243,6 +243,17 @@ def test_degrees_are_merged(self):
         course = Course.objects.get(name_de="Bauen")
         self.assertSetEqual(set(course.degrees.all()), set(Degree.objects.filter(name_de__in=["Master", "Bachelor"])))
 
+    def test_errors_are_merged(self):
+        excel_content = excel_data.create_memory_excel_file(excel_data.test_enrollment_data_error_merge_filedata)
+        __, warnings, errors = EnrollmentImporter.process(
+            excel_content, self.semester, self.vote_start_datetime, self.vote_end_date, test_run=False
+        )
+        self.assertIn("Both degrees have been set for the course", "".join(warnings[ImporterWarning.DEGREE]))
+        self.assertIn("is probably not, but must be", "".join(errors[ImporterError.IS_GRADED]))
+        self.assertIn("jaminar", "".join(errors[ImporterError.COURSE_TYPE_MISSING]))
+        self.assertIn("Beginner", "".join(errors[ImporterError.DEGREE_MISSING]))
+        self.assertIn("Grandmaster", "".join(errors[ImporterError.DEGREE_MISSING]))
+
     def test_course_type_and_degrees_are_retrieved_with_import_names(self):
         excel_content = excel_data.create_memory_excel_file(excel_data.test_enrollment_data_import_names_filedata)
 
diff --git a/evap/staff/tests/test_tools.py b/evap/staff/tests/test_tools.py
--- a/evap/staff/tests/test_tools.py
+++ b/evap/staff/tests/test_tools.py
@@ -7,12 +7,7 @@
 from evap.evaluation.models import Contribution, Course, Evaluation, UserProfile
 from evap.evaluation.tests.tools import WebTest
 from evap.rewards.models import RewardPointGranting, RewardPointRedemption
-from evap.staff.tools import (
-    delete_navbar_cache_for_users,
-    merge_dictionaries_of_sets,
-    merge_users,
-    remove_user_from_represented_and_ccing_users,
-)
+from evap.staff.tools import delete_navbar_cache_for_users, merge_users, remove_user_from_represented_and_ccing_users
 
 
 class NavbarCacheTest(WebTest):
@@ -262,9 +257,3 @@ def test_do_nothing_if_test_run(self):
         self.assertEqual([set(user1.delegates.all()), set(user1.cc_users.all())], [{delete_user}, {delete_user}])
         self.assertEqual([set(user2.delegates.all()), set(user2.cc_users.all())], [{delete_user}, {delete_user}])
         self.assertEqual(len(messages), 4)
-
-
-class MiscellaneousToolsTest(TestCase):
-    def test_merge_dictionaries_of_sets(self):
-        self.assertEqual(merge_dictionaries_of_sets({"a": set([1])}, {"b": set([2])}), {"a": set([1]), "b": set([2])})
-        self.assertEqual(merge_dictionaries_of_sets({"a": set([1])}, {"a": set([2])}), {"a": set([1, 2])})