Repository: e-valuation/EvaP

Issue:
Missing degree check in enrollment import
When importing enrollment data, the importer must check whether the degrees used in the uploaded file already exist (and show an error message if a degree doesn't yet exist and must be manually created).

However, the error message for non-existent degrees currently only works correctly if this degree is the one in the first occurrence of an evaluation. If the first occurrence has a valid degree and any of the following degrees does not exist, no error will be shown and the import will fail silently.

Instead, an error message should be shown if any entry in the imported file uses a non-existent degree.


Discussion:

Resolving patch:
diff --git a/evap/staff/importers.py b/evap/staff/importers.py
--- a/evap/staff/importers.py
+++ b/evap/staff/importers.py
@@ -12,7 +12,7 @@
 
 from evap.evaluation.models import Contribution, Course, CourseType, Degree, Evaluation, UserProfile
 from evap.evaluation.tools import clean_email
-from evap.staff.tools import create_user_list_html_string_for_message, ImportType
+from evap.staff.tools import create_user_list_html_string_for_message, ImportType, merge_dictionaries_of_sets
 
 
 def sorted_messages(messages):
@@ -405,6 +405,9 @@ def process_evaluation(self, evaluation_data, sheet, row):
                     ).format(sheet, row + 1, evaluation_data.name_en)
                 )
                 self.evaluations[evaluation_id].degrees |= evaluation_data.degrees
+                self.evaluations[evaluation_id].errors = merge_dictionaries_of_sets(
+                    self.evaluations[evaluation_id].errors, evaluation_data.errors
+                )
             elif evaluation_data != self.evaluations[evaluation_id]:
                 self.errors[ImporterError.COURSE].append(
                     _('Sheet "{}", row {}: The course\'s "{}" data differs from it\'s data in a previous row.').format(
@@ -420,8 +423,8 @@ def consolidate_enrollment_data(self):
             self.enrollments.append((evaluation_data, student_data))
 
     def check_evaluation_data_correctness(self, semester):
-        degree_names = set()
-        course_type_names = set()
+        missing_degree_names = set()
+        missing_course_type_names = set()
         for evaluation_data in self.evaluations.values():
             if Course.objects.filter(semester=semester, name_en=evaluation_data.name_en).exists():
                 self.errors[ImporterError.COURSE].append(
@@ -432,9 +435,9 @@ def check_evaluation_data_correctness(self, semester):
                     _("Course {} does already exist in this semester.").format(evaluation_data.name_de)
                 )
             if "degrees" in evaluation_data.errors:
-                degree_names |= evaluation_data.errors["degrees"]
+                missing_degree_names |= evaluation_data.errors["degrees"]
             if "course_type" in evaluation_data.errors:
-                course_type_names.add(evaluation_data.errors["course_type"])
+                missing_course_type_names.add(evaluation_data.errors["course_type"])
             if "is_graded" in evaluation_data.errors:
                 self.errors[ImporterError.IS_GRADED].append(
                     _('"is_graded" of course {} is {}, but must be {} or {}').format(
@@ -445,13 +448,13 @@ def check_evaluation_data_correctness(self, semester):
                     )
                 )
 
-        for degree_name in degree_names:
+        for degree_name in missing_degree_names:
             self.errors[ImporterError.DEGREE_MISSING].append(
                 _('Error: No degree is associated with the import name "{}". Please manually create it first.').format(
                     degree_name
                 )
             )
-        for course_type_name in course_type_names:
+        for course_type_name in missing_course_type_names:
             self.errors[ImporterError.COURSE_TYPE_MISSING].append(
                 _(
                     'Error: No course type is associated with the import name "{}". Please manually create it first.'
diff --git a/evap/staff/tools.py b/evap/staff/tools.py
--- a/evap/staff/tools.py
+++ b/evap/staff/tools.py
@@ -1,5 +1,6 @@
 import os
 from datetime import date, datetime, timedelta
+from typing import Dict, Set, Any
 from enum import Enum
 
 from django.contrib import messages
@@ -368,3 +369,11 @@ def remove_user_from_represented_and_ccing_users(user, ignored_users=None, test_
             cc_user.cc_users.remove(user)
             remove_messages.append(_("Removed {} from the CC users of {}.").format(user.full_name, cc_user.full_name))
     return remove_messages
+
+
+def merge_dictionaries_of_sets(a: Dict[Any, Set], b: Dict[Any, Set]) -> Dict[Any, Set]:
+    return {
+        **a,
+        **b,
+        **({key: (a[key] | b[key]) for key in a if key in b}),
+    }


Test patch:
diff --git a/evap/staff/fixtures/excel_files_test_data.py b/evap/staff/fixtures/excel_files_test_data.py
--- a/evap/staff/fixtures/excel_files_test_data.py
+++ b/evap/staff/fixtures/excel_files_test_data.py
@@ -132,6 +132,14 @@
     ]
 }
 
+test_unknown_degree_error_filedata = {
+    'Sheet 1': [
+        ['Degree', 'Student last name', 'Student first name', 'Student email address', 'Course kind', 'Course is graded', 'Course name (de)', 'Course name (en)', 'Responsible title', 'Responsible last name', 'Responsible first name', 'Responsible email address'],
+        ['bachelor', 'Doe', 'John', 'john.doe@institution.example.com', 'Vorlesung', 'yes', 'Neovim kompilieren', 'compiling Neovim', 'Prof. Dr.', 'Prorsus', 'Christoph', '123@institution.example.com'],
+        ['beginner', 'Roe', 'Jane', 'jane.roe@institution.example.com', 'Vorlesung', 'yes', 'Neovim kompilieren', 'compiling Neovim', 'Prof. Dr.', 'Prorsus', 'Christoph', '123@institution.example.com'],
+    ],
+}
+
 valid_user_courses_import_filedata = {
     'MA Belegungen': [
         ['Degree', 'Student last name', 'Student first name', 'Student email address', 'Course kind', 'Course is graded', 'Course name (de)', 'Course name (en)', 'Responsible title', 'Responsible last name', 'Responsible first name', 'Responsible email address'],
diff --git a/evap/staff/tests/test_importers.py b/evap/staff/tests/test_importers.py
--- a/evap/staff/tests/test_importers.py
+++ b/evap/staff/tests/test_importers.py
@@ -359,6 +359,13 @@ def test_duplicate_course_error(self):
             },
         )
 
+    def test_unknown_degree_error(self):
+        excel_content = excel_data.create_memory_excel_file(excel_data.test_unknown_degree_error_filedata)
+        __, __, errors = EnrollmentImporter.process(excel_content, baker.make(Semester), None, None, test_run=False)
+        missing_degree_errors = errors[ImporterError.DEGREE_MISSING]
+        self.assertEqual(len(missing_degree_errors), 1)
+        self.assertIn("manually create it first", missing_degree_errors[0])
+
     def test_replace_consecutive_and_trailing_spaces(self):
         excel_content = excel_data.create_memory_excel_file(
             excel_data.test_enrollment_data_consecutive_and_trailing_spaces_filedata
diff --git a/evap/staff/tests/test_tools.py b/evap/staff/tests/test_tools.py
--- a/evap/staff/tests/test_tools.py
+++ b/evap/staff/tests/test_tools.py
@@ -8,7 +8,12 @@
 from evap.evaluation.tests.tools import WebTest
 from evap.evaluation.models import Contribution, Course, Evaluation, UserProfile
 from evap.rewards.models import RewardPointGranting, RewardPointRedemption
-from evap.staff.tools import merge_users, delete_navbar_cache_for_users, remove_user_from_represented_and_ccing_users
+from evap.staff.tools import (
+    merge_users,
+    delete_navbar_cache_for_users,
+    remove_user_from_represented_and_ccing_users,
+    merge_dictionaries_of_sets,
+)
 
 
 class NavbarCacheTest(WebTest):
@@ -258,3 +263,9 @@ def test_do_nothing_if_test_run(self):
         self.assertEqual([set(user1.delegates.all()), set(user1.cc_users.all())], [{delete_user}, {delete_user}])
         self.assertEqual([set(user2.delegates.all()), set(user2.cc_users.all())], [{delete_user}, {delete_user}])
         self.assertEqual(len(messages), 4)
+
+
+class MiscellaneousToolsTest(TestCase):
+    def test_merge_dictionaries_of_sets(self):
+        self.assertEqual(merge_dictionaries_of_sets({"a": set([1])}, {"b": set([2])}), {"a": set([1]), "b": set([2])})
+        self.assertEqual(merge_dictionaries_of_sets({"a": set([1])}, {"a": set([2])}), {"a": set([1, 2])})