Repository: e-valuation/EvaP

Issue:
Auto merge degrees on import
Currently the enrollment importer shows an error message if the same course has a different degree than in a previous line of the enrollment file.
Instead, it should display a warning message if a course's data differs only in its degree from the previous lines for this course and add all degrees to the imported course data.


Discussion:

Resolving patch:
diff --git a/evap/staff/importers.py b/evap/staff/importers.py
--- a/evap/staff/importers.py
+++ b/evap/staff/importers.py
@@ -218,9 +218,9 @@ def _create_user_data_mismatch_warning(user, user_data, test_run):
             msg = _("The existing user would be overwritten with the following data:")
         else:
             msg = _("The existing user was overwritten with the following data:")
-        return (mark_safe(msg +
-            "<br> - " + ExcelImporter._create_user_string(user) + _(" (existing)") +
-            "<br> - " + ExcelImporter._create_user_string(user_data) + _(" (new)")))
+        return (mark_safe(msg
+            + "<br> - " + ExcelImporter._create_user_string(user) + _(" (existing)")
+            + "<br> - " + ExcelImporter._create_user_string(user_data) + _(" (new)")))
 
     @staticmethod
     def _create_user_inactive_warning(user, test_run):
@@ -261,7 +261,9 @@ def check_user_data_sanity(self, test_run):
 
 
 class EnrollmentImporter(ExcelImporter):
-    W_MANY = 'too_many_enrollments'  # extension of ExcelImporter.warnings keys
+    # extension of ExcelImporter.warnings keys
+    W_DEGREE = 'degree'
+    W_MANY = 'too_many_enrollments'
 
     def __init__(self):
         super().__init__()
@@ -285,7 +287,18 @@ def process_evaluation(self, evaluation_data, sheet, row):
                 self.evaluations[evaluation_id] = evaluation_data
                 self.names_de.add(evaluation_data.name_de)
         else:
-            if not evaluation_data == self.evaluations[evaluation_id]:
+            if (set(evaluation_data.degree_names) != set(self.evaluations[evaluation_id].degree_names)
+                    and evaluation_data.name_de == self.evaluations[evaluation_id].name_de
+                    and evaluation_data.name_en == self.evaluations[evaluation_id].name_en
+                    and evaluation_data.type_name == self.evaluations[evaluation_id].type_name
+                    and evaluation_data.is_graded == self.evaluations[evaluation_id].is_graded
+                    and evaluation_data.responsible_email == self.evaluations[evaluation_id].responsible_email):
+                self.warnings[self.W_DEGREE].append(
+                    _('Sheet "{}", row {}: The course\'s "{}" degree differs from it\'s degree in a previous row. Both degrees have been added to the course.')
+                    .format(sheet, row + 1, evaluation_data.name_en)
+                )
+                self.evaluations[evaluation_id].degree_names.extend(evaluation_data.degree_names)
+            elif evaluation_data != self.evaluations[evaluation_id]:
                 self.errors.append(_('Sheet "{}", row {}: The course\'s "{}" data differs from it\'s data in a previous row.').format(sheet, row + 1, evaluation_data.name_en))
 
     def consolidate_enrollment_data(self):
@@ -402,13 +415,14 @@ def process(cls, excel_content, semester, vote_start_datetime, vote_end_date, te
             else:
                 importer.write_enrollments_to_db(semester, vote_start_datetime, vote_end_date)
 
-            return importer.success_messages, importer.warnings, importer.errors
         except Exception as e:
             importer.errors.append(_("Import finally aborted after exception: '%s'" % e))
             if settings.DEBUG:
                 # re-raise error for further introspection if in debug mode
                 raise
 
+        return importer.success_messages, importer.warnings, importer.errors
+
 
 class UserImporter(ExcelImporter):
     def read_one_user(self, data):
@@ -593,5 +607,6 @@ def make_users_active(user_list):
     ExcelImporter.W_EMAIL: ugettext_lazy("Email mismatches"),
     ExcelImporter.W_DUPL: ugettext_lazy("Possible duplicates"),
     ExcelImporter.W_GENERAL: ugettext_lazy("General warnings"),
+    EnrollmentImporter.W_DEGREE: ugettext_lazy("Degree mismatches"),
     EnrollmentImporter.W_MANY: ugettext_lazy("Unusually high number of enrollments")
 }


Test patch:
diff --git a/evap/staff/fixtures/test_enrollment_data_degree_merge.xls b/evap/staff/fixtures/test_enrollment_data_degree_merge.xls
new file mode 100644
Binary files /dev/null and b/evap/staff/fixtures/test_enrollment_data_degree_merge.xls differ
diff --git a/evap/staff/tests/test_importers.py b/evap/staff/tests/test_importers.py
--- a/evap/staff/tests/test_importers.py
+++ b/evap/staff/tests/test_importers.py
@@ -1,10 +1,11 @@
 import os
+from collections import defaultdict
 from datetime import date, datetime
 from django.test import TestCase, override_settings
 from django.conf import settings
 from model_mommy import mommy
 
-from evap.evaluation.models import UserProfile, Semester, Evaluation, Contribution, CourseType
+from evap.evaluation.models import Course, Degree, UserProfile, Semester, Evaluation, Contribution, CourseType
 from evap.staff.importers import UserImporter, EnrollmentImporter, ExcelImporter, PersonImporter
 
 
@@ -125,43 +126,73 @@ def test_import_makes_inactive_user_active(self):
 
 class TestEnrollmentImporter(TestCase):
     filename_valid = os.path.join(settings.BASE_DIR, "staff/fixtures/test_enrollment_data.xls")
+    filename_valid_degree_merge = os.path.join(settings.BASE_DIR, "staff/fixtures/test_enrollment_data_degree_merge.xls")
     filename_invalid = os.path.join(settings.BASE_DIR, "staff/fixtures/invalid_enrollment_data.xls")
     filename_random = os.path.join(settings.BASE_DIR, "staff/fixtures/random.random")
 
-    def test_valid_file_import(self):
-        semester = mommy.make(Semester)
-        vote_start_datetime = datetime(2017, 1, 10)
-        vote_end_date = date(2017, 3, 10)
+    @classmethod
+    def setUpTestData(cls):
+        cls.semester = mommy.make(Semester)
+        cls.vote_start_datetime = datetime(2017, 1, 10)
+        cls.vote_end_date = date(2017, 3, 10)
         mommy.make(CourseType, name_de="Seminar")
         mommy.make(CourseType, name_de="Vorlesung")
 
+    def test_valid_file_import(self):
         with open(self.filename_valid, "rb") as excel_file:
             excel_content = excel_file.read()
 
-        success_messages, warnings, errors = EnrollmentImporter.process(excel_content, semester, None, None, test_run=True)
+        success_messages, warnings, errors = EnrollmentImporter.process(excel_content, self.semester, None, None, test_run=True)
         self.assertIn("The import run will create 23 courses/evaluations and 23 users:", "".join(success_messages))
         # check for one random user instead of for all 23
         self.assertIn("Ferdi Itaque (ferdi.itaque)", "".join(success_messages))
         self.assertEqual(errors, [])
         self.assertEqual(warnings, {})
 
-        success_messages, warnings, errors = EnrollmentImporter.process(excel_content, semester, vote_start_datetime, vote_end_date, test_run=False)
+        old_user_count = UserProfile.objects.all().count()
+
+        success_messages, warnings, errors = EnrollmentImporter.process(excel_content, self.semester, self.vote_start_datetime, self.vote_end_date, test_run=False)
         self.assertIn("Successfully created 23 courses/evaluations, 6 students and 17 contributors:", "".join(success_messages))
         self.assertIn("Ferdi Itaque (ferdi.itaque)", "".join(success_messages))
         self.assertEqual(errors, [])
         self.assertEqual(warnings, {})
 
+        self.assertEqual(Evaluation.objects.all().count(), 23)
+        expected_user_count = old_user_count + 23
+        self.assertEqual(UserProfile.objects.all().count(), expected_user_count)
+
+    def test_degrees_are_merged(self):
+        with open(self.filename_valid_degree_merge, "rb") as excel_file:
+            excel_content = excel_file.read()
+
+        expected_warnings = defaultdict(list)
+        expected_warnings[EnrollmentImporter.W_DEGREE].append(
+            'Sheet "MA Belegungen", row 3: The course\'s "Build" degree differs from it\'s degree in a previous row. Both degrees have been added to the course.'
+        )
+
+        success_messages, warnings, errors = EnrollmentImporter.process(excel_content, self.semester, None, None, test_run=True)
+        self.assertIn("The import run will create 1 courses/evaluations and 3 users", "".join(success_messages))
+        self.assertEqual(errors, [])
+        self.assertEqual(warnings, expected_warnings)
+
+        success_messages, warnings, errors = EnrollmentImporter.process(excel_content, self.semester, self.vote_start_datetime, self.vote_end_date, test_run=False)
+        self.assertIn("Successfully created 1 courses/evaluations, 2 students and 1 contributors", "".join(success_messages))
+        self.assertEqual(errors, [])
+        self.assertEqual(warnings, expected_warnings)
+
+        self.assertEqual(Course.objects.all().count(), 1)
+        self.assertEqual(Evaluation.objects.all().count(), 1)
+
+        course = Course.objects.get(name_de="Bauen")
+        self.assertSetEqual(set(course.degrees.all()), set(Degree.objects.filter(name_de__in=["Master", "Bachelor"])))
+
     @override_settings(IMPORTER_MAX_ENROLLMENTS=1)
     def test_enrollment_importer_high_enrollment_warning(self):
-        semester = mommy.make(Semester)
-        vote_start_datetime = datetime(2017, 1, 10)
-        vote_end_date = datetime(2017, 3, 10)
-
         with open(self.filename_valid, "rb") as excel_file:
             excel_content = excel_file.read()
 
-        __, warnings_test, __ = EnrollmentImporter.process(excel_content, semester, None, None, test_run=True)
-        __, warnings_no_test, __ = EnrollmentImporter.process(excel_content, semester, vote_start_datetime, vote_end_date, test_run=False)
+        __, warnings_test, __ = EnrollmentImporter.process(excel_content, self.semester, None, None, test_run=True)
+        __, warnings_no_test, __ = EnrollmentImporter.process(excel_content, self.semester, self.vote_start_datetime, self.vote_end_date, test_run=False)
 
         self.assertEqual(warnings_test, warnings_no_test)
         warnings_many = warnings_test[EnrollmentImporter.W_MANY]
@@ -173,14 +204,13 @@ def test_enrollment_importer_high_enrollment_warning(self):
         self.assertIn("Warning: User bastius.quid@external.example.com has 3 enrollments, which is a lot.", warnings_many)
 
     def test_random_file_error(self):
-        semester = mommy.make(Semester)
-        original_user_count = UserProfile.objects.count()
-
         with open(self.filename_random, "rb") as excel_file:
             excel_content = excel_file.read()
 
-        __, __, errors_test = EnrollmentImporter.process(excel_content, semester, None, None, test_run=True)
-        __, __, errors_no_test = EnrollmentImporter.process(excel_content, semester, None, None, test_run=False)
+        original_user_count = UserProfile.objects.count()
+
+        __, __, errors_test = EnrollmentImporter.process(excel_content, self.semester, None, None, test_run=True)
+        __, __, errors_no_test = EnrollmentImporter.process(excel_content, self.semester, None, None, test_run=False)
 
         self.assertEqual(errors_test, errors_no_test)
         self.assertIn("Couldn't read the file. Error: Unsupported format, or corrupt file:"
@@ -188,16 +218,13 @@ def test_random_file_error(self):
         self.assertEqual(UserProfile.objects.count(), original_user_count)
 
     def test_invalid_file_error(self):
-        semester = mommy.make(Semester)
-        mommy.make(CourseType, name_de="Seminar")
-        mommy.make(CourseType, name_de="Vorlesung")
-        original_user_count = UserProfile.objects.count()
-
         with open(self.filename_invalid, "rb") as excel_file:
             excel_content = excel_file.read()
 
-        __, __, errors_test = EnrollmentImporter.process(excel_content, semester, None, None, test_run=True)
-        __, __, errors_no_test = EnrollmentImporter.process(excel_content, semester, None, None, test_run=False)
+        original_user_count = UserProfile.objects.count()
+
+        __, __, errors_test = EnrollmentImporter.process(excel_content, self.semester, None, None, test_run=True)
+        __, __, errors_no_test = EnrollmentImporter.process(excel_content, self.semester, None, None, test_run=False)
 
         self.assertEqual(errors_test, errors_no_test)
         self.assertIn('Sheet "MA Belegungen", row 3: The users\'s data (email: bastius.quid@external.example.com) differs from it\'s data in a previous row.', errors_test)