Repository: e-valuation/EvaP

Issue:
Clean duplicates on user import
When importing a list of users that includes the same person multiple times, these duplicates should be ignored during the import (a message for each ignored user could be shown). Currently the importer throws an exception.


Discussion:

Resolving patch:
diff --git a/evap/staff/importers.py b/evap/staff/importers.py
--- a/evap/staff/importers.py
+++ b/evap/staff/importers.py
@@ -26,6 +26,9 @@ def __eq__(self, other):
         return (isinstance(other, self.__class__)
             and self.__dict__ == other.__dict__)
 
+    def __hash__(self):
+        return hash(tuple(sorted(self.__dict__.items())))
+
 
 class UserData(CommonEqualityMixin):
     """
@@ -122,6 +125,7 @@ def store_in_database(self, vote_start_datetime, vote_end_date, semester):
 class ExcelImporter():
     W_NAME = 'name'
     W_DUPL = 'duplicate'
+    W_IGNORED = 'ignored'
     W_GENERAL = 'general'
     W_INACTIVE = 'inactive'
 
@@ -151,14 +155,11 @@ def check_column_count(self, expected_column_count):
             if sheet.ncols != expected_column_count:
                 self.errors.append(_("Wrong number of columns in sheet '{}'. Expected: {}, actual: {}").format(sheet.name, expected_column_count, sheet.ncols))
 
-    def for_each_row_in_excel_file_do(self, parse_row_function):
+    def for_each_row_in_excel_file_do(self, row_function):
         for sheet in self.book.sheets():
             try:
                 for row in range(self.skip_first_n_rows, sheet.nrows):
-                    line_data = parse_row_function(sheet.row_values(row))
-                    # store data objects together with the data source location for problem tracking
-                    self.associations[(sheet.name, row)] = line_data
-
+                    row_function(sheet.row_values(row), sheet, row)
                 self.success_messages.append(_("Successfully read sheet '%s'.") % sheet.name)
             except Exception:
                 self.warnings[self.W_GENERAL].append(_("A problem occured while reading sheet {}.").format(sheet.name))
@@ -249,13 +250,11 @@ def __init__(self):
         self.enrollments = []
         self.names_de = set()
 
-    @staticmethod
-    def read_one_enrollment(data):
+    def read_one_enrollment(self, data, sheet, row):
         student_data = UserData(first_name=data[2], last_name=data[1], email=data[3], title='', is_responsible=False)
         responsible_data = UserData(first_name=data[10], last_name=data[9], title=data[8], email=data[11], is_responsible=True)
-        evaluation_data = EvaluationData(name_de=data[6], name_en=data[7], type_name=data[4], is_graded=data[5], degree_names=data[0],
-                responsible_email=responsible_data.email)
-        return (student_data, responsible_data, evaluation_data)
+        evaluation_data = EvaluationData(name_de=data[6], name_en=data[7], type_name=data[4], is_graded=data[5], degree_names=data[0], responsible_email=responsible_data.email)
+        self.associations[(sheet.name, row)] = (student_data, responsible_data, evaluation_data)
 
     def process_evaluation(self, evaluation_data, sheet, row):
         evaluation_id = evaluation_data.name_en
@@ -397,10 +396,25 @@ def process(cls, excel_content, semester, vote_start_datetime, vote_end_date, te
 
 
 class UserImporter(ExcelImporter):
-    @staticmethod
-    def read_one_user(data):
+
+    def __init__(self):
+        super().__init__()
+        self._read_user_data = dict()
+
+    def read_one_user(self, data, sheet, row):
         user_data = UserData(title=data[0], first_name=data[1], last_name=data[2], email=data[3], is_responsible=False)
-        return user_data
+        self.associations[(sheet.name, row)] = user_data
+        if user_data not in self._read_user_data:
+            self._read_user_data[user_data] = (sheet.name, row)
+        else:
+            orig_sheet, orig_row = self._read_user_data[user_data]
+            warningstring = _("The duplicated row {row} in sheet '{sheet}' was ignored. It was first found in sheet '{orig_sheet}' on row {orig_row}.").format(
+                    sheet=sheet.name,
+                    row=row + 1,
+                    orig_sheet=orig_sheet,
+                    orig_row=orig_row + 1,
+            )
+            self.warnings[self.W_IGNORED].append(warningstring)
 
     def consolidate_user_data(self):
         for (sheet, row), (user_data) in self.associations.items():
@@ -414,7 +428,7 @@ def save_users_to_db(self):
         new_participants = []
         created_users = []
         with transaction.atomic():
-            for (sheet, row), (user_data) in self.associations.items():
+            for user_data in self.users.values():
                 try:
                     user, created = user_data.store_in_database()
                     new_participants.append(user)
@@ -423,8 +437,7 @@ def save_users_to_db(self):
 
                 except Exception as e:
                     self.errors.append(_("A problem occured while writing the entries to the database."
-                                         " The original data location was row %(row)d of sheet '%(sheet)s'."
-                                         " The error message has been: '%(error)s'") % dict(row=row + 1, sheet=sheet, error=e))
+                                         " The error message has been: '%(error)s'") % dict(error=e))
                     raise
 
         msg = _("Successfully created {} users:").format(len(created_users))
@@ -577,6 +590,7 @@ def make_users_active(user_list):
     ExcelImporter.W_NAME: ugettext_lazy("Name mismatches"),
     ExcelImporter.W_INACTIVE: ugettext_lazy("Inactive users"),
     ExcelImporter.W_DUPL: ugettext_lazy("Possible duplicates"),
+    ExcelImporter.W_IGNORED: ugettext_lazy("Ignored duplicates"),
     ExcelImporter.W_GENERAL: ugettext_lazy("General warnings"),
     EnrollmentImporter.W_DEGREE: ugettext_lazy("Degree mismatches"),
     EnrollmentImporter.W_MANY: ugettext_lazy("Unusually high number of enrollments")


Test patch:
diff --git a/evap/staff/tests/test_importers.py b/evap/staff/tests/test_importers.py
--- a/evap/staff/tests/test_importers.py
+++ b/evap/staff/tests/test_importers.py
@@ -11,6 +11,7 @@
 
 class TestUserImporter(TestCase):
     filename_valid = os.path.join(settings.BASE_DIR, "staff/fixtures/valid_user_import.xls")
+    filename_duplicate = os.path.join(settings.BASE_DIR, "staff/fixtures/duplicate_user_import.xls")
     filename_invalid = os.path.join(settings.BASE_DIR, "staff/fixtures/invalid_user_import.xls")
     filename_random = os.path.join(settings.BASE_DIR, "staff/fixtures/random.random")
 
@@ -24,6 +25,8 @@ def setUpTestData(cls):
             cls.invalid_excel_content = excel_file.read()
         with open(cls.filename_random, "rb") as excel_file:
             cls.random_excel_content = excel_file.read()
+        with open(cls.filename_duplicate, "rb") as excel_file:
+            cls.duplicate_excel_content = excel_file.read()
 
     def test_test_run_does_not_change_database(self):
         original_users = list(UserProfile.objects.all())
@@ -71,6 +74,13 @@ def test_duplicate_warning(self):
                 " -  Lucilia Manilium, lucilia.manilium@institution.example.com (new)",
                 warnings_test[ExcelImporter.W_DUPL])
 
+    def test_ignored_duplicate_warning(self):
+        __, __, warnings_test, __ = UserImporter.process(self.duplicate_excel_content, test_run=True)
+        __, __, warnings_no_test, __ = UserImporter.process(self.duplicate_excel_content, test_run=False)
+
+        self.assertEqual(warnings_test, warnings_no_test)
+        self.assertTrue(any("The duplicated row 4 in sheet 'Users' was ignored. It was first found in sheet 'Users' on row 3." in warning for warning in warnings_test[ExcelImporter.W_IGNORED]))
+
     def test_random_file_error(self):
         original_user_count = UserProfile.objects.count()