Repository: e-valuation/EvaP

Issue:
Enrollment Import: Trimming and lowercasing
We should trim all fields of the enrollment data upon importing, and lowercase usernames and email adresses.

Maybe we should also do this when editing users/lectures in the UI.

if this is going live, we need to lowercase and trim all respective fields in the database of evaluierung and evap-dev.

if we decide to use database constraints, we need some to enforce this as well.



Discussion:
c67a78e8 can probably be reverted if this gets fixed

we decided to not use constraints.


Resolving patch:
diff --git a/evap/contributor/forms.py b/evap/contributor/forms.py
--- a/evap/contributor/forms.py
+++ b/evap/contributor/forms.py
@@ -57,7 +57,7 @@ def __init__(self, *args, **kwargs):
 
         # fix generated form
         self.fields['delegates'].required = False
-        self.fields['delegates'].queryset = User.objects.extra(select={'lower_username': 'lower(username)'}).order_by('lower_username')
+        self.fields['delegates'].queryset = User.objects.order_by('username')
         self.fields['delegates'].help_text = ""
 
         # load user fields
diff --git a/evap/fsr/forms.py b/evap/fsr/forms.py
--- a/evap/fsr/forms.py
+++ b/evap/fsr/forms.py
@@ -95,7 +95,7 @@ def __init__(self, *args, **kwargs):
         super(ContributionForm, self).__init__(*args, **kwargs)
         self.fields['contributor'].widget.attrs['class'] = 'form-control'
 
-        self.fields['contributor'].queryset = User.objects.extra(select={'lower_username': 'lower(username)'}).order_by('lower_username')
+        self.fields['contributor'].queryset = User.objects.order_by('username')
         self.fields['questionnaires'] = QuestionnaireMultipleChoiceField(Questionnaire.objects.filter(is_for_contributors=True, obsolete=False), label=_("Questionnaires"))
 
     def validate_unique(self):
@@ -344,10 +344,10 @@ def __init__(self, *args, **kwargs):
 
         # fix generated form
         self.fields['delegates'].required = False
-        self.fields['delegates'].queryset = User.objects.extra(select={'lower_username': 'lower(username)'}).order_by('lower_username')
+        self.fields['delegates'].queryset = User.objects.order_by('username')
         self.fields['delegates'].help_text = ""
         self.fields['cc_users'].required = False
-        self.fields['cc_users'].queryset = User.objects.extra(select={'lower_username': 'lower(username)'}).order_by('lower_username')
+        self.fields['cc_users'].queryset = User.objects.order_by('username')
         self.fields['cc_users'].help_text = ""
         self.fields['is_staff'].label = _(u"Student representative")
         self.fields['is_superuser'].label = _(u"EvaP Administrator")
@@ -380,10 +380,10 @@ def clean_username(self):
 
     def _post_clean(self, *args, **kw):
         # first save the user, so that the profile gets created for sure
-        self.instance.user.username = self.cleaned_data.get('username')
-        self.instance.user.first_name = self.cleaned_data.get('first_name')
-        self.instance.user.last_name = self.cleaned_data.get('last_name')
-        self.instance.user.email = self.cleaned_data.get('email')
+        self.instance.user.username = self.cleaned_data.get('username').strip().lower()
+        self.instance.user.first_name = self.cleaned_data.get('first_name').strip()
+        self.instance.user.last_name = self.cleaned_data.get('last_name').strip()
+        self.instance.user.email = self.cleaned_data.get('email').strip().lower()
         self.instance.user.is_staff = self.cleaned_data.get('is_staff')
         self.instance.user.is_superuser = self.cleaned_data.get('is_superuser')
         self.instance.user.save()
diff --git a/evap/fsr/importers.py b/evap/fsr/importers.py
--- a/evap/fsr/importers.py
+++ b/evap/fsr/importers.py
@@ -13,12 +13,12 @@
 class UserData(object):
     """Holds information about a user, retrieved from the Excel file."""
 
-    def __init__(self, username=None, first_name=None, last_name=None, title=None, email=None):
-        self.username = username
-        self.first_name = first_name
-        self.last_name = last_name
-        self.title = title
-        self.email = email
+    def __init__(self, username='', first_name='', last_name='', title='', email=''):
+        self.username = username.strip().lower()
+        self.first_name = first_name.strip()
+        self.last_name = last_name.strip()
+        self.title = title.strip()
+        self.email = email.strip().lower()
 
     def store_in_database(self):
         user = User(username=self.username,
@@ -52,11 +52,11 @@ def update(self, user):
 class CourseData(object):
     """Holds information about a course, retrieved from the Excel file."""
 
-    def __init__(self, name_de=None, name_en=None, kind=None, degree=None):
-        self.name_de = name_de
-        self.name_en = name_en
-        self.kind = kind
-        self.degree = degree
+    def __init__(self, name_de='', name_en='', kind='', degree=''):
+        self.name_de = name_de.strip()
+        self.name_en = name_en.strip()
+        self.kind = kind.strip()
+        self.degree = degree.strip()
 
     def store_in_database(self, vote_start_date, vote_end_date, semester):
         course = Course(name_de=self.name_de,
@@ -120,11 +120,6 @@ def validate_and_fix_enrollments(self):
         already in the database."""
 
         for (sheet, row), (student_data, contributor_data, course_data) in self.associations.items():
-            # try to infer first names from usernames
-            if not contributor_data.first_name:
-                first, sep, last = contributor_data.username.partition(".")
-                if sep == ".":
-                    contributor_data.first_name = first
             if student_data.email == None or student_data.email == "":
                 student_data.email = student_data.username + "@student.hpi.uni-potsdam.de"
 


Test patch:
diff --git a/evap/fsr/tests.py b/evap/fsr/tests.py
--- a/evap/fsr/tests.py
+++ b/evap/fsr/tests.py
@@ -68,12 +68,12 @@ def test_import(self):
                 responsible_count += 1
         self.assertEqual(responsible_count, 1, "Wrong number of responsible contributors after Excel import.")
 
-        check_student = User.objects.get(username="Diam.Synephebos")
+        check_student = User.objects.get(username="diam.synephebos")
         self.assertEqual(check_student.first_name, "Diam")
-        self.assertEqual(check_student.email, "Diam.Synephebos@student.hpi.uni-potsdam.de")
+        self.assertEqual(check_student.email, "diam.synephebos@student.hpi.uni-potsdam.de")
 
-        check_contributor = User.objects.get(username="Sanctus.Aliquyam")
-        self.assertEqual(check_contributor.first_name, "Sanctus")
+        check_contributor = User.objects.get(username="sanctus.aliquyam")
+        self.assertEqual(check_contributor.first_name, "")
         self.assertEqual(check_contributor.last_name, "Aliquyam")
         self.assertEqual(check_contributor.email, "567@web.de")