Repository: learningequality/kolibri

Issue:
include user database IDs in facility CSV imports and exports

### Observed behavior

It is desirable to be able to change usernames during CSV export/import, but this is currently not possible.

### Expected behavior

* UUID of user should be included during export
* If the UUID does not already exist, that user should be skipped with an error
* If the UUID is left blank, that user should be created
* Duplicate usernames are always errors

### User-facing consequences

Cannot perform bulk updates of usernames using CSV export/import




### Context

0.14.0



Discussion:
@indirectlylit  If we change to user uuid instead of the username as source of trust (what seems logical from a technical point of view, but less friendly for the users) why can't we allow duplicate usernames? it should not be a problem anymore. If not duplicate per facility to avoid problems with user logins, duplicate per device should not be a problem.
The import is scoped to a single facility, and we enforce unique usernames within a facility to prevent login issues as you say.

Duplicates in different facilities would not be an error.


Resolving patch:
diff --git a/kolibri/core/auth/management/commands/bulkexportusers.py b/kolibri/core/auth/management/commands/bulkexportusers.py
--- a/kolibri/core/auth/management/commands/bulkexportusers.py
+++ b/kolibri/core/auth/management/commands/bulkexportusers.py
@@ -39,6 +39,7 @@
 # TODO: decide whether these should be internationalized
 labels = OrderedDict(
     (
+        ("id", _("Database ID ({})".format("UUID"))),
         ("username", _("Username ({})".format("USERNAME"))),
         ("password", _("Password ({})".format("PASSWORD"))),
         ("full_name", _("Full name ({})".format("FULL_NAME"))),
diff --git a/kolibri/core/auth/management/commands/bulkimportusers.py b/kolibri/core/auth/management/commands/bulkimportusers.py
--- a/kolibri/core/auth/management/commands/bulkimportusers.py
+++ b/kolibri/core/auth/management/commands/bulkimportusers.py
@@ -3,6 +3,7 @@
 import ntpath
 import re
 import sys
+from uuid import UUID
 
 from django.conf import settings
 from django.contrib.auth.hashers import make_password
@@ -31,6 +32,7 @@
 
 # TODO: decide whether these should be internationalized
 fieldnames = (
+    "UUID",
     "USERNAME",
     "PASSWORD",
     "FULL_NAME",
@@ -63,6 +65,8 @@
 FILE_READ_ERROR = 8
 FILE_WRITE_ERROR = 9
 REQUIRED_PASSWORD = 10
+NON_EXISTENT_UUID = 11
+INVALID_UUID = 12
 
 MESSAGES = {
     UNEXPECTED_EXCEPTION: _("Unexpected exception [{}]: {}"),
@@ -82,6 +86,10 @@
     REQUIRED_PASSWORD: _(
         "The password field is required. To leave the password unchanged in existing users, insert an asterisk (*)"
     ),
+    NON_EXISTENT_UUID: _(
+        "Cannot update update '{}' because no user with that database ID exists in this facility"
+    ),
+    INVALID_UUID: _("Database ID is not valid"),
 }
 
 # Validators ###
@@ -139,6 +147,23 @@ def checker(v):
     return checker
 
 
+def valid_uuid(allow_null=True):
+    """
+    Return a value check function which raises a ValueError if the supplied
+    value is ot a valid uuid
+    """
+
+    def checker(v):
+        if allow_null and (v is None or v == ""):
+            return None
+        try:
+            UUID(v).version
+        except (ValueError, TypeError):
+            raise ValueError(v)
+
+    return checker
+
+
 def enumeration(*args):
     """
     Return a value check function which raises a ValueError if the value (case
@@ -301,6 +326,7 @@ def validate(self, data):
             # if there aren't any errors, let's add the user and classes
             if not error_flag:
                 self.check_classroom(row, username)
+                row["position"] = index + 1
                 self.users[username] = row
 
 
@@ -352,6 +378,7 @@ def add_arguments(self, parser):
     def csv_values_validation(self, reader, header_translation):
         per_line_errors = []
         validator = Validator(header_translation)
+        validator.add_check("UUID", valid_uuid(), MESSAGES[INVALID_UUID])
         validator.add_check(
             "FULL_NAME", value_length(125), MESSAGES[TOO_LONG].format("FULL_NAME")
         )
@@ -412,7 +439,6 @@ def csv_values_validation(self, reader, header_translation):
                     validator.coach_classrooms[
                         real_name
                     ] = validator.coach_classrooms.pop(classroom)
-
         return (
             per_line_errors,
             (validator.classrooms, validator.coach_classrooms),
@@ -463,6 +489,8 @@ def get_field_values(self, user_row):
         full_name = user_row.get(self.header_translation["FULL_NAME"], None)
 
         return {
+            "uuid": user_row.get(self.header_translation["UUID"], ""),
+            "username": user_row.get(self.header_translation["USERNAME"], ""),
             "password": password,
             "gender": gender,
             "birth_year": birth_year,
@@ -473,12 +501,12 @@ def get_field_values(self, user_row):
     def compare_fields(self, user_obj, values):
         changed = False
         for field in values:
-            if getattr(user_obj, field) != values[field]:
-                changed = True
-            if field == "password" and values["password"] is None:
-                changed = False
-            if changed:
-                setattr(user_obj, field, values[field])
+            if field == "uuid":
+                continue  # uuid can't be updated
+            if field != "password" or values["password"] is not None:
+                if getattr(user_obj, field) != values[field]:
+                    changed = True
+                    setattr(user_obj, field, values[field])
         return changed
 
     def build_users_objects(self, users):
@@ -486,32 +514,61 @@ def build_users_objects(self, users):
         update_users = list()
         keeping_users = list()
         per_line_errors = list()
+        users_uuid = [
+            u[self.header_translation["UUID"]]
+            for u in users.values()
+            if u[self.header_translation["UUID"]] != ""
+        ]
         existing_users = (
             FacilityUser.objects.filter(facility=self.default_facility)
-            .filter(username__in=users.keys())
-            .values_list("username", flat=True)
+            .filter(id__in=users_uuid)
+            .values_list("id", flat=True)
         )
 
         # creating the users takes half of the time
         progress = (100 / self.number_lines) * 0.5
-
         for user in users:
             self.progress_update(progress)
             user_row = users[user]
             values = self.get_field_values(user_row)
-            if user in existing_users:
+            if values["uuid"] in existing_users:
                 user_obj = FacilityUser.objects.get(
-                    username=user, facility=self.default_facility
+                    id=values["uuid"], facility=self.default_facility
                 )
                 keeping_users.append(user_obj)
+                if user_obj.username != user:
+                    # check for duplicated username in the facility
+                    existing_user = FacilityUser.objects.get(
+                        username=user, facility=self.default_facility
+                    )
+                    if existing_user:
+                        error = {
+                            "row": users[user]["position"],
+                            "username": user,
+                            "message": MESSAGES[DUPLICATED_USERNAME],
+                            "field": "USERNAME",
+                            "value": user,
+                        }
+                        per_line_errors.append(error)
+                        continue
                 if self.compare_fields(user_obj, values):
                     update_users.append(user_obj)
             else:
-                if not values["password"]:
+                if values["uuid"] != "":
+                    error = {
+                        "row": users[user]["position"],
+                        "username": user,
+                        "message": MESSAGES[NON_EXISTENT_UUID].format(user),
+                        "field": "PASSWORD",
+                        "value": "*",
+                    }
+                    per_line_errors.append(error)
+                elif not values["password"]:
                     error = {
-                        "row": "USERNAME:{}".format(user),
+                        "row": users[user]["position"],
+                        "username": user,
                         "message": MESSAGES[REQUIRED_PASSWORD],
-                        "field": "password",
+                        "field": "PASSWORD",
                         "value": "*",
                     }
                     per_line_errors.append(error)


Test patch:
diff --git a/integration_testing/features/super-admin/super-admin-bulk-import-users.feature b/integration_testing/features/super-admin/super-admin-bulk-import-users.feature
--- a/integration_testing/features/super-admin/super-admin-bulk-import-users.feature
+++ b/integration_testing/features/super-admin/super-admin-bulk-import-users.feature
@@ -29,10 +29,10 @@ Feature: Super Admin bulk import users into a facility
       And I see that old classes not listed in the CSV file have been deleted
       And the facility contains just classes and users listed in the CSV file
 
-# CSV file content example (for the first row, texts not inside parenthesis may be translated) :
-Username (USERNAME),Password (PASSWORD),Full name (FULL_NAME),User type (USER_TYPE),Identifier (IDENTIFIER),Birth year (BIRTH_YEAR),Gender (GENDER),Learner enrollment (ENROLLED_IN),Coach assignment (ASSIGNED_TO)
-jkrowling,passwd1,Too bad to be here,STUDENT,Potter1,1899,FEMALE,Literature 0,
-ignored_data,passwd2,You are not a coach,LEARNER,,,,,Ignored class
-new_coach,passwd3,Miguel de Cervantes,CLASS_COACH,Sancho1,1969,MALE,,Literature 1
-student1,passwd4,William Shakespeare,LEARNER,Otelo1,2001,MALE,Literature 1,
-student2,passwd5,Agatha Christie,LEARNER,Poirot1,1999,FEMALE,"Literature 1,Chemistry2",
+# CSV file content example (for the first row, texts not inside parenthesis may be translated). For the UUID column, it will be empty to create the user, otherwise it will contain an uuid4 :
+Database ID (UUID),Username (USERNAME),Password (PASSWORD),Full name (FULL_NAME),User type (USER_TYPE),Identifier (IDENTIFIER),Birth year (BIRTH_YEAR),Gender (GENDER),Learner enrollment (ENROLLED_IN),Coach assignment (ASSIGNED_TO)
+,jkrowling,passwd1,Too bad to be here,STUDENT,Potter1,1899,FEMALE,Literature 0,
+,ignored_data,passwd2,You are not a coach,LEARNER,,,,,Ignored class
+,new_coach,passwd3,Miguel de Cervantes,CLASS_COACH,Sancho1,1969,MALE,,Literature 1
+,student1,passwd4,William Shakespeare,LEARNER,Otelo1,2001,MALE,Literature 1,
+,student2,passwd5,Agatha Christie,LEARNER,Poirot1,1999,FEMALE,"Literature 1,Chemistry2",
diff --git a/kolibri/core/auth/test/test_bulk_import.py b/kolibri/core/auth/test/test_bulk_import.py
--- a/kolibri/core/auth/test/test_bulk_import.py
+++ b/kolibri/core/auth/test/test_bulk_import.py
@@ -2,6 +2,7 @@
 import io
 import sys
 import tempfile
+from uuid import uuid4
 
 import pytest
 from django.core.management import call_command
@@ -107,7 +108,7 @@ def setUp(self):
         FacilityUser.objects.all().delete()
         Classroom.objects.all().delete()
 
-    def create_csv(self, filepath, rows):
+    def create_csv(self, filepath, rows, remove_uuid=False):
         header_labels = list(labels.values())
 
         if sys.version_info[0] < 3:
@@ -119,17 +120,21 @@ def create_csv(self, filepath, rows):
             writer = csv.writer(f)
             writer.writerow(header_labels)
             for item in rows:
+                if remove_uuid:
+                    rows[0] = None
                 writer.writerow(item)
 
     def import_exported_csv(self):
         # Replace asterisk in passwords to be able to import it
+        # Remove UUID so new users are created
         _, new_filepath = tempfile.mkstemp(suffix=".csv")
         rows = list()
         with open(self.filepath) as source:
             reader = csv.reader(source, strict=True)
             for row in reader:
-                if row[1] == "*":
-                    row[1] = "temp_password"
+                row[0] = None
+                if row[2] == "*":
+                    row[2] = "temp_password"
                 rows.append(row)
         self.create_csv(new_filepath, rows[1:])  # remove header
         self.filepath = new_filepath
@@ -175,6 +180,7 @@ def test_password_is_required(self):
         _, new_filepath = tempfile.mkstemp(suffix=".csv")
         rows = [
             [
+                None,
                 "new_learner",
                 None,
                 None,
@@ -186,6 +192,7 @@ def test_password_is_required(self):
                 None,
             ],
             [
+                None,
                 "new_coach",
                 "*",
                 None,
@@ -197,6 +204,7 @@ def test_password_is_required(self):
                 "new_class",
             ],
             [
+                None,
                 "another_new_coach",
                 "passwd1",
                 None,
@@ -238,11 +246,13 @@ def test_password_is_required(self):
         # validation when checking db content should trigger an error for '*'  password for a non-existing user:
         assert "'value': '*'" in result[1]
         assert "new_coach" in result[1]
+        assert "'row': 2" in result[1]
 
     def test_asterisk_in_password(self):
         _, first_filepath = tempfile.mkstemp(suffix=".csv")
         rows = [
             [
+                None,
                 "new_learner",
                 "passwd1",
                 None,
@@ -254,6 +264,7 @@ def test_asterisk_in_password(self):
                 None,
             ],
             [
+                None,
                 "new_coach",
                 "passwd2",
                 None,
@@ -269,13 +280,18 @@ def test_asterisk_in_password(self):
         call_command(
             "bulkimportusers", first_filepath, facility=self.facility.id,
         )
-        passwd1 = FacilityUser.objects.get(username="new_learner").password
-        passwd2 = FacilityUser.objects.get(username="new_coach").password
+        user1 = FacilityUser.objects.get(username="new_learner")
+        passwd1 = user1.password
+        uid1 = user1.id
+        user2 = FacilityUser.objects.get(username="new_coach")
+        passwd2 = user2.password
+        uid2 = user2.id
 
         # let's edit the users with a new import
         _, second_filepath = tempfile.mkstemp(suffix=".csv")
         rows = [
             [
+                uid1,
                 "new_learner",
                 "passwd3",
                 None,
@@ -287,6 +303,7 @@ def test_asterisk_in_password(self):
                 None,
             ],
             [
+                uid2,
                 "new_coach",
                 "*",
                 None,
@@ -311,6 +328,7 @@ def test_delete_users_and_classes(self):
         _, new_filepath = tempfile.mkstemp(suffix=".csv")
         rows = [
             [
+                None,
                 "new_learner",
                 "passwd1",
                 None,
@@ -322,6 +340,7 @@ def test_delete_users_and_classes(self):
                 None,
             ],
             [
+                None,
                 "new_coach",
                 "passwd2",
                 None,
@@ -374,6 +393,7 @@ def test_add_users_and_classes(self):
         _, new_filepath = tempfile.mkstemp(suffix=".csv")
         rows = [
             [
+                None,
                 "new_learner",
                 "passwd1",
                 None,
@@ -384,6 +404,7 @@ def test_add_users_and_classes(self):
                 "classroom1,classroom0",
             ],
             [
+                None,
                 "new_coach",
                 "passwd2",
                 None,
@@ -423,6 +444,7 @@ def test_classes_names_case_insensitive(self):
         # first inside the same csv file
         rows = [
             [
+                None,
                 "learner1",
                 "passwd1",
                 None,
@@ -433,6 +455,7 @@ def test_classes_names_case_insensitive(self):
                 " My class,another class ",
             ],
             [
+                None,
                 "coach1",
                 "passwd2",
                 None,
@@ -454,6 +477,7 @@ def test_classes_names_case_insensitive(self):
         # now, testing it's insensitive with database names:
         rows = [
             [
+                None,
                 "learner2",
                 "passwd2",
                 None,
@@ -471,3 +495,27 @@ def test_classes_names_case_insensitive(self):
         )
         classrooms = Classroom.objects.all()
         assert len(classrooms) == 4
+
+    def test_non_existing_uuid(self):
+        self.import_exported_csv()
+        _, new_filepath = tempfile.mkstemp(suffix=".csv")
+        rows = list()
+        with open(self.filepath) as source:
+            reader = csv.reader(source, strict=True)
+            for row in reader:
+                row[0] = uuid4()
+                row[2] = "*"
+                rows.append(row)
+        self.create_csv(new_filepath, rows[1:])  # remove header
+        number_of_rows = len(rows) - 1  # exclude header
+        # import exported csv
+        out_log = StringIO()
+        call_command(
+            "bulkimportusers",
+            new_filepath,
+            facility=self.facility.id,
+            errorlines=out_log,
+        )
+        result = out_log.getvalue().strip().split("\n")
+
+        assert len(result) == number_of_rows