Repository: carpentries/amy

Issue:
Provide "use that person" during bulk upload.
1. Create CSV file.
2. Bulk upload.
3. Get told "There's a matching person in the database: Alan Turing. Use their email to merge."
4. But their email isn't displayed.
5. And it would be really helpful if there a button there that would switch the info to _that person_ in place.

In bulk add person, interpret "role" column and use <select>.
In bulk add persons (https://amy.software-carpentry.org/workshops/persons/bulkadd/) we are given a csv template which is useful.

However, the names of roles as they display to the end user are not the same as the names of roles as stored in the database.  Can we either have instructions on what the roles need to be or have the csv importer translate this?



Discussion:

@maneesha, here is the translation:

| name in db | verbose name |
| --- | --- |
| contributor | Contributed to lesson materials |
| organizer | Workshop organizer |
| learner | Learner |
| host | Workshop host |
| instructor | Instructor |
| helper | Helper |

Where could I put the instructions you asked for?

We should:
1. match any-case 
2. provide a `<select>` to choose from for roles when editing a role in the table

@chrismedrela The instructions should go on the Bulk Import page.  I also like @pbanaszkiewicz idea to match case and select to edit.

This wasn't completely fixed in #1023.


Resolving patch:
diff --git a/workshops/urls.py b/workshops/urls.py
--- a/workshops/urls.py
+++ b/workshops/urls.py
@@ -40,13 +40,17 @@
         url(r'^delete/$', views.AirportDelete.as_view(), name='airport_delete'),
     ])),
 
+    url(r'^bulk_upload/', include([
+        url(r'^$',views.person_bulk_add, name='person_bulk_add'),
+        url(r'^template/$',views.person_bulk_add_template, name='person_bulk_add_template'),
+        url(r'^confirm/$',views.person_bulk_add_confirmation, name='person_bulk_add_confirmation'),
+        url(r'^(?P<entry_id>\d+)/remove/$',views.person_bulk_add_remove_entry, name='person_bulk_add_remove_entry'),
+        url(r'^(?P<entry_id>\d+)/match_person/(?P<person_id>\d+)$',views.person_bulk_add_match_person, name='person_bulk_add_match_person'),
+    ])),
+
     url(r'^persons/', include([
         url(r'^$', views.AllPersons.as_view(), name='all_persons'),
         url(r'^add/$', views.PersonCreate.as_view(), name='person_add'),
-        url(r'^bulkadd/$',views.person_bulk_add, name='person_bulk_add'),
-        url(r'^bulkadd/template/$',views.person_bulk_add_template, name='person_bulk_add_template'),
-        url(r'^bulkadd/confirm/$',views.person_bulk_add_confirmation, name='person_bulk_add_confirmation'),
-        url(r'^bulkadd/(?P<entry_id>\d+)/remove/$',views.person_bulk_add_remove_entry, name='person_bulk_add_remove_entry'),
         url(r'^merge/$', views.persons_merge, name='persons_merge'),
     ])),
     url(r'^person/(?P<person_id>\d+)/', include([
diff --git a/workshops/util.py b/workshops/util.py
--- a/workshops/util.py
+++ b/workshops/util.py
@@ -137,47 +137,29 @@ def verify_upload_person_task(data):
         email = item.get('email', None)
         personal = item.get('personal', None)
         family = item.get('family', None)
+        person_id = item.get('existing_person_id', None)
         person = None
 
-        if email:
+        if person_id:
             try:
-                # check if first and last name matches person in the database
-                person = Person.objects.get(email__iexact=email)
-
-                for which, actual, uploaded in (
-                        ('personal', person.personal, personal),
-                        ('family', person.family, family)
-                ):
-                    if (actual == uploaded) or (not actual and not uploaded):
-                        pass
-                    else:
-                        errors.append('{0} mismatch: database "{1}" '
-                                      'vs uploaded "{2}".'
-                                      .format(which, actual, uploaded))
-
-            except Person.DoesNotExist:
-                # in this case we need to add a new person
-                pass
-
+                person = Person.objects.get(id=int(person_id))
+            except (ValueError, TypeError, Person.DoesNotExist):
+                person = None
+                info.append('Could not match selected person. New record will '
+                            'be created.')
             else:
-                if existing_event and person and existing_role:
-                    # person, their role and a corresponding event exist, so
-                    # let's check if the task exists
-                    try:
-                        Task.objects.get(event=existing_event, person=person,
-                                         role=existing_role)
-                    except Task.DoesNotExist:
-                        info.append('Task will be created.')
-                    else:
-                        info.append('Task already exists.')
-        else:
+                info.append('Existing record for person will be used.')
+
+        if not email and not person:
             info.append('It\'s highly recommended to add an email address.')
 
         if person:
-            # force username from existing record
+            # force details from existing record
+            item['personal'] = personal = person.personal
+            item['family'] = family = person.family
+            item['email'] = email = person.email
             item['username'] = person.username
             item['person_exists'] = True
-
         else:
             # force a newly created username
             if not item.get('username'):
@@ -186,26 +168,26 @@ def verify_upload_person_task(data):
 
             info.append('Person and task will be created.')
 
+        # let's check if there's someone else named this way
+        similar_persons = Person.objects.filter(
+            Q(personal=personal, family=family) |
+            Q(email=email) & ~Q(email='') & Q(email__isnull=False)
+        )
+        # need to cast to list, otherwise it won't JSON-ify
+        item['similar_persons'] = list(similar_persons.values(
+            'id', 'personal', 'middle', 'family', 'email', 'username',
+        ))
+
+        if existing_event and person and existing_role:
+            # person, their role and a corresponding event exist, so
+            # let's check if the task exists
             try:
-                # let's check if there's someone else named this way
-                similar_person = Person.objects.get(personal=personal,
-                                                    family=family)
-
-            except Person.DoesNotExist:
-                pass
-
-            except Person.MultipleObjectsReturned:
-                persons = [
-                    str(person) for person in
-                    Person.objects.filter(personal=personal, family=family)
-                ]
-                info.append('There\'s a couple of matching persons in the '
-                            'database: {}. '
-                            'Use email to merge.'.format(', '.join(persons)))
-
+                Task.objects.get(event=existing_event, person=person,
+                                 role=existing_role)
+            except Task.DoesNotExist:
+                info.append('Task will be created.')
             else:
-                info.append('There\'s a matching person in the database: {}. '
-                            'Use their email to merge.'.format(similar_person))
+                info.append('Task already exists.')
 
         # let's check what Person model validators want to say
         try:
@@ -254,14 +236,16 @@ def create_uploaded_persons_tasks(data):
                 fields = {key: row[key] for key in Person.PERSON_UPLOAD_FIELDS}
                 fields['username'] = row['username']
 
-                if fields['email']:
-                    # we should use existing Person or create one
-                    p, created = Person.objects.get_or_create(
-                        email__iexact=fields['email'], defaults=fields
-                    )
+                if row['person_exists'] and row['existing_person_id']:
+                    # we should use existing Person
+                    p = Person.objects.get(pk=row['existing_person_id'])
 
-                    if created:
-                        persons_created.append(p)
+                elif row['person_exists'] and not row['existing_person_id']:
+                    # we should use existing Person
+                    p = Person.objects.get(
+                        personal=fields['personal'], family=fields['family'],
+                        username=fields['username'], email=fields['email'],
+                    )
 
                 else:
                     # we should create a new Person without any email provided
@@ -273,7 +257,7 @@ def create_uploaded_persons_tasks(data):
                     e = Event.objects.get(slug=row['event'])
                     r = Role.objects.get(name=row['role'])
 
-                    # is the number of learners attending the event changed,
+                    # if the number of learners attending the event changed,
                     # we should update ``event.attendance``
                     if row['role'] == 'learner':
                         events.add(e)
@@ -284,12 +268,12 @@ def create_uploaded_persons_tasks(data):
                         tasks_created.append(t)
 
             except IntegrityError as e:
-
-                raise IntegrityError('{0} (for "{1}")'.format(str(e), row_repr))
+                raise IntegrityError('{0} (for "{1}")'.format(str(e),
+                                                              row_repr))
 
             except ObjectDoesNotExist as e:
                 raise ObjectDoesNotExist('{0} (for "{1}")'.format(str(e),
-                                                                row_repr))
+                                                                  row_repr))
 
     for event in events:
         # if event.attendance is lower than number of learners, then
diff --git a/workshops/views.py b/workshops/views.py
--- a/workshops/views.py
+++ b/workshops/views.py
@@ -544,12 +544,14 @@ def person_bulk_add_confirmation(request):
                           roles)
         for k, record in enumerate(data_update):
             personal, family, username, email, event, role = record
+            existing_person_id = persons_tasks[k].get('existing_person_id')
             # "field or None" converts empty strings to None values
             persons_tasks[k] = {
                 'personal': personal,
                 'family': family,
                 'username': username,
-                'email': email or None
+                'email': email or None,
+                'existing_person_id': existing_person_id,
             }
             # when user wants to drop related event they will send empty string
             # so we should unconditionally accept new value for event even if
@@ -570,12 +572,6 @@ def person_bulk_add_confirmation(request):
                                      "Please make sure to fix all errors "
                                      "listed below.")
 
-            context = {'title': 'Confirm uploaded data',
-                       'persons_tasks': persons_tasks,
-                       'any_errors': any_errors}
-            return render(request, 'workshops/person_bulk_add_results.html',
-                          context)
-
         # there must be "confirm" and no "cancel" in POST in order to save
         elif (request.POST.get('confirm', None) and
               not request.POST.get('cancel', None)):
@@ -591,12 +587,6 @@ def person_bulk_add_confirmation(request):
                                      "Please make sure to fix all errors "
                                      "listed below.".format(e))
                 any_errors = verify_upload_person_task(persons_tasks)
-                context = {'title': 'Confirm uploaded data',
-                           'persons_tasks': persons_tasks,
-                           'any_errors': any_errors}
-                return render(request,
-                              'workshops/person_bulk_add_results.html',
-                              context, status=400)
 
             else:
                 request.session['bulk-add-people'] = None
@@ -616,11 +606,16 @@ def person_bulk_add_confirmation(request):
         # alters persons_tasks via reference
         any_errors = verify_upload_person_task(persons_tasks)
 
-        context = {'title': 'Confirm uploaded data',
-                   'persons_tasks': persons_tasks,
-                   'any_errors': any_errors}
-        return render(request, 'workshops/person_bulk_add_results.html',
-                      context)
+    roles = Role.objects.all().values_list('name', flat=True)
+
+    context = {
+        'title': 'Confirm uploaded data',
+        'persons_tasks': persons_tasks,
+        'any_errors': any_errors,
+        'possible_roles': roles,
+    }
+    return render(request, 'workshops/person_bulk_add_results.html',
+                  context)
 
 
 @admin_required
@@ -647,6 +642,32 @@ def person_bulk_add_remove_entry(request, entry_id):
         return redirect('person_bulk_add')
 
 
+@admin_required
+@permission_required('workshops.add_person', raise_exception=True)
+def person_bulk_add_match_person(request, entry_id, person_id):
+    """Save information about matched person in the session-saved data."""
+    persons_tasks = request.session.get('bulk-add-people')
+
+    if persons_tasks:
+        entry_id = int(entry_id)
+        person_id = int(person_id)
+
+        try:
+            persons_tasks[entry_id]['existing_person_id'] = person_id
+            request.session['bulk-add-people'] = persons_tasks
+
+        except IndexError:
+            messages.warning(request, 'Could not find specified entry #{}'
+                                      .format(entry_id))
+
+        return redirect(person_bulk_add_confirmation)
+
+    else:
+        messages.warning(request, 'Could not locate CSV data, please try the '
+                                  'upload again.')
+        return redirect('person_bulk_add')
+
+
 class PersonCreate(OnlyForAdminsMixin, PermissionRequiredMixin,
                    AMYCreateView):
     permission_required = 'workshops.add_person'


Test patch:
diff --git a/workshops/test/test_bulk_add.py b/workshops/test/test_bulk_add.py
--- a/workshops/test/test_bulk_add.py
+++ b/workshops/test/test_bulk_add.py
@@ -177,16 +177,6 @@ def test_verify_email_caseinsensitive_matches(self):
             has_errors = verify_upload_person_task(bad_data)
             self.assertFalse(has_errors)
 
-    def test_verify_name_matching_existing_user(self):
-        bad_data = self.make_data()
-        bad_data[0]['email'] = 'harry@hogwarts.edu'
-        has_errors = verify_upload_person_task(bad_data)
-        self.assertTrue(has_errors)
-        errors = bad_data[0]['errors']
-        self.assertEqual(len(errors), 2)
-        self.assertTrue('personal' in errors[0])
-        self.assertTrue('family' in errors[1])
-
     def test_verify_existing_user_has_workshop_role_provided(self):
         bad_data = [
             {
@@ -214,6 +204,9 @@ def test_username_from_existing_person(self):
                 'email': 'harry@hogwarts.edu',
                 'event': '',
                 'role': '',
+                'existing_person_id': Person.objects
+                                            .get(email='harry@hogwarts.edu')
+                                            .pk,
             }
         ]
         verify_upload_person_task(data)
@@ -229,11 +222,42 @@ def test_username_from_nonexisting_person(self):
                 'email': 'h.frotter@hogwarts.edu',
                 'event': '',
                 'role': '',
+                'existing_person_id': None,
             }
         ]
         verify_upload_person_task(data)
         self.assertEqual('supplied_username', data[0]['username'])
 
+    def test_matched_similar_persons(self):
+        """Ensure function finds matching persons."""
+        data = [
+            {
+                'personal': 'Harry',
+                'family': 'Potter',
+                'username': 'supplied_username',
+                'email': 'h.frotter@hogwarts.edu',
+                'event': '',
+                'role': '',
+            },
+            {
+                'personal': 'Romuald',
+                'family': 'Weazel',
+                'username': '',
+                'email': 'rweasley@ministry.gov.uk',
+                'event': '',
+                'role': '',
+            }
+        ]
+        verify_upload_person_task(data)
+
+        self.assertEqual(len(data[0]['similar_persons']), 1)
+        self.assertEqual(len(data[1]['similar_persons']), 1)
+
+        self.assertEqual(data[0]['similar_persons'][0]['username'],
+                         'potter_harry')
+        self.assertEqual(data[1]['similar_persons'][0]['username'],
+                         'weasley_ron')
+
 
 class BulkUploadUsersViewTestCase(CSVBulkUploadTestBase):
 
@@ -285,6 +309,10 @@ def test_upload_existing_user(self):
 """
         data, _ = upload_person_task_csv(StringIO(csv))
 
+        # simulate user clicking "Use this user" next to matched person
+        data[0]['existing_person_id'] = \
+            Person.objects.get(email='harry@hogwarts.edu').pk
+
         # self.client is authenticated user so we have access to the session
         store = self.client.session
         store['bulk-add-people'] = data
@@ -375,6 +403,10 @@ def test_attendance_increases(self):
 """
         data, _ = upload_person_task_csv(StringIO(csv))
 
+        # simulate user clicking "Use this user" next to matched person
+        data[0]['existing_person_id'] = \
+            Person.objects.get(email='harry@hogwarts.edu').pk
+
         # self.client is authenticated user so we have access to the session
         store = self.client.session
         store['bulk-add-people'] = data
@@ -463,3 +495,14 @@ def test_removing_entry2(self):
         self.assertEqual(2, len(data))
         self.assertEqual(data[0]['personal'], 'Harry')
         self.assertEqual(data[1]['personal'], 'Hermione')
+
+
+class BulkUploadMatchPersonViewTestCase(CSVBulkUploadTestBase):
+    def setUp(self):
+        super().setUp()
+        Role.objects.create(name='Helper')
+        self.csv = """personal,family,email,event,role
+Harry,Potter,harry@hogwarts.edu,foobar,learner
+Hermione,Granger,hermione@hogwarts.edu,foobar,learner
+Ron,Weasley,ron@hogwarts.edu,foobar,learner
+"""