Repository: carpentries/amy

Issue:
Add form for bulk addition of new persons
The app should allow admins to upload a CSV file containing names, email addresses, and optionally roles as well.  The uploaded data should be validated and displayed for further editing before being added.
- If roles are not included in the data, everyone should be marked as a learner, but a warning should be raised.
- If roles are specified, an event must also be specified, and a task should be created for each person showing what role they had in that event.
- Instead of using a fixed column ordering, the upload form should allow users to specify which columns contain which fields on the fly.



Discussion:
Started work on this.

> Instead of using a fixed column ordering, the upload form should allow users to specify which columns contain which fields on the fly.

Can this requirement be dropped? I've worked on something like this before, and I've found that doing a bit of data manipulation in Excel/Calc/Sheets is less error prone. Or, this could be moved to a separate issue as an enhancement. 

However, if you really need it, I'll get something working.  


Resolving patch:
diff --git a/workshops/forms.py b/workshops/forms.py
--- a/workshops/forms.py
+++ b/workshops/forms.py
@@ -53,6 +53,12 @@ def clean(self):
         return cleaned_data
 
 
+class PersonBulkAddForm(forms.Form):
+    '''Represent CSV upload form for bulk adding people.'''
+
+    file = forms.FileField()
+
+
 class SearchForm(forms.Form):
     '''Represent general searching form.'''
 
@@ -64,3 +70,6 @@ class SearchForm(forms.Form):
     in_events = forms.BooleanField(label='in events',
                                    required=False,
                                    initial=True)
+    in_persons = forms.BooleanField(label='in persons',
+                                    required=False,
+                                    initial=True)
diff --git a/workshops/urls.py b/workshops/urls.py
--- a/workshops/urls.py
+++ b/workshops/urls.py
@@ -18,6 +18,7 @@
     url(r'^person/(?P<person_id>[\w\.-]+)/?$', views.person_details, name='person_details'),
     url(r'^person/(?P<person_id>[\w\.-]+)/edit$', views.PersonUpdate.as_view(), name='person_edit'),
     url(r'^persons/add/$', views.PersonCreate.as_view(), name='person_add'),
+    url(r'^persons/bulkadd/$',views.person_bulk_add, name='person_bulk_add'),
 
     url(r'^events/?$', views.all_events, name='all_events'),
     url(r'^event/(?P<event_ident>[\w-]+)/?$', views.event_details, name='event_details'),
diff --git a/workshops/views.py b/workshops/views.py
--- a/workshops/views.py
+++ b/workshops/views.py
@@ -1,16 +1,17 @@
 import re
 import yaml
+import csv
 import requests
 
+from django.contrib import messages
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.core.urlresolvers import reverse
 from django.db.models import Count, Q
-from django.http import HttpResponseRedirect
-from django.shortcuts import render, get_object_or_404
+from django.shortcuts import redirect, render, get_object_or_404
 from django.views.generic.edit import CreateView, UpdateView
 
-from workshops.check import check_file
-from workshops.forms import InstructorsForm, SearchForm
+from workshops.models import Site, Airport, Event, Person, Task, Cohort, Skill, Trainee, Badge, Award, Role
+from workshops.forms import SearchForm, InstructorsForm, PersonBulkAddForm
 from workshops.util import earth_distance
 
 from workshops.models import \
@@ -114,6 +115,9 @@ class AirportUpdate(UpdateView):
 
 #------------------------------------------------------------
 
+PERSON_UPLOAD_FIELDS = ['personal', 'middle', 'family', 'email']
+PERSON_TASK_UPLOAD_FIELDS = PERSON_UPLOAD_FIELDS + ['event', 'role']
+
 def all_persons(request):
     '''List all persons.'''
 
@@ -136,6 +140,58 @@ def person_details(request, person_id):
     return render(request, 'workshops/person.html', context)
 
 
+def person_bulk_add(request):
+    if request.method == 'POST':
+        form = PersonBulkAddForm(request.POST, request.FILES)
+        if form.is_valid():
+            try:
+                persons_tasks = _upload_person_task_csv(request, request.FILES['file'])
+            except csv.Error as e:
+                messages.add_message(request, messages.ERROR, "Error processing uploaded .CSV file: {}".format(e))
+            else:
+                for entry in persons_tasks:
+                    entry['person'].save()
+                    if entry['task']:
+                        entry['task'].person = entry['person'] # Because Django's ORM doesn't do this automatically.
+                        entry['task'].save()
+                context = {'title' : 'Process CSV File',
+                           'form': form, 
+                           'persons_tasks': persons_tasks}
+                return render(request, 'workshops/person_bulk_add_results.html', context)
+    else:
+        form = PersonBulkAddForm()
+
+    context = {'title' : 'Bulk Add People',
+               'form': form}
+    return render(request, 'workshops/person_bulk_add_form.html', context)
+
+
+def _upload_person_task_csv(request, uploaded_file):
+    persons_tasks = []
+    reader = csv.DictReader(uploaded_file)
+    for row in reader:
+        person_fields = dict((col, row[col].strip()) for col in PERSON_UPLOAD_FIELDS)
+        person = Person(**person_fields)
+        entry = {'person': person, 'task' : None}
+        if row.get('event', None) and row.get('role', None):
+            try:
+                event = Event.objects.get(slug=row['event'])
+                role = Role.objects.get(name=row['role'])
+                entry['task'] = Task(person=person, event=event, role=role)
+                import sys
+            except Event.DoesNotExist:
+                messages.add_message(request, messages.ERROR, \
+                                     'Event with slug {} does not exist.'.format(row['event']))
+            except Role.DoesNotExist:
+                messages.add_message(request, messages.ERROR, \
+                                     'Role with name {} does not exist.'.format(row['role'])) 
+            except Role.MultipleObjectsReturned:
+                messages.add_message(request, messages.ERROR, \
+                                     'More than one role named {} exists.'.format(row['role'])) 
+        persons_tasks.append(entry)
+    return persons_tasks
+
+
 class PersonCreate(CreateView):
     model = Person
     fields = '__all__'
@@ -146,6 +202,7 @@ class PersonUpdate(UpdateView):
     fields = '__all__'
     pk_url_kwarg = 'person_id'
 
+
 #------------------------------------------------------------
 
 def all_events(request):
@@ -358,7 +415,7 @@ def instructors(request):
 def search(request):
     '''Search the database by term.'''
 
-    term, sites, events = '', None, None
+    term, sites, events, persons = '', None, None, None
 
     if request.method == 'POST':
         form = SearchForm(request.POST)
@@ -373,6 +430,12 @@ def search(request):
                 events = Event.objects.filter(
                     Q(slug__contains=term) |
                     Q(notes__contains=term))
+            if form.cleaned_data['in_persons']:
+                persons = Person.objects.filter(
+                    Q(personal__contains=term) |
+                    Q(family__contains=term) |
+                    Q(email__contains=term) |
+                    Q(github__contains=term))
         else:
             pass # FIXME: error message
 
@@ -384,7 +447,8 @@ def search(request):
                'form': form,
                'term' : term,
                'sites' : sites,
-               'events' : events}
+               'events' : events,
+               'persons' : persons}
     return render(request, 'workshops/search.html', context)
 
 #------------------------------------------------------------


Test patch:
diff --git a/workshops/test/upload-person-3.csv b/workshops/test/upload-person-3.csv
new file mode 100644
--- /dev/null
+++ b/workshops/test/upload-person-3.csv
@@ -0,0 +1,4 @@
+personal,middle,family,email
+Px1,Mx1,Fx1,Ex1@Sx1.edu
+Px2,Mx2,Fx2,Ex2@Sx2.edu
+Px3,Mx3,Fx3,Ex3@Sx3.edu
diff --git a/workshops/test/upload-person-task-3.csv b/workshops/test/upload-person-task-3.csv
new file mode 100644
--- /dev/null
+++ b/workshops/test/upload-person-task-3.csv
@@ -0,0 +1,4 @@
+personal,middle,family,email,event,role
+Px1,Mx1,Fx1,Ex1@Sx1.edu,2011-11-07-toronto,learner
+Px2,Mx2,Fx2,Ex2@Sx2.edu,2011-11-07-toronto,instructor
+Px3,Mx3,Fx3,Ex3@Sx3.edu,2011-11-07-toronto,learner