Repository: carpentries/amy

Issue:
Add an easy way for a person to download all their data from AMY
This probably should be done via API endpoint.


Discussion:

Resolving patch:
diff --git a/api/serializers.py b/api/serializers.py
--- a/api/serializers.py
+++ b/api/serializers.py
@@ -10,6 +10,9 @@
     Organization,
     Task,
     Award,
+    TrainingRequest,
+    TrainingRequirement,
+    TrainingProgress,
 )
 
 
@@ -39,6 +42,14 @@ class Meta:
         fields = ('name', 'email', 'username')
 
 
+class PersonNameSerializer(serializers.ModelSerializer):
+    name = serializers.CharField(source='get_full_name')
+
+    class Meta:
+        model = Person
+        fields = ('name', )
+
+
 class ExportBadgesSerializer(serializers.ModelSerializer):
     persons = AwardPersonSerializer(many=True, source='award_set')
 
@@ -265,3 +276,120 @@ class Meta:
             'attendance', 'contact', 'country', 'venue', 'address',
             'latitude', 'longitude', 'notes', 'tasks', 'todos', 'assigned_to',
         )
+
+
+class TrainingRequestSerializer(serializers.ModelSerializer):
+    state = serializers.CharField(source='get_state_display')
+    domains = serializers.SlugRelatedField(
+        many=True, read_only=True, slug_field='name')
+    previous_involvement = serializers.SlugRelatedField(
+        many=True, read_only=True, slug_field='name')
+    previous_training = serializers.CharField(
+        source='get_previous_training_display')
+    previous_experience = serializers.CharField(
+        source='get_previous_experience_display')
+    programming_language_usage_frequency = serializers.CharField(
+        source='get_programming_language_usage_frequency_display')
+    teaching_frequency_expectation = serializers.CharField(
+        source='get_teaching_frequency_expectation_display')
+    max_travelling_frequency = serializers.CharField(
+        source='get_max_travelling_frequency_display')
+
+    class Meta:
+        model = TrainingRequest
+        fields = (
+            'created_at', 'last_updated_at',
+            'state', 'group_name', 'personal', 'middle', 'family', 'email',
+            'github', 'occupation', 'occupation_other', 'affiliation',
+            'location', 'country', 'underresourced', 'underrepresented',
+            'domains', 'domains_other', 'nonprofit_teaching_experience',
+            'previous_involvement', 'previous_training',
+            'previous_training_other', 'previous_training_explanation',
+            'previous_experience', 'previous_experience_other',
+            'previous_experience_explanation',
+            'programming_language_usage_frequency',
+            'teaching_frequency_expectation',
+            'teaching_frequency_expectation_other',
+            'max_travelling_frequency', 'max_travelling_frequency_other',
+            'reason', 'comment',
+            'training_completion_agreement', 'workshop_teaching_agreement',
+            'data_privacy_agreement', 'code_of_conduct_agreement',
+        )
+
+
+# The serializers below are meant to help display user's data without any
+# links in relational fields; instead, either an expanded model is displayed,
+# or - if it's simple enough - its' string representation.
+# The serializers are used mostly in ExportPersonDataView.
+
+
+class EventSerializerSimplified(EventSerializer):
+    class Meta:
+        model = Event
+        fields = (
+            'slug', 'start', 'end', 'tags', 'website_url', 'venue',
+            'address', 'country', 'latitude', 'longitude',
+        )
+
+
+class AwardSerializerExpandEvent(AwardSerializer):
+    event = EventSerializerSimplified(many=False, read_only=True)
+
+
+class TrainingRequirementSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = TrainingRequirement
+        fields = (
+            'name', 'url_required', 'event_required',
+        )
+
+
+class TrainingProgressSerializer(serializers.ModelSerializer):
+    requirement = TrainingRequirementSerializer(many=False, read_only=True)
+    state = serializers.CharField(source='get_state_display')
+    event = EventSerializerSimplified(many=False, read_only=True)
+    evaluated_by = PersonNameSerializer(many=False, read_only=True)
+
+    class Meta:
+        model = TrainingProgress
+        fields = (
+            'created_at', 'last_updated_at',
+            'requirement', 'state', 'discarded',
+            'evaluated_by', 'event', 'url',
+        )
+
+
+class TaskSerializerNoPerson(TaskSerializer):
+    event = EventSerializerSimplified(many=False, read_only=True)
+    role = serializers.SlugRelatedField(
+        many=False, read_only=True, slug_field='name')
+
+    class Meta:
+        model = Task
+        fields = ('event', 'role')
+
+
+class PersonSerializerAllData(PersonSerializer):
+    airport = AirportSerializer(many=False, read_only=True)
+    badges = BadgeSerializer(many=True, read_only=True)
+    awards = AwardSerializerExpandEvent(many=True, read_only=True, source='award_set')
+    tasks = TaskSerializerNoPerson(many=True, read_only=True,
+                                   source='task_set')
+    languages = serializers.SlugRelatedField(
+        many=True, read_only=True, slug_field='name')
+    training_requests = TrainingRequestSerializer(
+        many=True, read_only=True, source='trainingrequest_set')
+    training_progresses = TrainingProgressSerializer(
+        many=True, read_only=True, source='trainingprogress_set')
+
+    class Meta:
+        model = Person
+        fields = (
+            'username', 'personal', 'middle', 'family', 'email', 'gender',
+            'may_contact', 'publish_profile', 'airport',
+            'github', 'twitter', 'url', 'affiliation',
+            'user_notes', 'occupation', 'orcid',
+            'data_privacy_agreement',
+            'badges', 'lessons', 'languages', 'domains', 'awards', 'tasks',
+            'training_requests', 'training_progresses',
+        )
diff --git a/api/urls.py b/api/urls.py
--- a/api/urls.py
+++ b/api/urls.py
@@ -41,6 +41,9 @@
     url('^export/members/$',
         views.ExportMembersView.as_view(),
         name='export-members'),
+    url('^export/person_data/$',
+        views.ExportPersonDataView.as_view(),
+        name='export-person-data'),
     url('^events/published/$',
         views.PublishedEvents.as_view(),
         name='events-published'),
diff --git a/api/views.py b/api/views.py
--- a/api/views.py
+++ b/api/views.py
@@ -15,7 +15,7 @@
 )
 from rest_framework import viewsets
 from rest_framework.decorators import list_route
-from rest_framework.generics import ListAPIView
+from rest_framework.generics import ListAPIView, RetrieveAPIView
 from rest_framework.metadata import SimpleMetadata
 from rest_framework.pagination import PageNumberPagination
 from rest_framework.permissions import (
@@ -62,6 +62,7 @@
     AirportSerializer,
     AwardSerializer,
     PersonSerializer,
+    PersonSerializerAllData,
 )
 
 from .filters import (
@@ -125,6 +126,8 @@ def get(self, request, format=None):
                                            request=request, format=format)),
             ('export-members', reverse('api:export-members', request=request,
                                        format=format)),
+            ('export-person-data', reverse('api:export-person-data',
+                                           request=request, format=format)),
             ('events-published', reverse('api:events-published',
                                          request=request, format=format)),
             ('user-todos', reverse('api:user-todos',
@@ -264,6 +267,22 @@ def get_query_params_description(self):
         }
 
 
+class ExportPersonDataView(RetrieveAPIView):
+    permission_classes = (IsAuthenticated, )
+    serializer_class = PersonSerializerAllData
+    queryset = Person.objects.all()
+
+    def get_object(self):
+        """Get logged-in user data, make it impossible to gather someone else's
+        data this way."""
+        user = self.request.user
+
+        if user.is_anonymous:
+            return self.get_queryset().none()
+        else:
+            return self.get_queryset().get(pk=user.pk)
+
+
 class PublishedEvents(ListAPIView):
     """List published events."""
 


Test patch:
diff --git a/api/test/test_export.py b/api/test/test_export.py
--- a/api/test/test_export.py
+++ b/api/test/test_export.py
@@ -19,6 +19,13 @@
     Person,
     Airport,
     Role,
+    Organization,
+    Event,
+    Language,
+    KnowledgeDomain,
+    TrainingRequest,
+    TrainingRequirement,
+    TrainingProgress,
 )
 from workshops.util import universal_date_format
 
@@ -30,12 +37,16 @@ def setUp(self):
         # 0064
         Badge.objects.all().delete()
 
-    def login(self):
+    def setup_admin(self):
         self.admin = Person.objects.create_superuser(
                 username="admin", personal="Super", family="User",
                 email="sudo@example.org", password='admin')
         self.admin.data_privacy_agreement = True
         self.admin.save()
+
+    def login(self):
+        if not hasattr(self, 'admin'):
+            self.setup_admin()
         self.client.login(username='admin', password='admin')
 
 
@@ -494,3 +505,454 @@ def test_view_explicit_earliest(self):
         content = response.content.decode('utf-8')
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(json.loads(content), self.expecting)
+
+
+class TestExportingPersonData(BaseExportingTest):
+    def setUp(self):
+        # don't remove all badges
+        # super().setUp()
+
+        # prepare user
+        self.user = Person(
+            username="primary_user", personal="User", family="Primary",
+            email="primary_user@amy.com", is_active=True,
+            data_privacy_agreement=True,
+        )
+        self.user.set_password('password')
+        self.user.save()
+
+        # save API endpoint URL
+        self.url = reverse('api:export-person-data')
+
+    def login(self):
+        """Overwrite BaseExportingTest's login method: instead of loggin in
+        as an admin, use a normal user."""
+        self.client.login(username='primary_user', password='password')
+
+    def prepare_data(self, user):
+        """Populate relational fields for the user."""
+
+        # create and set airport for the user
+        airport = Airport.objects.create(
+            iata='DDD', fullname='Airport 55x105',
+            country='CM',
+            latitude=55.0, longitude=105.0,
+        )
+        self.user.airport = airport
+        self.user.save()
+
+        # create a fake organization
+        test_host = Organization.objects.create(
+            domain='example.com', fullname='Test Organization')
+
+        # create an event that will later be used
+        event = Event.objects.create(
+            start=datetime.date(2018, 6, 16),
+            end=datetime.date(2018, 6, 17),
+            slug='2018-06-16-AMY-event',
+            host=test_host,
+            url='http://example.org/2018-06-16-AMY-event',
+        )
+
+        # add a role
+        Role.objects.create(name='instructor', verbose_name='Instructor')
+
+        # add an admin user
+        self.setup_admin()
+
+        # award user some badges via awards (intermediary model)
+        # one badge was awarded for the event
+        award1 = Award.objects.create(
+            person=self.user,
+            badge=Badge.objects.get(name='swc-instructor'),
+            event=event,
+            awarded=datetime.date(2018, 6, 16),
+        )
+        # second badge was awarded without any connected event
+        award2 = Award.objects.create(
+            person=self.user,
+            badge=Badge.objects.get(name='dc-instructor'),
+            awarded=datetime.date(2018, 6, 16),
+        )
+
+        # user took part in the event as an instructor
+        self.user.task_set.create(
+            event=event, role=Role.objects.get(name='instructor'),
+        )
+
+        # user knows a couple of languages
+        self.user.languages.set(
+            Language.objects.filter(name__in=['English', 'French'])
+        )
+
+        # add training requests
+        training_request = TrainingRequest.objects.create(
+            # mixins
+            active=True,
+            data_privacy_agreement=True,
+            code_of_conduct_agreement=True,
+
+            state='p',  # pending
+            person=self.user,
+            group_name='Mosquitos',
+            personal='User',
+            middle='',
+            family='Primary',
+            email='primary_user@amy.com',
+            github='primary_user',
+            occupation='undisclosed',
+            occupation_other='',
+            affiliation='AMY',
+            location='Worldwide',
+            country='W3',
+            underresourced=False,
+            # need to set it below
+            # domains=KnowledgeDomain.objects.first(),
+            domains_other='E-commerce',
+            underrepresented='',
+            nonprofit_teaching_experience='Voluntary teacher',
+            # need to set it below
+            # previous_involvement=Role.objects.filter(name='instructor'),
+            previous_training='course',
+            previous_training_other='',
+            previous_training_explanation='A course for voluntary teaching',
+            previous_experience='ta',
+            previous_experience_other='',
+            previous_experience_explanation='After the course I became a TA',
+            programming_language_usage_frequency='weekly',
+            teaching_frequency_expectation='monthly',
+            max_travelling_frequency='not-at-all',
+            max_travelling_frequency_other='',
+            reason='I want to became an instructor',
+            comment='I like trains',
+            training_completion_agreement=True,
+            workshop_teaching_agreement=True,
+            notes='Admin notes invisible to the user',
+        )
+        training_request.domains.set([KnowledgeDomain.objects.first()])
+        training_request.previous_involvement.set(
+            Role.objects.filter(name='instructor'))
+
+        # add some training progress
+        TrainingProgress.objects.create(
+            trainee=self.user,
+            requirement=TrainingRequirement.objects.get(name='Discussion'),
+            state='p',  # passed
+            event=event,
+            evaluated_by=None,
+            discarded=False,
+            url=None,
+        )
+        TrainingProgress.objects.create(
+            trainee=self.user,
+            requirement=TrainingRequirement.objects.get(name='DC Homework'),
+            state='f',  # failed
+            event=None,
+            evaluated_by=self.admin,
+            discarded=False,
+            url='http://example.org/homework',
+        )
+
+    def test_unauthorized_access(self):
+        """Make sure only authenticated users can access."""
+        # logout
+        self.client.logout()
+
+        # retrieve endpoint
+        rv = self.client.get(self.url)
+
+        # make sure it's inaccessible
+        self.assertEqual(rv.status_code, 401)
+
+    def test_only_for_one_user(self):
+        """Make sure the results are available only for the logged-in user,
+        no-one else."""
+        # prepare a different user
+        self.second_user = Person(
+            username="secondary_user", personal="User", family="Secondary",
+            email="secondary_user@amy.com", is_active=True,
+            data_privacy_agreement=True,
+        )
+        self.second_user.set_password('password')
+        self.second_user.save()
+
+        # login as first user
+        self.client.login(username='primary_user', password='password')
+
+        # retrieve endpoint
+        rv = self.client.get(self.url)
+        self.assertEqual(rv.status_code, 200)
+        # make sure this endpoint returns current user data
+        self.assertEqual(rv.json()['username'], 'primary_user')
+
+        # login as second user
+        self.client.login(username='secondary_user', password='password')
+        rv = self.client.get(self.url)
+        self.assertEqual(rv.status_code, 200)
+        # make sure this endpoint does not return first user data now
+        self.assertEqual(rv.json()['username'], 'secondary_user')
+
+
+    def test_all_related_objects_shown(self):
+        """Test if all related fields are present in data output."""
+        self.login()
+
+        # retrieve endpoint
+        rv = self.client.get(self.url)
+        self.assertEqual(rv.status_code, 200)
+
+        # API results parsed as JSON
+        user_data = rv.json()
+        user_data_keys = user_data.keys()
+
+        # make sure these fields are NOT in the API output
+        missing_fields = [
+            'password',
+            'is_active',
+            'notes',
+        ]
+
+        # simple (non-relational) fields expected in API output
+        expected_fields = [
+            'data_privacy_agreement',
+            'personal',
+            'middle',
+            'family',
+            'email',
+            'username',
+            'gender',
+            'may_contact',
+            'publish_profile',
+            'github',
+            'twitter',
+            'url',
+            'user_notes',
+            'affiliation',
+            'occupation',
+            'orcid',
+        ]
+
+        # relational fields expected in API output
+        expected_relational = [
+            'airport',
+            'badges',
+            'lessons',
+            'domains',
+            'languages',
+            'tasks',  # renamed in serializer (was: task_set)
+            'awards',  # renamed in serializer (was: award_set)
+            'training_requests',  # renamed from "trainingrequest_set"
+            'training_progresses',  # renamed from "trainingprogress_set"
+        ]
+
+        # ensure missing fields are not to be found in API output
+        for field in missing_fields:
+            self.assertNotIn(field, user_data_keys)
+
+        # ensure required fields are present
+        for field in expected_fields + expected_relational:
+            self.assertIn(field, user_data_keys)
+
+    def test_relational_fields_structure(self):
+        """Make sure relational fields available via API endpoints
+        retain a specific structure."""
+        self.prepare_data(user=self.user)
+        self.login()
+
+        # retrieve endpoint
+        rv = self.client.get(self.url)
+        self.assertEqual(rv.status_code, 200)
+
+        # API results parsed as JSON
+        data = rv.json()
+
+        # expected data dict
+        expected = dict()
+
+        # test expected Airport output
+        expected['airport'] = {
+            'iata': 'DDD',
+            'fullname': 'Airport 55x105',
+            'country': 'CM',
+            'latitude': 55.0,
+            'longitude': 105.0,
+        }
+        self.assertEqual(data['airport'], expected['airport'])
+
+        # test expected Badges output
+        expected['badges'] = [
+            {
+                'name': 'swc-instructor',
+                'title': 'Software Carpentry Instructor',
+                'criteria': 'Teaching at Software Carpentry workshops or'
+                            ' online',
+            },
+            {
+                'name': 'dc-instructor',
+                'title': 'Data Carpentry Instructor',
+                'criteria': 'Teaching at Data Carpentry workshops or'
+                            ' online',
+            },
+        ]
+        self.assertEqual(data['badges'], expected['badges'])
+
+        # test expected Awards output
+        expected['awards'] = [
+            {
+                'badge': 'swc-instructor',
+                'awarded': '2018-06-16',
+                'event': {
+                    'slug': '2018-06-16-AMY-event',
+                    'start': '2018-06-16',
+                    'end': '2018-06-17',
+                    'tags': [],
+                    'website_url': 'http://example.org/2018-06-16-AMY-event',
+                    'venue': '',
+                    'address': '',
+                    'country': '',
+                    'latitude': None,
+                    'longitude': None,
+                }
+            },
+            {
+                'badge': 'dc-instructor',
+                'awarded': '2018-06-16',
+                'event': None,
+            },
+        ]
+        self.assertEqual(data['awards'], expected['awards'])
+
+        # test expected Tasks output
+        expected['tasks'] = [
+            {
+                'event': {
+                    'slug': '2018-06-16-AMY-event',
+                    'start': '2018-06-16',
+                    'end': '2018-06-17',
+                    'tags': [],
+                    'website_url': 'http://example.org/2018-06-16-AMY-event',
+                    'venue': '',
+                    'address': '',
+                    'country': '',
+                    'latitude': None,
+                    'longitude': None,
+                },
+                'role': 'instructor',
+            },
+        ]
+        self.assertEqual(data['tasks'], expected['tasks'])
+
+        # test expected Languages output
+        expected['languages'] = [
+            'English',
+            'French',
+        ]
+        self.assertEqual(data['languages'], expected['languages'])
+
+        # test expected TrainingRequests output
+        expected['training_requests'] = [
+            {
+                # these are generated by Django, so we borrow them from the
+                # output
+                'created_at': data['training_requests'][0]['created_at'],
+                'last_updated_at':
+                    data['training_requests'][0]['last_updated_at'],
+                'state': 'Pending',
+                'group_name': 'Mosquitos',
+                'personal': 'User',
+                'middle': '',
+                'family': 'Primary',
+                'email': 'primary_user@amy.com',
+                'github': 'primary_user',
+                'occupation': 'undisclosed',
+                'occupation_other': '',
+                'affiliation': 'AMY',
+                'location': 'Worldwide',
+                'country': 'W3',
+                'underresourced': False,
+                'domains': ['Chemistry'],
+                'domains_other': 'E-commerce',
+                'underrepresented': '',
+                'nonprofit_teaching_experience': 'Voluntary teacher',
+                'previous_involvement': ['instructor'],
+                'previous_training': 'A certification or short course',
+                'previous_training_other': '',
+                'previous_training_explanation':
+                    'A course for voluntary teaching',
+                'previous_experience': 'Teaching assistant for a full course',
+                'previous_experience_other': '',
+                'previous_experience_explanation':
+                    'After the course I became a TA',
+                'programming_language_usage_frequency': 'A few times a week',
+                'teaching_frequency_expectation': 'Several times a year',
+                'teaching_frequency_expectation_other': '',
+                'max_travelling_frequency': 'Not at all',
+                'max_travelling_frequency_other': '',
+                'reason': 'I want to became an instructor',
+                'comment': 'I like trains',
+                'training_completion_agreement': True,
+                'workshop_teaching_agreement': True,
+                'data_privacy_agreement': True,
+                'code_of_conduct_agreement': True,
+            }
+        ]
+
+        self.assertEqual(len(data['training_requests']), 1)
+        self.assertEqual(data['training_requests'][0],
+                         expected['training_requests'][0])
+
+        # test expected TrainingProgress output
+        expected['training_progresses'] = [
+            {
+                # these are generated by Django, so we borrow them from the
+                # output
+                'created_at': data['training_progresses'][0]['created_at'],
+                'last_updated_at':
+                    data['training_progresses'][0]['last_updated_at'],
+                'requirement': {
+                    'name': 'Discussion',
+                    'url_required': False,
+                    'event_required': False,
+                },
+                'state': 'Passed',
+                'discarded': False,
+                'evaluated_by': None,
+                'event': {
+                    'slug': '2018-06-16-AMY-event',
+                    'start': '2018-06-16',
+                    'end': '2018-06-17',
+                    'tags': [],
+                    'website_url': 'http://example.org/2018-06-16-AMY-event',
+                    'venue': '',
+                    'address': '',
+                    'country': '',
+                    'latitude': None,
+                    'longitude': None,
+                },
+                'url': None,
+            },
+            {
+                # these are generated by Django, so we borrow them from the
+                # output
+                'created_at': data['training_progresses'][1]['created_at'],
+                'last_updated_at':
+                    data['training_progresses'][1]['last_updated_at'],
+                'requirement': {
+                    'name': 'DC Homework',
+                    'url_required': True,
+                    'event_required': False,
+                },
+                'state': 'Failed',
+                'discarded': False,
+                'evaluated_by': {
+                    'name': 'Super User',
+                },
+                'event': None,
+                'url': 'http://example.org/homework',
+            },
+        ]
+        self.assertEqual(len(data['training_progresses']), 2)
+        self.assertEqual(data['training_progresses'][0],
+                         expected['training_progresses'][0])
+        self.assertEqual(data['training_progresses'][1],
+                         expected['training_progresses'][1])