Repository: uclapi/uclapi

Issue:
Oauth 500
Hi,

There were several people reporting issues to me when logging in with oauth and receiving a server error 500 message.

It works for most people using my app, I'm not sure what's the reason it fails for some.

If you need help digging up the log files, I can provide some confidential information (ie names, or ucl email addresses for failed accounts), please PM me in this case. 

![Screenshot](https://user-images.githubusercontent.com/5840038/64558272-c56c7600-d343-11e9-8b5c-7d4857a6201e.JPG)



Discussion:
The persons reporting 500s are most likely students enrolling in AY19/20. Since they have not fully matriculated yet, UCL is not yet passing us all the information we need.

We're working to figure out what information is missing, and to throw a more helpful error message (#1602). But for the time being, you might want to add a warning to your app.
OK, thanks

Resolving patch:
diff --git a/backend/uclapi/dashboard/views.py b/backend/uclapi/dashboard/views.py
--- a/backend/uclapi/dashboard/views.py
+++ b/backend/uclapi/dashboard/views.py
@@ -19,12 +19,46 @@
 
 @csrf_exempt
 def shibboleth_callback(request):
+    # should auth user login or signup
+    # then redirect to dashboard homepage
+
+    # Sometimes UCL doesn't give us the expected headers.
+    # If a critical header is missing we error out.
+    # If non-critical headers are missing we simply put a placeholder string.
+    try:
+        # This is used to find the correct user
+        eppn = request.META['HTTP_EPPN']
+        # We don't really use cn but because it's unique in the DB we can't
+        # really put a place holder value.
+        cn = request.META['HTTP_CN']
+        # (aka UPI), also unique in the DB
+        employee_id = request.META['HTTP_EMPLOYEEID']
+    except KeyError:
+        response = HttpResponse(
+            (
+
+                "Error 400 - Bad Request. <br>"
+                "UCL has sent incomplete headers. If the issues persist please"
+                "contact the UCL API Team to rectify this."
+            )
+        )
+        response.status_code = 400
+        return response
+
+    # TODO: Ask UCL what on earth are they doing by missing out headers, and
+    # remind them we need to to be informed of these types of changes.
+    # TODO: log to sentry that fields were missing...
+    department = request.META.get('HTTP_DEPARTMENT', '')
+    given_name = request.META.get('HTTP_GIVENNAME', '')
+    display_name = request.META.get('HTTP_DISPLAYNAME', '')
+    groups = request.META.get('HTTP_UCLINTRANETGROUPS', '')
+
     # We first check whether the user is a member of any UCL Intranet Groups.
     # This is a quick litmus test to determine whether they should have
     # access to the dashboard.
     # We deny access to test accounts and alumni, neither of which have
     # this Shibboleth attribute.
-    if 'HTTP_UCLINTRANETGROUPS' not in request.META:
+    if not groups:
         response = HttpResponse(
             (
                 "Error 403 - denied. <br>"
@@ -34,17 +68,10 @@ def shibboleth_callback(request):
         response.status_code = 403
         return response
 
-    # should auth user login or signup
-    # then redirect to dashboard homepage
-    eppn = request.META['HTTP_EPPN']
-    groups = request.META['HTTP_UCLINTRANETGROUPS']
-    cn = request.META['HTTP_CN']
-    department = request.META.get('HTTP_DEPARTMENT', '')
-    given_name = request.META['HTTP_GIVENNAME']
-    display_name = request.META['HTTP_DISPLAYNAME']
-    employee_id = request.META['HTTP_EMPLOYEEID']
-
     try:
+        # TODO: Handle MultipleObjectsReturned exception.
+        # email field isn't unique at database level (on our side).
+        # Alternatively, switch to employee_id (which is unique).
         user = User.objects.get(email=eppn)
     except ObjectDoesNotExist:
         # create a new user
@@ -63,14 +90,17 @@ def shibboleth_callback(request):
 
         request.session["user_id"] = new_user.id
     else:
-        # user exists already, update values
-        request.session["user_id"] = user.id
-        user.full_name = display_name
-        user.given_name = given_name
+        # User exists already, so update the values if new ones are non-empty.
+        user = User.objects.get(email=eppn)
+        user.employee_id = employee_id
+        if display_name:
+            user.full_name = display_name
+        if given_name:
+            user.given_name = given_name
         if department:
             user.department = department
-        user.raw_intranet_groups = groups
-        user.employee_id = employee_id
+        if groups:
+            user.raw_intranet_groups = groups
         user.save()
 
     return redirect(dashboard)
diff --git a/backend/uclapi/oauth/views.py b/backend/uclapi/oauth/views.py
--- a/backend/uclapi/oauth/views.py
+++ b/backend/uclapi/oauth/views.py
@@ -130,17 +130,33 @@ def shibcallback(request):
     # string sent via Shibboleth
     app = App.objects.get(client_id=client_id)
 
-    eppn = request.META['HTTP_EPPN']
-    cn = request.META['HTTP_CN']
-    # UCL's Shibboleth isn't passing us the department anymore...
-    # TODO: Ask UCL what on earth are they doing, and remind them we need to to
-    # be informed of these types of changes.
-    # TODO: Some of these fields are non-critical, think of defaults incase UCL
-    # starts removing/renaming more of these fields...
+    # Sometimes UCL doesn't give us the expected headers.
+    # If a critical header is missing we error out.
+    # If non-critical headers are missing we simply put a placeholder string.
+    try:
+        # This is used to find the correct user
+        eppn = request.META['HTTP_EPPN']
+        # We don't really use cn but because it's unique in the DB we can't
+        # really put a place holder value.
+        cn = request.META['HTTP_CN']
+        # (aka UPI), also unique in the DB
+        employee_id = request.META['HTTP_EMPLOYEEID']
+    except KeyError:
+        response = PrettyJsonResponse({
+            "ok": False,
+            "error": ("UCL has sent incomplete headers. If the issues persist"
+                      "please contact the UCL API Team to rectify this.")
+        })
+        response.status_code = 400
+        return response
+
+    # TODO: Ask UCL what on earth are they doing by missing out headers, and
+    # remind them we need to to be informed of these types of changes.
+    # TODO: log to sentry that fields were missing...
     department = request.META.get('HTTP_DEPARTMENT', '')
-    given_name = request.META['HTTP_GIVENNAME']
-    display_name = request.META['HTTP_DISPLAYNAME']
-    employee_id = request.META['HTTP_EMPLOYEEID']
+    given_name = request.META.get('HTTP_GIVENNAME', '')
+    display_name = request.META.get('HTTP_DISPLAYNAME', '')
+    groups = request.META.get('HTTP_UCLINTRANETGROUPS', '')
 
     # We check whether the user is a member of any UCL Intranet Groups.
     # This is a quick litmus test to determine whether they should be able to
@@ -149,9 +165,7 @@ def shibcallback(request):
     # Test accounts also do not have this attribute, but we can check the
     # department attribute for the Shibtests department.
     # This lets App Store reviewers log in to apps that use the UCL API.
-    if 'HTTP_UCLINTRANETGROUPS' in request.META:
-        groups = request.META['HTTP_UCLINTRANETGROUPS']
-    else:
+    if not groups:
         if department == "Shibtests" or eppn == SHIB_TEST_USER:
             groups = "shibtests"
         else:
@@ -166,6 +180,9 @@ def shibcallback(request):
 
     # If a user has never used the API before then we need to sign them up
     try:
+        # TODO: Handle MultipleObjectsReturned exception.
+        # email field isn't unique at database level (on our side).
+        # Alternatively, switch to employee_id (which is unique).
         user = User.objects.get(email=eppn)
     except User.DoesNotExist:
         # create a new user
@@ -181,14 +198,17 @@ def shibcallback(request):
 
         user.save()
     else:
-        # User exists already, so update the values
+        # User exists already, so update the values if new ones are non-empty.
         user = User.objects.get(email=eppn)
-        user.full_name = display_name
-        user.given_name = given_name
-        if department:  # UCL doesn't pass this anymore it seems...
-            user.department = department
-        user.raw_intranet_groups = groups
         user.employee_id = employee_id
+        if display_name:
+            user.full_name = display_name
+        if given_name:
+            user.given_name = given_name
+        if department:
+            user.department = department
+        if groups:
+            user.raw_intranet_groups = groups
         user.save()
 
     # Log the user into the system using their User ID
@@ -584,16 +604,40 @@ def get_student_number(request, *args, **kwargs):
 def myapps_shibboleth_callback(request):
     # should auth user login or signup
     # then redirect to my apps homepage
-    eppn = request.META['HTTP_EPPN']
-    groups = request.META['HTTP_UCLINTRANETGROUPS']
-    cn = request.META['HTTP_CN']
-    department = request.META['HTTP_DEPARTMENT']
-    given_name = request.META['HTTP_GIVENNAME']
-    display_name = request.META['HTTP_DISPLAYNAME']
-    employee_id = request.META['HTTP_EMPLOYEEID']
+
+    # Sometimes UCL doesn't give us the expected headers.
+    # If a critical header is missing we error out.
+    # If non-critical headers are missing we simply put a placeholder string.
+    try:
+        # This is used to find the correct user
+        eppn = request.META['HTTP_EPPN']
+        # We don't really use cn but because it's unique in the DB we can't
+        # really put a place holder value.
+        cn = request.META['HTTP_CN']
+        # (aka UPI), also unique in the DB
+        employee_id = request.META['HTTP_EMPLOYEEID']
+    except KeyError:
+        response = PrettyJsonResponse({
+            "ok": False,
+            "error": ("UCL has sent incomplete headers. If the issues persist"
+                      "please contact the UCL API Team to rectify this.")
+        })
+        response.status_code = 400
+        return response
+
+    # TODO: Ask UCL what on earth are they doing by missing out headers, and
+    # remind them we need to to be informed of these types of changes.
+    # TODO: log to sentry that fields were missing...
+    department = request.META.get('HTTP_DEPARTMENT', '')
+    given_name = request.META.get('HTTP_GIVENNAME', '')
+    display_name = request.META.get('HTTP_DISPLAYNAME', '')
+    groups = request.META.get('HTTP_UCLINTRANETGROUPS', '')
 
     try:
         user = User.objects.get(email=eppn)
+        # TODO: Handle MultipleObjectsReturned exception.
+        # email field isn't unique at database level (on our side).
+        # Alternatively, switch to employee_id (which is unique).
     except User.DoesNotExist:
         # create a new user
         new_user = User(
@@ -610,13 +654,17 @@ def myapps_shibboleth_callback(request):
 
         request.session["user_id"] = new_user.id
     else:
-        # user exists already, update values
-        request.session["user_id"] = user.id
-        user.full_name = display_name
-        user.given_name = given_name
-        user.department = department
-        user.raw_intranet_groups = groups
+        # User exists already, so update the values if new ones are non-empty.
+        user = User.objects.get(email=eppn)
         user.employee_id = employee_id
+        if display_name:
+            user.full_name = display_name
+        if given_name:
+            user.given_name = given_name
+        if department:
+            user.department = department
+        if groups:
+            user.raw_intranet_groups = groups
         user.save()
 
     return redirect("/oauth/myapps")


Test patch:
diff --git a/backend/uclapi/oauth/tests.py b/backend/uclapi/oauth/tests.py
--- a/backend/uclapi/oauth/tests.py
+++ b/backend/uclapi/oauth/tests.py
@@ -550,7 +550,8 @@ def test_valid_shibcallback_real_account(self):
             "Test New Name"
         )
 
-        # Now lets test for when UCL doesn't give us the department.
+        # Now lets test for when UCL doesn't give us department the department,
+        # givenname and displayname.
         response = self.client.get(
             '/oauth/shibcallback',
             {
@@ -558,9 +559,7 @@ def test_valid_shibcallback_real_account(self):
             },
             HTTP_EPPN='testxxx@ucl.ac.uk',
             HTTP_CN='testxxx',
-            # NOTE: missing HTTP_DEPARTMENT
-            HTTP_GIVENNAME='Test New Name',
-            HTTP_DISPLAYNAME='Test User',
+            # NOTE: missing HTTP_DEPARTMENT, HTTP_GIVENNAME, HTTP_DISPLAYNAME
             HTTP_EMPLOYEEID='xxxtest01',
             HTTP_UCLINTRANETGROUPS='ucl-all;ucl-tests-all'
         )
@@ -569,12 +568,67 @@ def test_valid_shibcallback_real_account(self):
         # Reload the test user from DB
         test_user_ = User.objects.get(id=test_user_.id)
 
-        # If no department is passed, we don't want to overwrite the previous
-        # value with an empty string.
+        # If a non-critical HTTP header is not passed, we don't want to
+        # overwrite the previous value with an empty string.
         self.assertEqual(
             test_user_.department,
             "Dept of Tests"
         )
+        self.assertEqual(
+            test_user_.given_name,
+            "Test New Name"
+        )
+        self.assertEqual(
+            test_user_.full_name,
+            "Test User"
+        )
+
+        # Now let's test when critical fields are missing
+        response = self.client.get(
+            '/oauth/shibcallback',
+            {
+                'appdata': signed_data
+            },
+            # NOTE: missing critical field eppn
+            HTTP_CN='testxxx',
+            HTTP_EMPLOYEEID='xxxtest01',
+            HTTP_DEPARTMENT='Dept of Tests',
+            HTTP_GIVENNAME='Test New Name',
+            HTTP_DISPLAYNAME='Test User',
+            HTTP_UCLINTRANETGROUPS='ucl-all;ucl-tests-all'
+        )
+        self.assertEqual(response.status_code, 400)
+
+        response = self.client.get(
+            '/oauth/shibcallback',
+            {
+                'appdata': signed_data
+            },
+            HTTP_EPPN='testxxx@ucl.ac.uk',
+            # NOTE: missing critical field cn
+            HTTP_DEPARTMENT='Dept of Tests',
+            HTTP_GIVENNAME='Test New Name',
+            HTTP_DISPLAYNAME='Test User',
+            HTTP_EMPLOYEEID='xxxtest01',
+            HTTP_UCLINTRANETGROUPS='ucl-all;ucl-tests-all'
+        )
+        self.assertEqual(response.status_code, 400)
+
+        response = self.client.get(
+            '/oauth/shibcallback',
+            {
+                'appdata': signed_data
+            },
+            HTTP_EPPN='testxxx@ucl.ac.uk',
+            HTTP_CN='testxxx',
+            # NOTE: missing critical field employee_id (aka UPI)
+            HTTP_DEPARTMENT='Dept of Tests',
+            HTTP_GIVENNAME='Test New Name',
+            HTTP_DISPLAYNAME='Test User',
+            HTTP_UCLINTRANETGROUPS='ucl-all;ucl-tests-all'
+        )
+        self.assertEqual(response.status_code, 400)
+
 
     def test_valid_shibcallback_test_account(self):
         dev_user_ = User.objects.create(