Repository: carpentries/amy

Issue:
Import event information from event web site
The following workflow would make my life easier:
1.  Submit the URL of a workshop's GitHub website via a form.
2.  AMY gathers information from the workshop's `index.html` (just as we do for the main web site) and display it in an editable form.
3.  The admin can alter/delete/add values, then submit to create the event.

This would be useful because we're importing information about self-organized workshops with growing frequency.  The trickiest part will be to match instructor/helper names from the website to names in our database, but even if that's imperfect, this will give admins a head start.



Discussion:

Resolving patch:
diff --git a/workshops/urls.py b/workshops/urls.py
--- a/workshops/urls.py
+++ b/workshops/urls.py
@@ -35,6 +35,7 @@
     url(r'^event/(?P<event_ident>[\w-]+)/delete$', views.event_delete, name='event_delete'),
     url(r'^events/add/$', views.EventCreate.as_view(), name='event_add'),
     url(r'^event/(?P<event_ident>[\w-]+)/validate/?$', views.validate_event, name='validate_event'),
+    url(r'^events/import/?$', views.event_import, name='event_import'),
 
     url(r'^tasks/?$', views.all_tasks, name='all_tasks'),
     url(r'^task/(?P<task_id>\d+)/?$', views.task_details, name='task_details'),
diff --git a/workshops/util.py b/workshops/util.py
--- a/workshops/util.py
+++ b/workshops/util.py
@@ -1,12 +1,15 @@
 # coding: utf-8
-from math import pi, sin, cos, acos
 import csv
+from math import pi, sin, cos, acos
+import re
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import IntegrityError, transaction
 from django.core.paginator import Paginator as DjangoPaginator
+import requests
 
-from .models import Event, Role, Person, Task, Award
+from workshops.check import get_header
+from workshops.models import Event, Role, Person, Task, Award
 
 
 class InternalError(Exception):
@@ -339,3 +342,79 @@ def merge_persons(person_from, person_to):
 
     # removes tasks, awards, qualifications in a cascading way
     person_from.delete()
+
+
+class WrongEventURL(Exception):
+    pass
+
+
+def normalize_event_index_url(url):
+    """From any event URL, make one URL to the raw content.
+
+    For example:
+
+    * http://user.github.io/SLUG/
+    * http://user.github.io/SLUG/index.html
+    * https://github.com/user/SLUG/
+    * https://github.com/user/SLUG/blob/gh-pages/index.html
+    * https://raw.githubusercontent.com/user/SLUG/gh-pages/index.html
+
+    …will become:
+    https://raw.githubusercontent.com/user/SLUG/gh-pages/index.html
+    """
+    template = ('https://raw.githubusercontent.com/{username}/{slug}'
+                '/gh-pages/index.html')
+    FMT = [
+        r'https?://(?P<name>[^\.]+)\.github\.io/(?P<repo>[^/]+)',
+        r'https?://(?P<name>[^\.]+)\.github\.io/(?P<repo>[^/]+)/index\.html',
+        r'https://github\.com/(?P<name>[^/]+)/(?P<repo>[^/]+)',
+        (r'https://github\.com/(?P<name>[^/]+)/(?P<repo>[^/]+)/'
+         r'blob/gh-pages/index\.html'),
+        (r'https://raw.githubusercontent.com/(?P<name>[^/]+)/(?P<repo>\S+)'
+         r'/gh-pages/index.html'),
+    ]
+    for format in FMT:
+        results = re.findall(format, url)
+        if results:
+            username, slug = results[0]
+            # caution: if groups in URL change order, then the formatting
+            # below will be broken, because it relies on re.findall() output,
+            # which is a tuple (:sad:)
+            return template.format(username=username, slug=slug), slug
+
+    raise WrongEventURL("This event URL is incorrect: {0}".format(url))
+
+
+def parse_tags_from_event_index(orig_url):
+    url, slug = normalize_event_index_url(orig_url)
+    response = requests.get(url)
+
+    # will throw requests.exceptions.HTTPError if status is not OK
+    response.raise_for_status()
+
+    _, headers = get_header(response.text)
+
+    # put instructors, helpers and venue into notes
+    notes = """VENUE: {venue}, {address}, {country}
+
+INSTRUCTORS: {instructors}
+
+HELPERS: {helpers}
+
+CONTACT: {contact}""".format(
+        venue=headers.get('venue', ''),
+        address=headers.get('address', ''),
+        country=headers.get('country', ''),
+        instructors=", ".join(headers.get('instructor', '')),
+        helpers=", ".join(headers.get('helper', '')),
+        contact=headers.get('contact', ''),
+    )
+
+    return {
+        'slug': slug,
+        'start': headers.get('startdate', ''),
+        'end': headers.get('enddate', ''),
+        'url': orig_url,
+        'reg_key': headers.get('eventbrite', ''),
+        'notes': notes,
+    }
diff --git a/workshops/views.py b/workshops/views.py
--- a/workshops/views.py
+++ b/workshops/views.py
@@ -11,13 +11,18 @@
 from django.contrib.auth.forms import SetPasswordForm, PasswordChangeForm
 from django.core.paginator import EmptyPage, PageNotAnInteger
 from django.core.urlresolvers import reverse
-from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
+from django.core.exceptions import (
+    ObjectDoesNotExist,
+    PermissionDenied,
+    SuspiciousOperation,
+)
 from django.conf import settings
-from django.http import Http404, HttpResponse
+from django.http import Http404, HttpResponse, JsonResponse
 from django.db import IntegrityError, transaction
 from django.db.models import Count, Sum, Q, F, Model, ProtectedError
 from django.db.models import Case, When, Value, IntegerField
 from django.shortcuts import redirect, render, get_object_or_404
+from django.views.decorators.http import require_POST
 from django.views.generic.base import ContextMixin
 from django.views.generic.edit import CreateView, UpdateView
 from django.contrib.auth.decorators import login_required
@@ -46,7 +51,8 @@
 )
 from workshops.util import (
     upload_person_task_csv,  verify_upload_person_task,
-    create_uploaded_persons_tasks, InternalError, Paginator, merge_persons
+    create_uploaded_persons_tasks, InternalError, Paginator, merge_persons,
+    normalize_event_index_url, WrongEventURL, parse_tags_from_event_index
 )
 
 from workshops.filters import (
@@ -710,21 +716,24 @@ def validate_event(request, event_ident):
     '''Check the event's home page *or* the specified URL (for testing).'''
     page_url, error_messages = None, []
     event = Event.get_by_ident(event_ident)
-    github_url = request.GET.get('url', None) # for manual override
+    github_url = request.GET.get('url', None)  # for manual override
     if github_url is None:
         github_url = event.url
-    if github_url is not None:
-        page_url = github_url.replace('github.com', 'raw.githubusercontent.com').rstrip('/') + '/gh-pages/index.html'
 
-        try:
-            response = requests.get(page_url)
+    try:
+        page_url, _ = normalize_event_index_url(github_url)
+        response = requests.get(page_url)
+
+        if response.status_code != 200:
+            error_messages.append('Request for {0} returned status code {1}'
+                                  .format(page_url, response.status_code))
+        else:
+            error_messages = check_file(page_url, response.text)
+    except WrongEventURL:
+        error_messages = ["This is not a proper event URL.", ]
+    except requests.ConnectionError:
+        error_messages = ["Network connection error.", ]
 
-            if response.status_code != 200:
-                error_messages.append('Request for {0} returned status code {1}'.format(page_url, response.status_code))
-            else:
-                error_messages = check_file(page_url, response.text)
-        except requests.ConnectionError:
-            error_messages = ["Network connection error.", ]
     context = {'title' : 'Validate Event {0}'.format(event),
                'event' : event,
                'page' : page_url,
@@ -735,7 +744,7 @@ def validate_event(request, event_ident):
 class EventCreate(LoginRequiredMixin, CreateViewContext):
     model = Event
     form_class = EventForm
-    template_name = 'workshops/generic_form.html'
+    template_name = 'workshops/event_create_form.html'
 
 
 @login_required
@@ -819,6 +828,21 @@ def event_delete(request, event_ident):
     except ProtectedError as e:
         return _failed_to_delete(request, event, e.protected_objects)
 
+
+@login_required
+@require_POST
+def event_import(request):
+    """Read tags from remote URL and return them as JSON.
+
+    This is used to read tags from workshop index page and then fill up fields
+    on event_create form."""
+    try:
+        url = request.POST['url']
+        translated_data = parse_tags_from_event_index(url)
+        return JsonResponse(translated_data)
+    except (KeyError, WrongEventURL):
+        raise SuspiciousOperation('Missing or wrong `url` POST parameter.')
+
 #------------------------------------------------------------
 
 @login_required


Test patch:
diff --git a/workshops/test/test_util.py b/workshops/test/test_util.py
--- a/workshops/test/test_util.py
+++ b/workshops/test/test_util.py
@@ -1,16 +1,23 @@
 # coding: utf-8
 import cgi
-from datetime import datetime
+from datetime import datetime, date
 from io import StringIO
 from importlib import import_module
+from unittest.mock import patch
 
 from django.conf import settings
 from django.contrib.sessions.serializers import JSONSerializer
 from django.test import TestCase
 from django.core.urlresolvers import reverse
+import requests
 
 from ..models import Site, Event, Role, Person, Task
-from ..util import upload_person_task_csv, verify_upload_person_task
+from ..util import (
+    upload_person_task_csv,
+    verify_upload_person_task,
+    normalize_event_index_url,
+    parse_tags_from_event_index,
+)
 
 from .base import TestBase
 
@@ -321,3 +328,106 @@ def test_upload_existing_user_existing_task(self):
         self.assertEqual(tasks_pre, tasks_post)
         self.assertEqual(users_pre, users_post)
         self.assertEqual(rv.status_code, 200)
+
+
+class TestEventURLNormalization(TestCase):
+    def setUp(self):
+        self.test_cases = [
+            'http://user-name.github.io/2015-07-13-City/',
+            'https://user-name.github.io/2015-07-13-City/',
+            'http://user-name.github.io/2015-07-13-City',
+            'https://user-name.github.io/2015-07-13-City',
+            'http://user-name.github.io/2015-07-13-City/index.html',
+            'https://github.com/user-name/2015-07-13-City/',
+            'https://github.com/user-name/2015-07-13-City',
+            ('https://github.com/user-name/2015-07-13-City/blob/'
+             'gh-pages/index.html'),
+        ]
+
+        self.output = ('https://raw.githubusercontent.com/user-name/'
+                       '2015-07-13-City/gh-pages/index.html')
+
+    def test_normalization(self):
+        for url in self.test_cases:
+            assert normalize_event_index_url(url)[0] == self.output
+
+
+class TestParsingEventHeaders(TestCase):
+
+    # Response.status_code apparently doesn't exist by default
+    @patch.object(requests.models.Response, 'status_code', 404, create=True)
+    def test_wrong_url(self):
+        with self.assertRaises(requests.exceptions.HTTPError):
+            url = 'http://test.github.io/2015-07-13-test/'
+            parse_tags_from_event_index(url)
+
+    @patch.object(requests, 'get')
+    def test_parsing_event_index(self, mock_get):
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.text = """---
+layout: workshop
+root: .
+venue: Euphoric State University
+address: Highway to Heaven 42, Academipolis
+country: USA
+language: us
+latlng: 36.998977, -109.045173
+humandate: Jul 13-14, 2015
+humantime: 9:00 - 17:00
+startdate: 2015-07-13
+enddate: 2015-07-14
+instructor: ["Hermione Granger", "Harry Potter", "Ron Weasley",]
+helper: ["Peter Parker", "Tony Stark", "Natasha Romanova",]
+contact: hermione@granger.co.uk, rweasley@ministry.gov.uk
+etherpad:
+eventbrite: 10000000
+---
+"""
+        url = 'http://test.github.io/2015-07-13-test/'
+        notes = """VENUE: Euphoric State University, Highway to Heaven 42, Academipolis, USA
+
+INSTRUCTORS: Hermione Granger, Harry Potter, Ron Weasley
+
+HELPERS: Peter Parker, Tony Stark, Natasha Romanova
+
+CONTACT: hermione@granger.co.uk, rweasley@ministry.gov.uk"""
+
+        expected = {
+            'slug': '2015-07-13-test',
+            'start': date(2015, 7, 13),
+            'end': date(2015, 7, 14),
+            'url': url,
+            'reg_key': 1e7,
+            'notes': notes,
+        }
+
+        self.assertEqual(parse_tags_from_event_index(url), expected)
+
+    @patch.object(requests, 'get')
+    def test_parsing_malformed_file(self, mock_get):
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.text = """---
+venues: Euphoric State University
+addresses: Highway to Heaven 42, Academipolis
+countries: USA
+startdates: 2015-07-13
+enddates: 2015-07-14
+instructors: ["Hermione Granger", "Harry Potter", "Ron Weasley",]
+helpers: ["Peter Parker", "Tony Stark", "Natasha Romanova",]
+contacts: hermione@granger.co.uk, rweasley@ministry.gov.uk
+eventbrites: 10000000
+---
+"""
+        url = 'http://test.github.io/2015-07-13-test/'
+        notes = "VENUE: , , \n\nINSTRUCTORS: \n\nHELPERS: \n\nCONTACT: "
+
+        expected = {
+            'slug': '2015-07-13-test',
+            'start': '',
+            'end': '',
+            'url': url,
+            'reg_key': '',
+            'notes': notes,
+        }
+
+        self.assertEqual(parse_tags_from_event_index(url), expected)