Repository: City-of-Helsinki/linkedevents

Issue:
Modify kulke importer to import courses



Discussion:

Resolving patch:
diff --git a/events/importer/base.py b/events/importer/base.py
--- a/events/importer/base.py
+++ b/events/importer/base.py
@@ -317,6 +317,27 @@ def info_make_link_id(info):
                     link_obj.save()
                 obj._changed = True
 
+        extension_data = info.get('extension_course')
+        if extension_data is not None:
+            from extension_course.models import Course
+
+            try:
+                course = obj.extension_course
+                course._changed = False
+                self._set_field(course, 'enrolment_start_time', extension_data['enrolment_start_time'])
+                self._set_field(course, 'enrolment_end_time', extension_data['enrolment_end_time'])
+                course_changed = course._changed
+            except Course.DoesNotExist:
+                Course.objects.create(
+                    event=obj,
+                    enrolment_start_time=extension_data['enrolment_start_time'],
+                    enrolment_end_time=extension_data['enrolment_end_time'],
+                )
+                course_changed = True
+
+            if course_changed:
+                obj._changed = True
+
         if obj._changed or obj._created:
             if obj._created:
                 verb = "created"
diff --git a/events/importer/kulke.py b/events/importer/kulke.py
--- a/events/importer/kulke.py
+++ b/events/importer/kulke.py
@@ -8,7 +8,7 @@
 from pytz import timezone
 from django.conf import settings
 from django.core.validators import URLValidator
-from django.core.exceptions import ValidationError, ObjectDoesNotExist
+from django.core.exceptions import ValidationError, ObjectDoesNotExist, ImproperlyConfigured
 from django.db import IntegrityError
 from django_orghierarchy.models import Organization
 
@@ -62,8 +62,9 @@
     53, 54, 55
 ]
 
-# These events are courses - not to be published
-SKIP_EVENTS_WITH_CATEGORY = {
+# Events having one of these categories are courses - they are excluded when importing events
+# and only they are included when importing courses.
+COURSE_CATEGORIES = {
     70, 71, 72, 73, 75, 77, 79, 80,
     81, 83, 84, 85, 87, 316, 629, 632,
     728, 729, 730, 735,
@@ -73,7 +74,7 @@
 def _query_courses():
     filter_out_keywords = map(
         make_kulke_id,
-        SKIP_EVENTS_WITH_CATEGORY
+        COURSE_CATEGORIES
     )
     return Event.objects.filter(
         data_source='kulke'
@@ -134,6 +135,9 @@ def _delete_courses():
     355: CHILDREN + YOUTH
 }
 
+# these are added to all courses
+COURSE_KEYWORDS = ('p9270',)
+
 # retain the above for simplicity, even if kulke importer internally requires full keyword ids
 KEYWORDS_TO_ADD_TO_AUDIENCE = ['yso:{}'.format(i) for i in KEYWORDS_TO_ADD_TO_AUDIENCE]
 
@@ -224,6 +228,9 @@ def setup(self):
 
         self.categories = categories
 
+        course_keyword_ids = ['yso:{}'.format(kw) for kw in COURSE_KEYWORDS]
+        self.course_keywords = set(Keyword.objects.filter(id__in=course_keyword_ids))
+
     def parse_kulke_categories(self):
         categories = {}
         categories_file = os.path.join(
@@ -300,7 +307,7 @@ def _html_format(text):
             formatted_paragraphs.append(formatted_paragraph)
         return ''.join(formatted_paragraphs)
 
-    def _import_event(self, lang, event_el, events):
+    def _import_event(self, lang, event_el, events, is_course=False):
         def text(t):
             return unicodetext(event_el.find('event' + t))
 
@@ -317,8 +324,8 @@ def text_content(k):
 
         eid = int(event_el.attrib['id'])
 
-        if text_content('servicecode') != 'Pelkkä ilmoitus':
-            # Skip courses
+        if text_content('servicecode') != 'Pelkkä ilmoitus' and not is_course:
+            # Skip courses when importing events
             return False
 
         if self.options['single']:
@@ -423,13 +430,15 @@ def text_content(k):
 
             event['end_time'] = end_time
 
-        # todo: verify enrolment use cases, proper fields
-        event['custom']['enrolment']['start_time'] = dateutil.parser.parse(
-            text('enrolmentstarttime')
-        )
-        event['custom']['enrolment']['end_time'] = dateutil.parser.parse(
-            text('enrolmentendtime')
-        )
+        if is_course:
+            event['extension_course'] = {
+                'enrolment_start_time': dateutil.parser.parse(
+                    text('enrolmentstarttime')
+                ),
+                'enrolment_end_time': dateutil.parser.parse(
+                    text('enrolmentendtime')
+                )
+            }
 
         if 'offers' not in event:
             event['offers'] = [recur_dict()]
@@ -471,6 +480,10 @@ def text_content(k):
                 except Keyword.DoesNotExist:
                     print('Could not find {}'.format(kulke_id))
 
+            if is_course:
+                event_keywords.update(self.course_keywords)
+                event_audience.update(self.course_keywords & set(KEYWORDS_TO_ADD_TO_AUDIENCE))
+
             event['keywords'] = event_keywords
             event['audience'] = event_audience
 
@@ -671,6 +684,15 @@ def _save_recurring_superevents(self, recurring_groups):
 
     def import_events(self):
         print("Importing Kulke events")
+        self._import_events()
+
+    def import_courses(self):
+        print("Importing Kulke courses")
+        if 'extension_course' not in settings.INSTALLED_APPS:
+            raise ImproperlyConfigured("Course extension must be installed when importing Kulke courses.")
+        self._import_events(importing_courses=True)
+
+    def _import_events(self, importing_courses=False):
         self.url_validator = URLValidator()
         events = recur_dict()
         recurring_groups = dict()
@@ -679,25 +701,20 @@ def import_events(self):
                 settings.IMPORT_FILE_PATH, 'kulke', 'events-%s.xml' % lang)
             root = etree.parse(events_file)
             for event_el in root.xpath('/eventdata/event'):
-                success = self._import_event(lang, event_el, events)
+                success = self._import_event(lang, event_el, events, importing_courses)
                 if success:
                     self._gather_recurring_events(lang, event_el, events, recurring_groups)
 
         events.default_factory = None
 
-        filter_out_keywords = set(map(
+        course_keywords = set(map(
             make_kulke_id,
-            SKIP_EVENTS_WITH_CATEGORY
+            COURSE_CATEGORIES,
         ))
-        for eid, event in events.items():
-            skip = False
-            for kw in event['keywords']:
-                if kw.id in filter_out_keywords:
-                    skip = True
-                    break
-            if skip:
-                continue
-            self.save_event(event)
+
+        for event in events.values():
+            if any(kw.id in course_keywords for kw in event['keywords']) == importing_courses:
+                self.save_event(event)
 
         self._verify_recurs(recurring_groups)
         aggregates = self._save_recurring_superevents(recurring_groups)
diff --git a/events/management/commands/event_import.py b/events/management/commands/event_import.py
--- a/events/management/commands/event_import.py
+++ b/events/management/commands/event_import.py
@@ -9,7 +9,7 @@
 class Command(BaseCommand):
     help = "Import event data"
 
-    importer_types = ['places', 'events', 'keywords']
+    importer_types = ['places', 'events', 'keywords', 'courses']
 
     def __init__(self):
         super().__init__()


Test patch: