Repository: ifrcgo/go-api
Issue: Only events are coming out of ES endpoint

Figure out why appeals and field reports aren't appearing in search.

Changed files:
main/urls.py, api/management/commands/index_elasticsearch.py, api/models.py, api/views.py, api/indexes.py

diff --git a/api/indexes.py b/api/indexes.py
index 50a7c1f5..0f5d9dbb 100644
--- a/api/indexes.py
+++ b/api/indexes.py
@@ -1,14 +1,28 @@
-PageIndexMapping = {
+EventPageMapping = {
+    'properties': {
+        'id': {'type': 'keyword'},
+        'name': {'type': 'text'},
+        'dtype': {'type': 'text'},
+        'location': {'type': 'text'},
+        'summary': {'type': 'text'},
+    }
+}
+
+AppealPageMapping = {
+    'properties': {
+        'id': {'type': 'keyword'},
+        'name': {'type': 'text'},
+        'dtype': {'type': 'text'},
+        'location': {'type': 'text'},
+    }
+}
+
+ReportPageMapping = {
     'properties': {
         'id': {'type': 'text'},
         'name': {'type': 'text'},
-        'type': {'type': 'keyword'},
-        'countries': {'type': 'keyword'},
-        'dtype': {'type': 'keyword'},
+        'dtype': {'type': 'text'},
+        'location': {'type': 'text'},
         'summary': {'type': 'text'},
-        'status': {'type': 'keyword'},
-        'created_at': {'type': 'date'},
-        'start_date': {'type': 'date'},
-        'end_date': {'type': 'date'},
     }
 }
diff --git a/api/management/commands/index_elasticsearch.py b/api/management/commands/index_elasticsearch.py
index bea3d991..9cc937ab 100644
--- a/api/management/commands/index_elasticsearch.py
+++ b/api/management/commands/index_elasticsearch.py
@@ -5,51 +5,56 @@ from elasticsearch.helpers import bulk
 from elasticsearch import Elasticsearch
 
 from api.esconnection import ES_CLIENT
-from api.indexes import PageIndexMapping
+from api.indexes import EventPageMapping, AppealPageMapping, ReportPageMapping
 from api.models import Event, Appeal, FieldReport
 
 class Command(BaseCommand):
     help = 'Create a new elasticsearch index and bulk-index existing objects'
 
     def handle(self, *args, **options):
-        print('Recreating pages index')
-        self.recreate_index()
+        print('Recreating indices')
+        self.recreate_index('page_event', EventPageMapping)
+        self.recreate_index('page_appeal', AppealPageMapping)
+        self.recreate_index('page_report', ReportPageMapping)
 
         print('Indexing events')
-        self.push_table_to_index(model=Event)
+        self.push_table_to_index(index='page_event', model=Event)
 
         print('Indexing appeals')
-        self.push_table_to_index(model=Appeal)
+        self.push_table_to_index(index='page_appeal', model=Appeal)
 
         print('Indexing field reports')
-        self.push_table_to_index(model=FieldReport)
+        self.push_table_to_index(index='page_report', model=FieldReport)
 
-    def recreate_index(self):
+    def recreate_index(self, index_name, index_mapping):
         indices_client = IndicesClient(client=ES_CLIENT)
-        index_name = 'pages'
         if indices_client.exists(index_name):
             indices_client.delete(index=index_name)
         indices_client.create(index=index_name)
         indices_client.put_mapping(doc_type='page',
                                    index=index_name,
-                                   body=PageIndexMapping)
+                                   body=index_mapping)
 
 
-    def push_table_to_index(self, model):
+    def push_table_to_index(self, index, model):
+        print('Indexing %s models' % model.objects.all().count())
+        query = model.objects.all()
         data = [
-            self.convert_for_bulk(s) for s in model.objects.all()
+            self.convert_for_bulk(index, s) for s in list(query)
         ]
+        #print(data)
         created, errors = bulk(client=ES_CLIENT, actions=data)
         print('Created %s records' % created)
         if len(errors):
             print('Produced the following errors:')
             print('[%s]' % ', '.join(map(str, errors)))
 
-    def convert_for_bulk(self, model_object):
+    def convert_for_bulk(self, index, model_object):
         data = model_object.indexing()
+        print(data)
         metadata = {
             '_op_type': 'create',
-            '_index': 'pages',
+            '_index': index,
             '_type': 'page',
             '_id': model_object.es_id(),
         }
diff --git a/api/models.py b/api/models.py
index bba6303e..1a6cde70 100644
--- a/api/models.py
+++ b/api/models.py
@@ -69,22 +69,20 @@ class Event(models.Model):
 
     def indexing(self):
         countries = [getattr(c, 'name') for c in self.countries.all()]
-        obj = {
+        return {
             'id': self.id,
             'name': self.name,
-            'type': 'event',
-            'countries': ','.join(map(str, countries)) if len(countries) else None,
             'dtype': getattr(self.dtype, 'name', None),
+            'location': ', '.join(map(str, countries)) if len(countries) else None,
             'summary': self.summary,
-            'created_at': self.created_at,
-            'start_date': self.start_date(),
-            'end_date': self.end_date(),
         }
-        return obj
 
     def es_id(self):
         return 'event-%s' % self.id
 
+    def es_index(self):
+        return 'page_event'
+
     def save(self, *args, **kwargs):
         # On save, if `disaster_start_date` is not set, make it the current time
         if not self.id and not self.disaster_start_date:
@@ -155,20 +153,19 @@ class Appeal(models.Model):
     region = models.ForeignKey(Region, null=True)
 
     def indexing(self):
-        obj = {
+        return {
             'id': self.aid,
-            'type': 'appeal',
-            'countries': getattr(self.country, 'name', None),
-            'created_at': self.created_at,
-            'start_date': self.start_date,
-            'status': self.status,
-            'end_date': self.end_date,
+            'name': self.name,
+            'dtype': getattr(self.dtype, 'name', None),
+            'location': getattr(self.country, 'name', None),
         }
-        return obj
 
     def es_id(self):
         return 'appeal-%s' % self.id
 
+    def es_index(self):
+        return 'page_appeal'
+
     def __str__(self):
         return self.aid
 
@@ -270,20 +267,20 @@ class FieldReport(models.Model):
 
     def indexing(self):
         countries = [getattr(c, 'name') for c in self.countries.all()]
-        obj = {
+        return {
             'id': self.rid,
-            'type': 'fieldreport',
-            'countries': ','.join(map(str, countries)) if len(countries) else None,
+            'name': self.summary,
             'dtype': getattr(self.dtype, 'name', None),
-            'summary': self.summary,
-            'status': self.status,
-            'created_at': self.created_at,
+            'location': ','.join(map(str, countries)) if len(countries) else None,
+            'summary': self.description,
         }
-        return obj
 
     def es_id(self):
         return 'fieldreport-%s' % self.id
 
+    def es_index(self):
+        return 'page_report'
+
     def __str__(self):
         return self.rid
 
@@ -347,7 +344,7 @@ post_save.connect(create_profile, sender=settings.AUTH_USER_MODEL)
 def index_es(sender, instance, created, **kwargs):
     if ES_CLIENT is not None:
         ES_CLIENT.index(
-            index='pages',
+            index=instance.es_index(),
             doc_type='page',
             id=instance.es_id(),
             body=instance.indexing(),
diff --git a/api/views.py b/api/views.py
index d69d3af0..7bd94f8c 100644
--- a/api/views.py
+++ b/api/views.py
@@ -42,28 +42,25 @@ class PublicJsonRequestView(View):
         return self.handle_get(request, *args, **kwargs)
 
 
-class es_keyword_search(PublicJsonRequestView):
+class es_page_search(PublicJsonRequestView):
     def handle_get(self, request, *args, **kwargs):
-        object_type = request.GET.get('type', None)
-        keyword = request.GET.get('keyword', None)
-
-        if keyword is None:
+        page_type = request.GET.get('type', None)
+        phrase = request.GET.get('keyword', None)
+        if phrase is None:
             return bad_request('Must include a `keyword`')
-
+        if page_type is None:
+            page_type = '*'
+        index = 'page_%s' % page_type
         query = {
             'bool': {
-                'must': {'prefix': {'name': keyword }}
+                'must': {'prefix': {'name': phrase }}
             }
         }
-        if object_type is not None:
-            query['bool']['filter'] = {'term': {'type' : object_type}}
-
         results = ES_CLIENT.search(
-            index='pages',
+            index=index,
             doc_type='page',
             body=json.dumps({'query': query}),
         )
-
         return JsonResponse(results['hits'])
 
 
diff --git a/main/urls.py b/main/urls.py
index c7ad7548..dbc4738b 100644
--- a/main/urls.py
+++ b/main/urls.py
@@ -31,7 +31,7 @@ from api.resources import (
 from notifications.resources import SurgeAlertResource
 from api.views import (
     get_auth_token,
-    es_keyword_search,
+    es_page_search,
     aggregate_by_time,
     update_subscription_preferences,
 )
@@ -54,7 +54,7 @@ v1_api.register(SurgeAlertResource())
 urlpatterns = [
     url(r'^admin/', admin.site.urls),
     url(r'^api/', include(v1_api.urls)),
-    url(r'^api/v1/es_search/', es_keyword_search.as_view()),
+    url(r'^api/v1/es_search/', es_page_search.as_view()),
     url(r'^api/v1/graphql/', GraphQLView.as_view(graphiql=True)),
     url(r'^api/v1/aggregate/', aggregate_by_time.as_view()),
     url(r'^get_auth_token', get_auth_token.as_view()),