Repository: data-for-change/anyway

Issue:
 Improve system performance by using pagination between frontend and backend
Improve system performance by using pagination between Frontend and Backend. Perform the pagination so that accidents from the last years will appear first on the frontend. This way the responsiveness of UI will increase. 


Discussion:
@r-darwish Thanks for your contribution. Please see if this issue can be interesting for you.
I'll take a look
Should the pagination apply to both cluster view and markers view, or just the markers view?
Lets start with markers view only.
But this raises the question about the markers that contain more than one accident, on markers view.
Can this be done without having ugly refresh on the UI?
If so, do continue, otherwise lets discuss how can we create a nice User experience.
Thanks! 


Resolving patch:
diff --git a/config.py b/config.py
--- a/config.py
+++ b/config.py
@@ -14,6 +14,7 @@
 SQLALCHEMY_TRACK_MODIFICATIONS = True
 SENDGRID_USERNAME = os.environ.get('SENDGRID_USERNAME')
 SENDGRID_PASSWORD = os.environ.get('SENDGRID_PASSWORD')
+ENTRIES_PER_PAGE = os.environ.get('ENTRIES_PER_PAGE', 100)
 SQLALCHEMY_POOL_RECYCLE = 60
 
 
diff --git a/main.py b/main.py
--- a/main.py
+++ b/main.py
@@ -42,6 +42,7 @@
 from base import user_optional
 from models import (Marker, DiscussionMarker, HighlightPoint, Involved, User, ReportPreferences,
                      Vehicle, Role, GeneralPreferences)
+from config import ENTRIES_PER_PAGE
 
 
 app = utilities.init_flask(__name__)
@@ -91,12 +92,17 @@ def shutdown_session(exception=None):
     db_session.remove()
 
 
-def generate_json(accidents, discussions, is_thin):
+def generate_json(accidents, discussions, is_thin, total_records=None):
     markers = accidents.all()
+
     if not is_thin:
         markers += discussions.all()
+
+    if total_records is None:
+        total_records = len(accidents)
+
     entries = [ marker.serialize(is_thin) for marker in markers ]
-    return jsonify({"markers" : entries })
+    return jsonify({"markers" : entries , 'pagination': {'totalRecords': total_records}})
 
 
 def generate_csv(results):
@@ -126,7 +132,9 @@ def generate_csv(results):
              'start_time': (int, 25), 'end_time': (int, 25), 'weather': (int, 0), 'road': (int, 0),
              'separation': (int, 0), 'surface': (int, 0), 'acctype': (int, 0), 'controlmeasure': (int, 0),
              'district': (int, 0), 'case_type': (int, 0), 'fetch_markers': (bool, True), 'fetch_vehicles': (bool, True),
-             'fetch_involved': (bool, True)}
+             'fetch_involved': (bool, True),
+             'page': (int, 0),
+             'per_page': (int, 0)}
 
 def get_kwargs():
     kwargs = {arg: arg_type(request.values.get(arg, default_value)) for (arg, (arg_type, default_value)) in ARG_TYPES.iteritems()}
@@ -148,14 +156,14 @@ def markers():
     kwargs = get_kwargs()
     logging.debug('querying markers in bounding box: %s' % kwargs)
     is_thin = (kwargs['zoom'] < CONST.MINIMAL_ZOOM)
-    accidents = Marker.bounding_box_query(is_thin, yield_per=50, involved_and_vehicles=False, **kwargs)
+    result = Marker.bounding_box_query(is_thin, yield_per=50, involved_and_vehicles=False, **kwargs)
 
     discussion_args = ('ne_lat', 'ne_lng', 'sw_lat', 'sw_lng', 'show_discussions')
     discussions = DiscussionMarker.bounding_box_query(**{arg: kwargs[arg] for arg in discussion_args})
 
     if request.values.get('format') == 'csv':
         date_format = '%Y-%m-%d'
-        return Response(generate_csv(accidents), headers={
+        return Response(generate_csv(result.markers), headers={
             "Content-Type": "text/csv",
             "Content-Disposition": 'attachment; '
                                    'filename="Anyway-accidents-from-{0}-to-{1}.csv"'
@@ -163,7 +171,7 @@ def markers():
         })
 
     else: # defaults to json
-        return generate_json(accidents, discussions, is_thin)
+         return generate_json(result.markers, discussions, is_thin, total_records=result.total_records)
 
 
 @app.route("/charts-data", methods=["GET"])
@@ -381,6 +389,7 @@ def index(marker=None, message=None):
     today = datetime.date.today()
     context['default_end_date_format'] = request.values.get('end_date', today.strftime('%Y-%m-%d'))
     context['default_start_date_format'] = request.values.get('start_date', (today - datetime.timedelta(days=1095)).strftime('%Y-%m-%d'))
+    context['entries_per_page'] = ENTRIES_PER_PAGE
     return render_template('index.html', **context)
 
 
diff --git a/models.py b/models.py
--- a/models.py
+++ b/models.py
@@ -3,7 +3,7 @@
 import json
 import logging
 from constants import CONST
- 
+from collections import namedtuple
 from sqlalchemy import Column, Integer, String, Boolean, Float, ForeignKey, DateTime, Text, Index, desc, sql, Table, \
         ForeignKeyConstraint, func
 from sqlalchemy.orm import relationship, load_only, backref
@@ -13,6 +13,9 @@
 from database import Base, db_session
 from flask.ext.security import UserMixin, RoleMixin
 
+
+MarkerResult = namedtuple('MarkerResult', ['markers', 'total_records'])
+
 db_encoding = 'utf-8'
 
 logging.basicConfig(level=logging.DEBUG)
@@ -247,6 +250,8 @@ def bounding_box_query(is_thin=False, yield_per=None, involved_and_vehicles=Fals
 
         approx = kwargs.get('approx', True)
         accurate = kwargs.get('accurate', True)
+        page = kwargs.get('page')
+        per_page = kwargs.get('per_page')
 
         if not kwargs.get('show_markers', True):
             return Marker.query.filter(sql.false())
@@ -333,6 +338,10 @@ def bounding_box_query(is_thin=False, yield_per=None, involved_and_vehicles=Fals
         if is_thin:
             markers = markers.options(load_only("id", "longitude", "latitude"))
 
+        total_records = markers.count()
+        if page and per_page:
+            markers = markers.offset((page - 1 ) * per_page).limit(per_page)
+
         if involved_and_vehicles:
             fetch_markers = kwargs.get('fetch_markers', True)
             fetch_vehicles = kwargs.get('fetch_vehicles', True)
@@ -350,7 +359,7 @@ def bounding_box_query(is_thin=False, yield_per=None, involved_and_vehicles=Fals
             return markers.all() if markers is not None else [], vehicles.all() if vehicles is not None else [], \
                    involved.all() if involved is not None else []
         else:
-            return markers
+            return MarkerResult(markers=markers, total_records=total_records)
 
     @staticmethod
     def get_marker(marker_id):


Test patch:
diff --git a/tests/test_bounding_box_query.py b/tests/test_bounding_box_query.py
--- a/tests/test_bounding_box_query.py
+++ b/tests/test_bounding_box_query.py
@@ -26,7 +26,7 @@ def setUp(self):
                   'end_time': 25, 'road': 0, 'ne_lat': 32.072427482938345}
 
         self.query_args = kwargs
-        self.query = Marker.bounding_box_query(yield_per=50, **kwargs)
+        self.query = Marker.bounding_box_query(yield_per=50, **kwargs)[0]
 
     def tearDown(self):
         self.query = None
@@ -39,35 +39,35 @@ def test_location_filters(self):
     def test_accurate_filter(self):
         kwargs = self.query_args.copy()
         kwargs['approx'] = False
-        markers = Marker.bounding_box_query(yield_per=50, **kwargs)
+        markers = Marker.bounding_box_query(yield_per=50, **kwargs)[0]
         for marker in markers:
             self.assertTrue(marker.locationAccuracy == 1)
 
     def test_approx_filter(self):
         kwargs = self.query_args.copy()
         kwargs['accurate'] = False
-        markers = Marker.bounding_box_query(yield_per=50, **kwargs)
+        markers = Marker.bounding_box_query(yield_per=50, **kwargs)[0]
         for marker in markers:
             self.assertTrue(marker.locationAccuracy != 1)
 
     def test_fatal_severity_filter(self):
         kwargs = self.query_args.copy()
         kwargs['show_fatal'] = False
-        markers = Marker.bounding_box_query(yield_per=50, **kwargs)
+        markers = Marker.bounding_box_query(yield_per=50, **kwargs)[0]
         for marker in markers:
             self.assertTrue(marker.severity != 1)
     
     def test_severe_severity_filter(self):
         kwargs = self.query_args.copy()
         kwargs['show_severe'] = False
-        markers = Marker.bounding_box_query(yield_per=50, **kwargs)
+        markers = Marker.bounding_box_query(yield_per=50, **kwargs)[0]
         for marker in markers:
             self.assertTrue(marker.severity != 2)
 
     def test_light_severity_filter(self):
         kwargs = self.query_args.copy()
         kwargs['show_light'] = False
-        markers = Marker.bounding_box_query(yield_per=50, **kwargs)
+        markers = Marker.bounding_box_query(yield_per=50, **kwargs)[0]
         for marker in markers:
             self.assertTrue(marker.severity != 3)