Repository: data-for-change/anyway

Issue:
Don't use cache for infographics data when testing locally
In function get_infographics_data in infographics_utils, check FLASK_ENV var and if it's "development", call create_infographics_data(news_flash_id,years_ago) instead of using cache.

Motivation: we cannot test modification to existing APIs if cache is used


Discussion:

Resolving patch:
diff --git a/anyway/infographics_utils.py b/anyway/infographics_utils.py
--- a/anyway/infographics_utils.py
+++ b/anyway/infographics_utils.py
@@ -2,6 +2,7 @@
 import logging
 import datetime
 import json
+import os
 from functools import lru_cache
 
 import pandas as pd
@@ -110,7 +111,7 @@ def get_accident_count_by_accident_type(location_info, start_time, end_time):
 
 
 def get_top_road_segments_accidents_per_km(
-        resolution, location_info, start_time=None, end_time=None, limit=5
+    resolution, location_info, start_time=None, end_time=None, limit=5
 ):
     if resolution != "כביש בינעירוני":  # relevent for non urban roads only
         return {}
@@ -126,19 +127,19 @@ def get_top_road_segments_accidents_per_km(
             (RoadSegments.to_km - RoadSegments.from_km).label("segment_length"),
             cast(
                 (
-                        func.count(AccidentMarkerView.road_segment_name)
-                        / (RoadSegments.to_km - RoadSegments.from_km)
+                    func.count(AccidentMarkerView.road_segment_name)
+                    / (RoadSegments.to_km - RoadSegments.from_km)
                 ),
                 Numeric(10, 4),
             ).label("accidents_per_km"),
         )
-            .filter(AccidentMarkerView.road1 == RoadSegments.road)
-            .filter(AccidentMarkerView.road_segment_number == RoadSegments.segment)
-            .filter(AccidentMarkerView.road1 == location_info["road1"])
-            .filter(AccidentMarkerView.road_segment_name is not None)
-            .group_by(AccidentMarkerView.road_segment_name, RoadSegments.from_km, RoadSegments.to_km)
-            .order_by(desc("accidents_per_km"))
-            .limit(limit)
+        .filter(AccidentMarkerView.road1 == RoadSegments.road)
+        .filter(AccidentMarkerView.road_segment_number == RoadSegments.segment)
+        .filter(AccidentMarkerView.road1 == location_info["road1"])
+        .filter(AccidentMarkerView.road_segment_name is not None)
+        .group_by(AccidentMarkerView.road_segment_name, RoadSegments.from_km, RoadSegments.to_km)
+        .order_by(desc("accidents_per_km"))
+        .limit(limit)
     )
 
     result = pd.read_sql_query(query.statement, query.session.bind)
@@ -146,7 +147,7 @@ def get_top_road_segments_accidents_per_km(
 
 
 def get_accidents_stats(
-        table_obj, filters=None, group_by=None, count=None, start_time=None, end_time=None
+    table_obj, filters=None, group_by=None, count=None, start_time=None, end_time=None
 ):
     filters = filters or {}
     filters["provider_code"] = [
@@ -179,7 +180,7 @@ def get_injured_filters(location_info):
 
 
 def get_most_severe_accidents_with_entities(
-        table_obj, filters, entities, start_time, end_time, limit=10
+    table_obj, filters, entities, start_time, end_time, limit=10
 ):
     filters = filters or {}
     filters["provider_code"] = [
@@ -255,8 +256,8 @@ def filter_and_group_injured_count_per_age_group(data_of_ages):
         for item in range_dict.items():
             item_min_range, item_max_range = item
             if (
-                    item_min_range <= min_age <= item_max_range
-                    and item_min_range <= max_age <= item_max_range
+                item_min_range <= min_age <= item_max_range
+                and item_min_range <= max_age <= item_max_range
             ):
                 string_age_range = f"{item_min_range:02}-{item_max_range:02}"
                 dict_by_required_age_group[string_age_range] += count
@@ -330,7 +331,7 @@ def get_most_severe_accidents_table(location_info, start_time, end_time):
         )
         # TODO: remove injured_count after FE adaptation to light and severe counts
         accident["injured_count"] = (
-                accident["severe_injured_count"] + accident["light_injured_count"]
+            accident["severe_injured_count"] + accident["light_injured_count"]
         )
         del (
             accident["accident_timestamp"],
@@ -350,8 +351,8 @@ def count_accidents_by_driver_type(data):
         elif vehicle_type in BE_CONST.PRIVATE_DRIVER_VEHICLE_TYPES:
             driver_types[driver_type_hebrew_dict["private_vehicle_driver"]] += count
         elif (
-                vehicle_type in BE_CONST.LIGHT_ELECTRIC_VEHICLE_TYPES
-                or vehicle_type in BE_CONST.OTHER_VEHICLES_TYPES
+            vehicle_type in BE_CONST.LIGHT_ELECTRIC_VEHICLE_TYPES
+            or vehicle_type in BE_CONST.OTHER_VEHICLES_TYPES
         ):
             driver_types[driver_type_hebrew_dict["other_driver"]] += count
     output = []
@@ -440,7 +441,9 @@ def convert_roads_fatal_accidents_to_frontend_view(data_dict):
     data_list = []
     for key, value in data_dict.items():
         if key == head_on_collisions_comparison_dict["head_to_head_collision"]:
-            data_list.append({"desc": head_on_collisions_comparison_dict["head_to_head"], "count": value})
+            data_list.append(
+                {"desc": head_on_collisions_comparison_dict["head_to_head"], "count": value}
+            )
         else:
             data_list.append({"desc": key, "count": value})
 
@@ -538,7 +541,7 @@ def percentage_accidents_by_car_type_national_data_cache(start_time, end_time):
 
 
 def stats_accidents_by_car_type_with_national_data(
-        involved_by_vehicle_type_data, start_time, end_time
+    involved_by_vehicle_type_data, start_time, end_time
 ):
     out = []
     data_by_segment = percentage_accidents_by_car_type(involved_by_vehicle_type_data)
@@ -800,55 +803,71 @@ def create_infographics_data(news_flash_id, number_of_years_ago):
     output["widgets"].append(accident_count_by_car_type.serialize())
 
     def injured_accidents_with_pedestrians_mock_data():  # Temporary for Frontend
-        return [{'year': 2009,
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 12,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 3,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 0 },
-                {'year': 2010,
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 24,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 0,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 1 },
-                {'year': 2011,
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 9,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 2,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 1 },
-                {'year': 2012,
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 21,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 2,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 4 },
-                {'year': 2013,
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 21,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 2,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 4 },
-                {'year': 2014,
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 10,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 0,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 1 },
-                {'year': 2015,
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 13,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 2,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 0 }]
+        return [
+            {
+                "year": 2009,
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 12,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 3,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 0,
+            },
+            {
+                "year": 2010,
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 24,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 0,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 1,
+            },
+            {
+                "year": 2011,
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 9,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 2,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 1,
+            },
+            {
+                "year": 2012,
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 21,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 2,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 4,
+            },
+            {
+                "year": 2013,
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 21,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 2,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 4,
+            },
+            {
+                "year": 2014,
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 10,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 0,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 1,
+            },
+            {
+                "year": 2015,
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 13,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 2,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 0,
+            },
+        ]
 
     injured_accidents_with_pedestrians = Widget(
         name="injured_accidents_with_pedestrians",
@@ -859,51 +878,42 @@ def injured_accidents_with_pedestrians_mock_data():  # Temporary for Frontend
     output["widgets"].append(injured_accidents_with_pedestrians.serialize())
 
     def accident_severity_by_cross_location_mock_data():  # Temporary for Frontend
-        return [{'cross_location_text': "במעבר חצייה",
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 37,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 6,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 0 },
-                {'cross_location_text': "לא במעבר חצייה",
-                 'light_injury_severity_text': "פצוע קל",
-                 'light_injury_severity_count': 11,
-                 'severe_injury_severity_text': "פצוע קשה",
-                 'severe_injury_severity_count': 10,
-                 'killed_injury_severity_text': "הרוג",
-                 'killed_injury_severity_count': 0 }]
+        return [
+            {
+                "cross_location_text": "במעבר חצייה",
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 37,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 6,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 0,
+            },
+            {
+                "cross_location_text": "לא במעבר חצייה",
+                "light_injury_severity_text": "פצוע קל",
+                "light_injury_severity_count": 11,
+                "severe_injury_severity_text": "פצוע קשה",
+                "severe_injury_severity_count": 10,
+                "killed_injury_severity_text": "הרוג",
+                "killed_injury_severity_count": 0,
+            },
+        ]
 
     accident_severity_by_cross_location = Widget(
         name="accident_severity_by_cross_location",
         rank=19,
         items=accident_severity_by_cross_location_mock_data(),
-        text={"comment": "בן יהודה תל אביב בין השנים 2008-2020"}
+        text={"comment": "בן יהודה תל אביב בין השנים 2008-2020"},
     )
     output["widgets"].append(accident_severity_by_cross_location.serialize())
 
     def motorcycle_accidents_vs_all_accidents_mock_data():  # Temporary for Frontend
-        return [{
-                "location": "כביש 20",
-                "vehicle": "אופנוע",
-                "percentage": 0.5
-                },
-                {
-                "location": "כביש 20",
-                "vehicle": "אחר",
-                "percentage": 0.5
-                },
-                {
-                "location": "כל הארץ",
-                "vehicle": "אחר",
-                "percentage": 0.802680566
-                },
-                {
-                "location": "כל הארץ",
-                "vehicle": "אופנוע",
-                "percentage": 0.197319434
-                }
-                ]
+        return [
+            {"location": "כביש 20", "vehicle": "אופנוע", "percentage": 0.5},
+            {"location": "כביש 20", "vehicle": "אחר", "percentage": 0.5},
+            {"location": "כל הארץ", "vehicle": "אחר", "percentage": 0.802680566},
+            {"location": "כל הארץ", "vehicle": "אופנוע", "percentage": 0.197319434},
+        ]
 
     motorcycle_accidents_vs_all_accidents = Widget(
         name="motorcycle_accidents_vs_all_accidents",
@@ -914,72 +924,45 @@ def motorcycle_accidents_vs_all_accidents_mock_data():  # Temporary for Frontend
     output["widgets"].append(motorcycle_accidents_vs_all_accidents.serialize())
 
     def accidents_count_pedestrians_per_vehicle_street_vs_all_mock_data():  # Temporary for Frontend
-        return [{
-                "location": "כל הארץ",
-                "vehicle": "מכונית",
-                "num_of_accidents": 61307
-                },
-                {
-                "location": "כל הארץ",
-                "vehicle": "רכב כבד",
-                "num_of_accidents": 15801
-                },
-                {
-                "location": "כל הארץ",
-                "vehicle": "אופנוע",
-                "num_of_accidents": 3884
-                },
-                {
-                "location": "כל הארץ",
-                "vehicle": "אופניים וקורקינט ממונע",
-                "num_of_accidents": 1867
-                },
-                {
-                "location": "כל הארץ",
-                "vehicle": "אחר",
-                "num_of_accidents": 229
-                },
-                {
-                "location": "בן יהודה",
-                "vehicle": "מכונית",
-                "num_of_accidents": 64
-                },
-                {
-                "location": "בן יהודה",
-                "vehicle": "אופנוע",
-                "num_of_accidents": 40
-                },
-                {
-                "location": "בן יהודה",
-                "vehicle": "רכב כבד",
-                "num_of_accidents": 22
-                },
-                {
-                "location": "בן יהודה",
-                "vehicle": "אופניים וקורקינט ממונע",
-                "num_of_accidents": 9
-                }
-                ]
+        return [
+            {"location": "כל הארץ", "vehicle": "מכונית", "num_of_accidents": 61307},
+            {"location": "כל הארץ", "vehicle": "רכב כבד", "num_of_accidents": 15801},
+            {"location": "כל הארץ", "vehicle": "אופנוע", "num_of_accidents": 3884},
+            {"location": "כל הארץ", "vehicle": "אופניים וקורקינט ממונע", "num_of_accidents": 1867},
+            {"location": "כל הארץ", "vehicle": "אחר", "num_of_accidents": 229},
+            {"location": "בן יהודה", "vehicle": "מכונית", "num_of_accidents": 64},
+            {"location": "בן יהודה", "vehicle": "אופנוע", "num_of_accidents": 40},
+            {"location": "בן יהודה", "vehicle": "רכב כבד", "num_of_accidents": 22},
+            {"location": "בן יהודה", "vehicle": "אופניים וקורקינט ממונע", "num_of_accidents": 9},
+        ]
 
     accidents_count_pedestrians_per_vehicle_street_vs_all = Widget(
         name="accidents_count_pedestrians_per_vehicle_street_vs_all",
         rank=21,
         items=accidents_count_pedestrians_per_vehicle_street_vs_all_mock_data(),
-        text={"title": "פגיעות בהולכי רגל ברחוב בן יהודה בתל אביב לפי סוג רכב פוגע, בהשוואה לתאונות עירוניות בכל הארץ"},
+        text={
+            "title": "פגיעות בהולכי רגל ברחוב בן יהודה בתל אביב לפי סוג רכב פוגע, בהשוואה לתאונות עירוניות בכל הארץ"
+        },
     )
     output["widgets"].append(accidents_count_pedestrians_per_vehicle_street_vs_all.serialize())
 
     return json.dumps(output, default=str)
 
+
 def get_infographics_data(news_flash_id, years_ago):
-    try:
-        res = infographics_data_cache_updater.get_infographics_data_from_cache(news_flash_id, years_ago)
-    except Exception as e:
-        logging.error(
-            f"Exception while retrieving from infographics cache({news_flash_id},{years_ago})"
-            f":cause:{e.__cause__}, class:{e.__class__}"
-        )
-        res = {}
-    if not res:
-        logging.error(f"infographics_data({news_flash_id}, {years_ago}) not found in cache")
-    return res
+    if os.environ.get("FLASK_ENV") == "development":
+        return create_infographics_data(news_flash_id, years_ago)
+    else:
+        try:
+            res = infographics_data_cache_updater.get_infographics_data_from_cache(
+                news_flash_id, years_ago
+            )
+        except Exception as e:
+            logging.error(
+                f"Exception while retrieving from infographics cache({news_flash_id},{years_ago})"
+                f":cause:{e.__cause__}, class:{e.__class__}"
+            )
+            res = {}
+        if not res:
+            logging.error(f"infographics_data({news_flash_id}, {years_ago}) not found in cache")
+        return res


Test patch:
diff --git a/tests/test_infographics_data_cache.py b/tests/test_infographics_data_cache.py
--- a/tests/test_infographics_data_cache.py
+++ b/tests/test_infographics_data_cache.py
@@ -1,3 +1,4 @@
+import os
 import unittest
 from unittest import TestCase
 from unittest.mock import Mock, patch
@@ -16,6 +17,7 @@ def test_get_not_existing_from_cache(self):
         self.assertEqual(cache_data, {}, "returned value from cache should be None")
 
     @patch("anyway.infographics_utils.infographics_data_cache_updater")
+    @patch.dict(os.environ, {"FLASK_ENV": "test"})
     def test_get_existing(self, get_from_cache):
         expected = {"data": "data"}
         get_from_cache.get_infographics_data_from_cache.return_value = expected
@@ -27,6 +29,7 @@ def test_get_existing(self, get_from_cache):
         self.assertEqual(res, expected, f"{expected} should be found in cache")
 
     @patch("anyway.infographics_utils.infographics_data_cache_updater")
+    @patch.dict(os.environ, {"FLASK_ENV": "test"})
     def test_get_not_existing(self, get_from_cache):
         get_from_cache.get_infographics_data_from_cache.return_value = {}
         res = get_infographics_data(7, 1)
@@ -34,6 +37,7 @@ def test_get_not_existing(self, get_from_cache):
         self.assertEqual(res, {}, f"(7,1) should not be found in cache")
 
     @patch("anyway.infographics_utils.infographics_data_cache_updater")
+    @patch.dict(os.environ, {"FLASK_ENV": "test"})
     def test_get_throws_exception(self, get_from_cache):
         get_from_cache.get_infographics_data_from_cache.side_effect = RuntimeError
         res = get_infographics_data(7, 1)