Repository: mozmeao/snippets-service

Issue:
Change job_fetch_metrics only update metrics on Freq Capped Jobs
We want `fetch_metrics` job to only fetch data on Freq Capped Jobs in very regular intervals so we can pull fast out of bundles a Job that reached it's cap.

Make Impression / Block / Clicks columns in Job list view get their numbers from DailyJobMetrics.

![image](https://user-images.githubusercontent.com/584352/71070387-5cd33780-2183-11ea-98d7-800fb30ca0a8.png)
 



Discussion:

Resolving patch:
diff --git a/snippets/base/admin/adminmodels.py b/snippets/base/admin/adminmodels.py
--- a/snippets/base/admin/adminmodels.py
+++ b/snippets/base/admin/adminmodels.py
@@ -979,11 +979,11 @@ class JobAdmin(admin.ModelAdmin):
         'job_status',
         'publish_start',
         'publish_end',
-        'metric_impressions_humanized',
-        'metric_clicks_humanized',
-        'metric_clicks_ctr',
-        'metric_blocks_humanized',
-        'metric_blocks_ctr',
+        'impressions_humanized',
+        'clicks_humanized',
+        'clicks_ctr',
+        'blocks_humanized',
+        'blocks_ctr',
     ]
     list_display_links = [
         'id',
@@ -1015,12 +1015,11 @@ class JobAdmin(admin.ModelAdmin):
         'id',
         'created',
         'modified',
-        'metric_impressions_humanized',
-        'metric_clicks_humanized',
-        'metric_blocks_humanized',
-        'metric_clicks_ctr',
-        'metric_blocks_ctr',
-        'metric_last_update',
+        'impressions_humanized',
+        'clicks_humanized',
+        'blocks_humanized',
+        'clicks_ctr',
+        'blocks_ctr',
         'redash_link',
         'completed_on',
     ]
@@ -1068,10 +1067,9 @@ class JobAdmin(admin.ModelAdmin):
         }),
         ('Metrics', {
             'fields': (
-                'metric_impressions_humanized',
-                ('metric_clicks_humanized', 'metric_clicks_ctr'),
-                ('metric_blocks_humanized', 'metric_blocks_ctr'),
-                'metric_last_update',
+                'impressions_humanized',
+                ('clicks_humanized', 'clicks_ctr'),
+                ('blocks_humanized', 'blocks_ctr'),
                 'redash_link',
             ),
         }),
@@ -1138,33 +1136,33 @@ def get_queryset(self, request):
         )
         return queryset
 
-    def metric_impressions_humanized(self, obj):
+    def impressions_humanized(self, obj):
         return intcomma(obj.impressions or 0)
-    metric_impressions_humanized.short_description = 'Impressions'
+    impressions_humanized.short_description = 'Impressions'
 
-    def metric_clicks_humanized(self, obj):
+    def clicks_humanized(self, obj):
         return intcomma(obj.clicks or 0)
-    metric_clicks_humanized.short_description = 'Clicks'
+    clicks_humanized.short_description = 'Clicks'
 
-    def metric_blocks_humanized(self, obj):
+    def blocks_humanized(self, obj):
         return intcomma(obj.blocks or 0)
-    metric_blocks_humanized.short_description = 'Blocks'
+    blocks_humanized.short_description = 'Blocks'
 
-    def metric_clicks_ctr(self, obj):
+    def clicks_ctr(self, obj):
         if not (obj.clicks or obj.impressions):
             return 'N/A'
         ratio = (obj.clicks / obj.impressions) * 100
         ratio_class = 'ratio-red' if ratio < 0.02 else 'ratio-green'
         return format_html(f'<span class="{ratio_class}">{ratio:.2f}%</span>')
-    metric_clicks_ctr.short_description = 'CTR'
+    clicks_ctr.short_description = 'CTR'
 
-    def metric_blocks_ctr(self, obj):
+    def blocks_ctr(self, obj):
         if not (obj.blocks or obj.impressions):
             return 'N/A'
         ratio = (obj.blocks / obj.impressions) * 100
         ratio_class = 'ratio-red' if ratio < 0.25 else 'ratio-green'
         return format_html(f'<span class="{ratio_class}">{ratio:.2f}%</span>')
-    metric_blocks_ctr.short_description = 'CTR'
+    blocks_ctr.short_description = 'CTR'
 
     def redash_link(self, obj):
         publish_end = (
diff --git a/snippets/base/management/commands/fetch_metrics.py b/snippets/base/management/commands/fetch_metrics.py
--- a/snippets/base/management/commands/fetch_metrics.py
+++ b/snippets/base/management/commands/fetch_metrics.py
@@ -1,8 +1,7 @@
-from datetime import datetime, timedelta
+from datetime import datetime
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
-from django.db.models import Q
 
 import sentry_sdk
 from redash_dynamic_query import RedashDynamicQuery
@@ -23,8 +22,6 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         now = datetime.utcnow()
-        today = datetime(year=now.year, month=now.month, day=now.day, hour=0, minute=0, second=0)
-        before_seven_days = today - timedelta(days=7)
 
         if not settings.REDASH_API_KEY:
             raise CommandError('Enviroment variable REDASH_API_KEY is required.')
@@ -35,37 +32,11 @@ def handle(self, *args, **options):
             max_wait=settings.REDASH_MAX_WAIT,
         )
 
-        if options['force_update']:
-            jobs = Job.objects.exclude(Q(publish_end=None))
-        else:
-            # Update Jobs if when PUBLISHED haven't been updated for
-            # REDASH_UPDATE_INTERVAL seconds or when COMPLETED haven't been
-            # updated for 8 hours. Typically we need 15 seconds of Telemetry
-            # time per Job, so that's more than 10 minutes for 50 Jobs. With
-            # this trick we reduce the number of queries made in 24 hours
-            # against Telemetry, while still maintaining good enough data
-            # freshness.
-            last_update_timestamp_published = (
-                now - timedelta(seconds=settings.REDASH_UPDATE_INTERVAL))
-            last_update_timestamp_completed = now - timedelta(hours=8)
-            jobs = Job.objects.filter(
-                Q(id__in=(
-                    Job.objects.exclude(
-                        Q(publish_end=None) |
-                        Q(metric_last_update__gte=last_update_timestamp_published)
-                    ) .filter(
-                        Q(status=Job.PUBLISHED)
-                    )
-                )) |
-                Q(id__in=(
-                    Job.objects.exclude(
-                        Q(publish_end=None) |
-                        Q(metric_last_update__gte=last_update_timestamp_completed)
-                    ) .filter(
-                        (Q(status=Job.COMPLETED) & Q(publish_end__gte=before_seven_days))
-                    )
-                ))
-            ).order_by('id')
+        jobs = Job.objects.filter(status=Job.PUBLISHED).exclude(
+            limit_impressions=0,
+            limit_clicks=0,
+            limit_blocks=0,
+        ).order_by('id')
 
         self.stdout.write(f'Fetching Updates for {jobs.count()} Jobs.')
 
@@ -76,13 +47,16 @@ def handle(self, *args, **options):
             blocks = 0
             bind_data = {
                 'start_date': job.publish_start.strftime('%Y-%m-%d'),
-                # Publish end date plus 7 days to include metrics received with
-                # delay.
-                'end_date': (job.publish_end + timedelta(days=7)).strftime('%Y-%m-%d'),
+                'end_date': now.strftime('%Y-%m-%d'),
                 'message_id': job.id,
             }
 
             data_fetched = 0
+            # We need to fetch metrics from two different data sources
+            # (RedShift and BigQuery) to capture all metrics. Firefox
+            # switched to BigQuery on Firefox 72. We expect to be able
+            # to remove RedShift querying in a year from 72's launch
+            # (Jan 2021). Issue #1285
             for query in [settings.REDASH_JOB_QUERY_ID, settings.REDASH_JOB_QUERY_BIGQUERY_ID]:
                 try:
                     result = redash.query(query, bind_data)
diff --git a/snippets/base/models.py b/snippets/base/models.py
--- a/snippets/base/models.py
+++ b/snippets/base/models.py
@@ -1810,6 +1810,12 @@ class Job(models.Model):
         verbose_name='Global Block Limit',
         help_text='Job will complete if number of Blocks exceeds this number. Set to 0 to disable.',
     )
+    # metric_* fields are used to stop Jobs which have reached limits set in limit_* fields.
+    metric_last_update = models.DateTimeField('Last Update', default='1970-01-01', editable=False)
+    metric_impressions = models.PositiveIntegerField(default=0, editable=False)
+    metric_clicks = models.PositiveIntegerField(default=0, editable=False)
+    metric_blocks = models.PositiveIntegerField(default=0, editable=False)
+
     distribution = models.ForeignKey(
         'Distribution',
         on_delete=models.PROTECT,
@@ -1818,11 +1824,6 @@ class Job(models.Model):
                    'left to Default. Useful for running Normandy experiments.'),
     )
 
-    metric_last_update = models.DateTimeField('Last Update', default='1970-01-01', editable=False)
-    metric_impressions = models.PositiveIntegerField(default=0, editable=False)
-    metric_clicks = models.PositiveIntegerField(default=0, editable=False)
-    metric_blocks = models.PositiveIntegerField(default=0, editable=False)
-
     client_limit_lifetime = models.PositiveIntegerField(
         verbose_name='Max Lifetime Impressions',
         default=0,


Test patch:
diff --git a/snippets/base/tests/test_commands.py b/snippets/base/tests/test_commands.py
--- a/snippets/base/tests/test_commands.py
+++ b/snippets/base/tests/test_commands.py
@@ -33,42 +33,28 @@ def test_base(self):
 @override_settings(REDASH_API_KEY='secret')
 class FetchMetricsTests(TestCase):
     def test_base(self):
-        now = datetime.utcnow()
         job_running = JobFactory(
             status=models.Job.PUBLISHED,
             publish_start='2050-01-05 01:00',
-            publish_end='2050-01-06 02:00')
-        job_ended_yesterday = JobFactory(
-            status=models.Job.COMPLETED,
-            publish_start='2019-01-01 01:00',
-            publish_end=now - timedelta(days=1))
+            publish_end='2050-01-08 02:00',
+            limit_impressions=1000,
+        )
         # Without end date
-        JobFactory(
+        job_running2 = JobFactory(
             status=models.Job.PUBLISHED,
-            publish_start='2050-01-05 01:00',
-            publish_end=None)
-        # Ended before then days
-        JobFactory(
-            status=models.Job.COMPLETED,
-            publish_start=now - timedelta(days=11),
-            publish_end=now - timedelta(days=10))
-        # Ended yesteday but updated before 4 hours
-        JobFactory(
-            status=models.Job.COMPLETED,
-            publish_start='2019-01-01 01:00',
-            publish_end=now - timedelta(days=1),
-            metric_last_update=now - timedelta(hours=4),
+            publish_start='2050-01-04 01:00',
+            publish_end=None,
+            limit_blocks=1000,
         )
-
         request_data_first = {
             'start_date': '2050-01-05',
-            'end_date': '2050-01-13',
+            'end_date': '2050-01-06',
             'message_id': job_running.id,
         }
         request_data_second = {
-            'start_date': '2019-01-01',
-            'end_date': (job_ended_yesterday.publish_end + timedelta(days=7)).strftime('%Y-%m-%d'),
-            'message_id': job_ended_yesterday.id,
+            'start_date': '2050-01-04',
+            'end_date': '2050-01-06',
+            'message_id': job_running2.id,
         }
         return_data_first = {
             'query_result': {
@@ -160,10 +146,12 @@ def test_base(self):
         }
 
         with patch('snippets.base.management.commands.fetch_metrics.RedashDynamicQuery') as rdq:
-            rdq.return_value.query.side_effect = [
-                return_data_first, return_data_second, return_data_third, return_data_fourth
-            ]
-            call_command('fetch_metrics', stdout=Mock())
+            with patch('snippets.base.management.commands.fetch_metrics.datetime', wraps=datetime) as datetime_mock:  # noqa
+                datetime_mock.utcnow.return_value = datetime(2050, 1, 6)
+                rdq.return_value.query.side_effect = [
+                    return_data_first, return_data_second, return_data_third, return_data_fourth
+                ]
+                call_command('fetch_metrics', stdout=Mock())
 
         rdq.return_value.query.assert_has_calls([
             call(settings.REDASH_JOB_QUERY_ID, request_data_first),
@@ -177,15 +165,18 @@ def test_base(self):
         self.assertEqual(job_running.metric_blocks, 15)
         self.assertEqual(job_running.metric_clicks, 120)
 
-        job_ended_yesterday.refresh_from_db()
-        self.assertEqual(job_ended_yesterday.metric_impressions, 270)
-        self.assertEqual(job_ended_yesterday.metric_blocks, 150)
-        self.assertEqual(job_ended_yesterday.metric_clicks, 45)
+        job_running2.refresh_from_db()
+        self.assertEqual(job_running2.metric_impressions, 270)
+        self.assertEqual(job_running2.metric_blocks, 150)
+        self.assertEqual(job_running2.metric_clicks, 45)
 
     def test_no_data_fetched(self):
-        JobFactory(status=models.Job.PUBLISHED,
-                   publish_start='2050-01-05 01:00',
-                   publish_end='2050-01-06 02:00')
+        JobFactory(
+            status=models.Job.PUBLISHED,
+            publish_start='2050-01-05 01:00',
+            publish_end='2050-01-06 02:00',
+            limit_clicks=1000,
+        )
 
         # Error raised while fetch Telemetry Data
         with patch('snippets.base.management.commands.fetch_metrics.RedashDynamicQuery') as rdq:
@@ -203,9 +194,12 @@ def test_no_data_fetched(self):
     def test_no_data_fetched_from_one_source(self):
         # Test that even if one of the two data sources fail, the metrics are
         # not saved in the db.
-        job = JobFactory(status=models.Job.PUBLISHED,
-                         publish_start='2050-01-05 01:00',
-                         publish_end='2050-01-06 02:00')
+        job = JobFactory(
+            status=models.Job.PUBLISHED,
+            publish_start='2050-01-05 01:00',
+            publish_end='2050-01-06 02:00',
+            limit_clicks=1000,
+        )
         with patch('snippets.base.management.commands.fetch_metrics.RedashDynamicQuery') as rdq:
             rdq.return_value.query.side_effect = [
                 {