Repository: mozmeao/snippets-service

Issue:
Fetch Daily Impressions Data once per week
DailyImpressions are used to calculate an approximation of how many
Impressions are actually valid impressions, i.e. User has spend enough
time on the page to read and understand the message.

Metrics have shown that the percentage of time spend of New Tab Page is
pretty stable and around 63% of the time Users stay 5 seconds of more on
the page.

Since the number is pretty stable, we don't need to pull it every day.
Instead this PR changes the pull frequency to once per week to save on
resources.

- [x] can we set completed_on for CANCELED Jobs as well?
- [x] if so, migrate the existing data to have proper dates



Discussion:

Resolving patch:
diff --git a/snippets/base/etl.py b/snippets/base/etl.py
--- a/snippets/base/etl.py
+++ b/snippets/base/etl.py
@@ -1,9 +1,10 @@
 import collections
 import json
-
+from datetime import datetime, timedelta
 from urllib.parse import urlencode
 
 from django.conf import settings
+from django.db.models import Q
 from django.db.transaction import atomic
 from redash_dynamic_query import RedashDynamicQuery
 
@@ -42,7 +43,17 @@ def redash_rows(query_name, date):
 
 
 def prosses_rows(rows, key='message_id'):
-    job_ids = [str(x) for x in Job.objects.all().values_list('id', flat=True)]
+    now = datetime.utcnow()
+
+    # To fight Telemetry spam, process metrics for Jobs currently Published or
+    # Completed the last 7 days.
+    jobs = Job.objects.filter(
+        # Still published
+        Q(status=Job.PUBLISHED) |
+        # Or completed during the last 7 days
+        Q(completed_on__gte=now - timedelta(days=7))
+    )
+    job_ids = [str(x) for x in jobs.values_list('id', flat=True)]
     new_rows = []
     for row in sorted(rows, key=lambda x: x[key]):
         # Remove rows with invalid Job IDs
diff --git a/snippets/base/management/commands/fetch_daily_metrics.py b/snippets/base/management/commands/fetch_daily_metrics.py
--- a/snippets/base/management/commands/fetch_daily_metrics.py
+++ b/snippets/base/management/commands/fetch_daily_metrics.py
@@ -40,12 +40,22 @@ def handle(self, *args, **options):
                 check_date += timedelta(days=1)
 
         for d in dates:
+            successful_jobs = 0
+            # Fetch Daily Impression data from last Monday relative to `d` date
+            # to calculate Adjusted Impressions / Clicks / Blocks.
+            last_monday = d - timedelta(days=d.weekday())
+            if not models.DailyImpressions.objects.filter(date=last_monday).exists():
+                self.stdout.write(f'Fetching impression data for {last_monday}.')
+                if etl.update_impressions(last_monday):
+                    successful_jobs += 1
+            else:
+                successful_jobs += 1
+
             self.stdout.write(f'Fetching data for {d}.')
+            if etl.update_job_metrics(d):
+                successful_jobs += 1
 
-            if not all([
-                    etl.update_impressions(d),
-                    etl.update_job_metrics(d),
-            ]):
+            if successful_jobs != 2:
                 # We didn't manage to fetch all data, something is wrong.
                 raise CommandError('Cannot fetch data from Telemetry.')
 
diff --git a/snippets/base/models.py b/snippets/base/models.py
--- a/snippets/base/models.py
+++ b/snippets/base/models.py
@@ -1985,7 +1985,7 @@ def change_status(self, status, user=None, send_slack=True, reason=''):
             return
 
         self.status = status
-        if status == self.COMPLETED:
+        if status in [self.CANCELED, self.COMPLETED]:
             self.completed_on = datetime.utcnow()
         self.save()
 
@@ -2306,33 +2306,16 @@ def save(self, *args, **kwargs):
         return super().save(*args, **kwargs)
 
     def calculate_adj_impression(self):
-        try:
-            di = DailyImpressions.objects.get(date=self.date)
-        except DailyImpressions.DoesNotExist:
-            percentage = self.DEFAULT_IMPRESSION_PERCENTAGE
+        # Find the most recent DailyImpressions object to `date`.
+        di = DailyImpressions.objects.filter(date__lte=self.date).order_by("-date").first()
+        if not di:
+            percentage_impressions = self.DEFAULT_IMPRESSION_PERCENTAGE
             percentage_clients = self.DEFAULT_CLIENT_PERCENTAGE
         else:
-            total_impressions = 0
-            valid_impressions = 0
-            total_clients = 0
-            valid_clients = 0
-
-            for detail in di.details:
-                total_impressions += detail['counts']
-                total_clients += detail['no_clients']
-                if int(detail['duration']) >= self.IMPRESSION_THRESHOLD_SECONDS:
-                    valid_impressions += detail['counts']
-                    valid_clients += detail['no_clients']
-
-            if total_impressions < 100_000:
-                # Sample too small, set default
-                percentage = self.DEFAULT_IMPRESSION_PERCENTAGE
-                percentage_clients = self.DEFAULT_CLIENT_PERCENTAGE
-            else:
-                percentage = valid_impressions / total_impressions
-                percentage_clients = valid_clients / total_clients
-
-        adj_im = int((self.impression * percentage) + 0.5)
+            percentage_impressions = di.percentage_impressions
+            percentage_clients = di.percentage_clients
+
+        adj_im = int((self.impression * percentage_impressions) + 0.5)
         adj_clients = int((self.impression_no_clients * percentage_clients) + 0.5)
         adj_clients_total = int((self.impression_no_clients_total * percentage_clients) + 0.5)
 
@@ -2369,3 +2352,40 @@ class DailyImpressions(models.Model):
     data_fetched_on = models.DateTimeField(auto_now_add=True)
     date = models.DateField(editable=False, db_index=True, unique=True)
     details = JSONField(default=dict, editable=False)
+
+    @property
+    def percentage_impressions(self):
+        total_impressions = 0
+        valid_impressions = 0
+
+        for detail in self.details:
+            total_impressions += detail['counts']
+            if int(detail['duration']) >= JobDailyPerformance.IMPRESSION_THRESHOLD_SECONDS:
+                valid_impressions += detail['counts']
+
+        if total_impressions < 100_000:
+            # Sample too small, set default
+            percentage_impressions = JobDailyPerformance.DEFAULT_IMPRESSION_PERCENTAGE
+        else:
+            percentage_impressions = valid_impressions / total_impressions
+
+        return percentage_impressions
+
+    @property
+    def percentage_clients(self):
+        total_clients = 0
+        valid_clients = 0
+
+        for detail in self.details:
+            # Not all data have `no_clients` key.
+            total_clients += detail.get('no_clients', 0)
+            if int(detail['duration']) >= JobDailyPerformance.IMPRESSION_THRESHOLD_SECONDS:
+                valid_clients += detail.get('no_clients', 0)
+
+        if total_clients < 10_000:
+            # Sample too small, set default
+            percentage_clients = JobDailyPerformance.DEFAULT_CLIENT_PERCENTAGE
+        else:
+            percentage_clients = valid_clients / total_clients
+
+        return percentage_clients


Test patch: