Repository: mozmeao/snippets-service

Issue:
Create Freq Capping Goals for Jobs
Create Global Frequency Capping goals for Jobs (introduced in #1034). 

Goals:
 - Total impressions
 - Total clicks and button clicks (combined)
 - Total blocks

A `Job` can have multiple goals (freq capping, datetime), first one reached will un-publish the `Job`.

Requires #1036. Also `Job`s with Freq Cap goals must get data from Redash more often (TBD).

## Use case:
 - Publish a Snippet for 1M impressions. It's fine it snippet gets 5K-10K impressions more.


## Questions
 - [x] How do we handle Jobs with Freq Capping when no Telemetry data is available? 
   - Jobs that have impression, block or click limits set and no metrics data updated for at least 24 hours will be marked Complete.


Discussion:

Resolving patch:
diff --git a/snippets/base/management/commands/update_jobs.py b/snippets/base/management/commands/update_jobs.py
--- a/snippets/base/management/commands/update_jobs.py
+++ b/snippets/base/management/commands/update_jobs.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from django.contrib.auth import get_user_model
 from django.core.management.base import BaseCommand
@@ -60,6 +60,23 @@ def handle(self, *args, **options):
             count_limit[limit] = jobs.count()
             count_total_completed += count_limit[limit]
 
+        # Disable Jobs that have Impression, Click or Block limits but don't
+        # have metrics data for at least 24h. This is to handle cases where the
+        # Metrics Pipeline is broken.
+        yesterday = datetime.utcnow() - timedelta(days=1)
+        jobs = (Job.objects
+                .filter(status=Job.PUBLISHED)
+                .exclude(limit_impressions=0, limit_clicks=0, limit_blocks=0)
+                .filter(metric_last_update__lt=yesterday))
+        for job in jobs:
+            job.change_status(
+                status=Job.COMPLETED,
+                user=user,
+                reason=f'Premature termination due to missing metrics.',
+            )
+        count_premature_termination = jobs.count()
+        count_total_completed += count_premature_termination
+
         count_running = Job.objects.filter(status=Job.PUBLISHED).count()
 
         self.stdout.write(
@@ -69,5 +86,6 @@ def handle(self, *args, **options):
             f'  - Reached Impressions Limit: {count_limit["impressions"]}\n'
             f'  - Reached Clicks Limit: {count_limit["clicks"]}\n'
             f'  - Reached Blocks Limit: {count_limit["blocks"]}\n'
+            f'  - Premature Termination due to missing metrics: {count_premature_termination}\n'
             f'Total Jobs Running: {count_running}\n'
         )


Test patch:
diff --git a/snippets/base/tests/test_commands.py b/snippets/base/tests/test_commands.py
--- a/snippets/base/tests/test_commands.py
+++ b/snippets/base/tests/test_commands.py
@@ -158,55 +158,81 @@ def test_no_data_fetched(self):
 
 class UpdateJobsTests(TestCase):
     def test_base(self):
+        now = datetime.utcnow()
         job_without_end_date = JobFactory(
             status=models.Job.PUBLISHED,
             publish_end=None)
         job_that_has_ended = JobFactory(
             status=models.Job.PUBLISHED,
-            publish_end=datetime.utcnow())
+            publish_end=now)
         job_ending_in_the_future = JobFactory(
             status=models.Job.PUBLISHED,
             limit_impressions=10000,
-            publish_end=datetime.utcnow() + timedelta(days=1))
+            metric_last_update=now,
+            publish_end=now + timedelta(days=1))
         job_scheduled_ready_to_go = JobFactory(
             status=models.Job.SCHEDULED,
-            publish_start=datetime.utcnow())
+            publish_start=now)
         job_scheduled_in_the_future = JobFactory(
             status=models.Job.SCHEDULED,
-            publish_start=datetime.utcnow() + timedelta(days=1))
+            publish_start=now + timedelta(days=1))
         job_impression_limit_reached = JobFactory(
             status=models.Job.PUBLISHED,
             limit_impressions=10000,
             metric_impressions=10000,
-            publish_end=datetime.utcnow() + timedelta(days=1))
+            metric_last_update=now,
+            publish_end=now + timedelta(days=1))
         job_click_limit_reached = JobFactory(
             status=models.Job.PUBLISHED,
             limit_impressions=10000,
             metric_impressions=1000,
             limit_clicks=1000,
             metric_clicks=1001,
-            publish_end=datetime.utcnow() + timedelta(days=1))
+            metric_last_update=now,
+            publish_end=now + timedelta(days=1))
         job_block_limit_reached = JobFactory(
             status=models.Job.PUBLISHED,
             limit_blocks=10,
             metric_blocks=1000,
-            publish_end=datetime.utcnow() + timedelta(days=1)
+            metric_last_update=now - timedelta(minutes=30),
+            publish_end=now + timedelta(days=1)
         )
         job_impression_limit_not_reached = JobFactory(
             status=models.Job.PUBLISHED,
             limit_impressions=100,
             metric_impressions=10,
-            publish_end=datetime.utcnow() + timedelta(days=1))
+            metric_last_update=now - timedelta(hours=12),
+            publish_end=now + timedelta(days=1))
         job_click_limit_not_reached = JobFactory(
             status=models.Job.PUBLISHED,
             limit_clicks=100,
             metric_clicks=10,
-            publish_end=datetime.utcnow() + timedelta(days=1))
+            metric_last_update=now - timedelta(hours=1),
+            publish_end=now + timedelta(days=1))
         job_block_limit_not_reached = JobFactory(
             status=models.Job.PUBLISHED,
             limit_blocks=100,
             metric_blocks=10,
-            publish_end=datetime.utcnow() + timedelta(days=1))
+            metric_last_update=now,
+            publish_end=now + timedelta(days=1))
+        job_impression_limit_not_reached_but_no_data = JobFactory(
+            status=models.Job.PUBLISHED,
+            limit_impressions=100,
+            metric_impressions=10,
+            metric_last_update=now - timedelta(hours=26),
+            publish_end=now + timedelta(days=1))
+        job_click_limit_not_reached_but_no_data = JobFactory(
+            status=models.Job.PUBLISHED,
+            limit_clicks=100,
+            metric_clicks=10,
+            metric_last_update=now - timedelta(hours=30),
+            publish_end=now + timedelta(days=1))
+        job_block_limit_not_reached_but_no_data = JobFactory(
+            status=models.Job.PUBLISHED,
+            limit_blocks=100,
+            metric_blocks=10,
+            metric_last_update=now - timedelta(hours=40),
+            publish_end=now + timedelta(days=1))
 
         job_cancelled = JobFactory(status=models.Job.CANCELED)
         job_completed = JobFactory(status=models.Job.COMPLETED)
@@ -226,6 +252,9 @@ def test_base(self):
         job_impression_limit_not_reached.refresh_from_db()
         job_click_limit_not_reached.refresh_from_db()
         job_block_limit_not_reached.refresh_from_db()
+        job_impression_limit_not_reached_but_no_data.refresh_from_db()
+        job_click_limit_not_reached_but_no_data.refresh_from_db()
+        job_block_limit_not_reached_but_no_data.refresh_from_db()
 
         self.assertEqual(job_without_end_date.status, models.Job.PUBLISHED)
         self.assertEqual(job_that_has_ended.status, models.Job.COMPLETED)
@@ -240,6 +269,9 @@ def test_base(self):
         self.assertEqual(job_impression_limit_not_reached.status, models.Job.PUBLISHED)
         self.assertEqual(job_click_limit_not_reached.status, models.Job.PUBLISHED)
         self.assertEqual(job_block_limit_not_reached.status, models.Job.PUBLISHED)
+        self.assertEqual(job_impression_limit_not_reached_but_no_data.status, models.Job.COMPLETED)
+        self.assertEqual(job_click_limit_not_reached_but_no_data.status, models.Job.COMPLETED)
+        self.assertEqual(job_block_limit_not_reached_but_no_data.status, models.Job.COMPLETED)
 
 
 class GenerateBundles(TestCase):