Repository: akvo/akvo-rsr

Issue:
Migrate optimy cronjob to django-q
After #4680 we should finally replace cron jobs with jobs executed in the django-q worker. This will allow us to have visibility of the jobs, their status, and add alerts to GCP to track failed jobs.


Discussion:

Resolving patch:
diff --git a/akvo/rsr/management/commands/sync_django_q_schedules.py b/akvo/rsr/management/commands/sync_django_q_schedules.py
new file mode 100644
--- /dev/null
+++ b/akvo/rsr/management/commands/sync_django_q_schedules.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Akvo Reporting is covered by the GNU Affero General Public License.
+# See more details in the license.txt file located at the root folder of the Akvo RSR module.
+# For additional details on the GNU license please see < http://www.gnu.org/licenses/agpl.html >.
+
+"""Synchronizes the django-q schedules from the settings with those in the DB
+
+New ones will be added to the DB
+Modified ones will be modified in the DB
+Schedules in the DB but missing in the settings will be removed from the DB
+
+Usage:
+
+    python manage.py sync_django_q_schedules
+
+"""
+from django.core.management.base import BaseCommand
+
+from akvo.rsr.usecases.django_q.schedules import sync_with_settings
+
+
+class Command(BaseCommand):
+    help = __doc__
+
+    def handle(self, *args, **options):
+        sync_with_settings()
diff --git a/akvo/rsr/usecases/django_q/__init__.py b/akvo/rsr/usecases/django_q/__init__.py
new file mode 100644
diff --git a/akvo/rsr/usecases/django_q/schedules.py b/akvo/rsr/usecases/django_q/schedules.py
new file mode 100644
--- /dev/null
+++ b/akvo/rsr/usecases/django_q/schedules.py
@@ -0,0 +1,128 @@
+import logging
+from dataclasses import dataclass
+from typing import Dict, List, Set
+
+from django.conf import settings
+from django.db.models import QuerySet
+from django.db.transaction import atomic
+from django_q.models import Schedule
+from rest_framework.serializers import ModelSerializer
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SyncAction:
+    """
+    The operations to be done on the DB in order to sync the django-q schedules from the settings
+    with the existing django-q schedules in the DB
+    """
+    to_add: List[Schedule]
+    to_modify: List[Schedule]
+    to_delete: List[Schedule]
+
+    def __str__(self):
+        return "SyncAction(to_add=%(to_add)s, to_modify=%(to_modify)s, to_delete=%(to_delete)s)" % dict(
+            to_add=len(self.to_add),
+            to_modify=len(self.to_modify),
+            to_delete=len(self.to_delete),
+        )
+
+
+class ScheduleSerializer(ModelSerializer):
+    class Meta:
+        model = Schedule
+        fields = [
+            "name",
+            "func",
+            "args",
+            "kwargs",
+            "cron",
+        ]
+
+
+def sync_with_settings():
+    """
+    Synchronizes the django-q schedules in the DB with those defined in the app settings
+    """
+    schedules = get_setting_schedules()
+    logger.info(f"Got {len(schedules)} schedules from settings")
+    sync_action = calc_sync(schedules, list(Schedule.objects.all()))
+    logger.info(f"Sync actions: {sync_action}")
+    db_schedules = apply_sync(sync_action)
+    logger.info(f"Schedules in DB {db_schedules.count()}")
+
+
+def get_setting_schedules() -> List[Schedule]:
+    """
+    Converts the schedules configuration in the app settings to django-q schedule objects
+    """
+    data = []
+    for key, schedule_conf in settings.AKVO_JOBS.items():
+        args = schedule_conf.get("args")
+        if args is None:
+            args = tuple()
+        # Being explicit here as django-q does this implicitly before calling the func
+        elif not isinstance(args, tuple):
+            args = (args, )
+        conf = {
+            **schedule_conf,
+            "name": key,
+            # stored as strings in DB
+            "args": str(args),
+            "kwargs": str(schedule_conf.get("kwargs") or {}),
+        }
+        data.append(conf)
+    serializer = ScheduleSerializer(data=data, many=True)
+    serializer.is_valid(True)
+    return [Schedule(schedule_type=Schedule.CRON, **item) for item in serializer.validated_data]
+
+
+def calc_sync(setting_schedules: List[Schedule], db_schedules: List[Schedule] = None) -> SyncAction:
+    """
+    Calculates the operations that have to be taken in order to sync the schedules in the settings
+     with the schedules in the db
+    """
+    if db_schedules is None:
+        db_schedules = list(Schedule.objects.all())
+
+    in_map: Dict[str, Schedule] = {schedule.name: schedule for schedule in setting_schedules}
+    db_map: Dict[str, Schedule] = {schedule.name: schedule for schedule in db_schedules}
+
+    in_set: Set[str] = set(in_map.keys())
+    db_set: Set[str] = set(db_map.keys())
+
+    def getter(_dict):
+        return lambda item: _dict.get(item)
+
+    to_add = list(map(getter(in_map), in_set - db_set))
+    to_delete = list(map(getter(db_map), db_set - in_set))
+
+    # Update the db loaded objects with the fields from the settings
+    to_modify = []
+    for name in in_set.intersection(db_set):
+        db_schedule = db_map[name]
+        settings_schedule = in_map[name]
+
+        for field in ScheduleSerializer.Meta.fields:
+            setattr(db_schedule, field, getattr(settings_schedule, field))
+
+        to_modify.append(db_schedule)
+
+    return SyncAction(
+        to_add=to_add,
+        to_modify=to_modify,
+        to_delete=to_delete,
+    )
+
+
+@atomic
+def apply_sync(action: SyncAction) -> QuerySet[Schedule]:
+    """
+    Applies the operations required to sync the schedules in the settings with the schedules in the DB
+    """
+    Schedule.objects.bulk_create(action.to_add)
+    Schedule.objects.filter(id__in=[schedule.id for schedule in action.to_delete]).delete()
+    Schedule.objects.bulk_update(action.to_modify, fields=ScheduleSerializer.Meta.fields)
+
+    return Schedule.objects.all()


Test patch:
diff --git a/akvo/rsr/tests/usecases/test_schedules.py b/akvo/rsr/tests/usecases/test_schedules.py
new file mode 100644
--- /dev/null
+++ b/akvo/rsr/tests/usecases/test_schedules.py
@@ -0,0 +1,193 @@
+from typing import Dict
+
+from django.test import override_settings
+from django_q.models import Schedule
+
+from akvo.rsr.tests.base import BaseTestCase
+from akvo.rsr.usecases.django_q.schedules import (
+    ScheduleSerializer, calc_sync, get_setting_schedules,
+    sync_with_settings,
+)
+
+
+@override_settings(AKVO_JOBS={})
+class EmptyScheduleSettingsTestCase(BaseTestCase):
+    """
+    Test cases where syncing would delete all schedules in the DB if there were any
+    """
+
+    def test_get_setting_schedules(self):
+        schedules = get_setting_schedules()
+        self.assertEqual(len(schedules), 0)
+
+    def test_calc_sync_with_objects(self):
+        Schedule.objects.create(name="test1", cron="* * * * *")
+        Schedule.objects.create(name="test2", cron="* * * * *")
+        Schedule.objects.create(name="test3", cron="* * * * *")
+
+        action = calc_sync(get_setting_schedules())
+
+        self.assertEqual(len(action.to_add), 0)
+        self.assertEqual(len(action.to_modify), 0)
+        self.assertEqual(len(action.to_delete), 3)
+
+    def test_sync_with_objects(self):
+        Schedule.objects.create(name="test1", cron="* * * * *")
+        Schedule.objects.create(name="test2", cron="* * * * *")
+        Schedule.objects.create(name="test3", cron="* * * * *")
+
+        sync_with_settings()
+
+        self.assertEqual(Schedule.objects.count(), 0)
+
+
+@override_settings(AKVO_JOBS={
+    "addition": dict(func="noop", cron="*/5 * * * *"),
+    "addition2": dict(func="noop", cron="*/10 * * * *"),
+    "addition3": dict(func="noop", cron="*/15 * * * *"),
+})
+class AddScheduleSettingsTestCase(BaseTestCase):
+    """
+    Test cases where syncing would add schedules to the DB
+    """
+
+    def test_get_setting_schedules(self):
+        self.maxDiff = None
+        schedules = get_setting_schedules()
+        self.assertEqual(len(schedules), 3)
+
+        self.assertDictEqual({
+            schedule.name: ScheduleSerializer(instance=schedule).data
+            for schedule in schedules
+        }, {
+            "addition": dict(name="addition", func="noop", cron="*/5 * * * *", args='()', kwargs='{}'),
+            "addition2": dict(name="addition2", func="noop", cron="*/10 * * * *", args='()', kwargs='{}'),
+            "addition3": dict(name="addition3", func="noop", cron="*/15 * * * *", args='()', kwargs='{}'),
+        })
+
+    def test_calc_sync_with_empty_db(self):
+        action = calc_sync(get_setting_schedules())
+
+        self.assertEqual(len(action.to_add), 3)
+        self.assertEqual(len(action.to_modify), 0)
+        self.assertEqual(len(action.to_delete), 0)
+
+    def test_sync_with_empty_db(self):
+        sync_with_settings()
+
+        self.assertEqual(Schedule.objects.filter(schedule_type=Schedule.CRON).count(), 3)
+
+
+@override_settings(AKVO_JOBS={
+    "addition": dict(func="noop", cron="*/5 * * * *"),
+    "addition2": dict(func="noop", cron="*/10 * * * *"),
+    "addition3": dict(func="noop", cron="*/15 * * * *"),
+})
+class ModifyDBSchedulesTestCase(BaseTestCase):
+    """
+    Test cases where syncing would modify existing schedules in the DB
+    """
+
+    def setUp(self):
+        super().setUp()
+        self.maxDiff = None
+        Schedule.objects.create(name="addition", cron="* * * * *")
+        Schedule.objects.create(name="addition2", cron="* * * * *")
+        Schedule.objects.create(name="addition3", cron="* * * * *")
+
+    def test_calc_sync(self):
+        action = calc_sync(get_setting_schedules())
+
+        self.assertEqual(len(action.to_add), 0)
+        self.assertEqual(len(action.to_modify), 3)
+        self.assertEqual(len(action.to_delete), 0)
+
+    def test_sync(self):
+        sync_with_settings()
+
+        db_schedules = list(Schedule.objects.all())
+
+        self.assertEqual(len(db_schedules), 3)
+        self.assertDictEqual({
+            schedule.name: ScheduleSerializer(instance=schedule).data
+            for schedule in db_schedules
+        }, {
+            "addition": dict(name="addition", func="noop", cron="*/5 * * * *", args='()', kwargs='{}'),
+            "addition2": dict(name="addition2", func="noop", cron="*/10 * * * *", args='()', kwargs='{}'),
+            "addition3": dict(name="addition3", func="noop", cron="*/15 * * * *", args='()', kwargs='{}'),
+        })
+
+
+class GetSettingsSchedulesTest(BaseTestCase):
+    """
+    Specifically test get_setting_schedules
+    """
+
+    def _get_with(self, akvo_jobs: dict) -> Dict[str, Schedule]:
+        with self.settings(AKVO_JOBS=akvo_jobs):
+            return {schedule.name: schedule for schedule in get_setting_schedules()}
+
+    def _test_schedule_arg_param(self, schedule_name, arg_value, expected_args=None, arg_type="args"):
+        """
+        Ensure passing "arg" or "kwargs", when constructing a Schedule, is converted to the right type and value
+
+        :param schedule_name: resulting Schedule.name
+        :param arg_value:
+        :param expected_args: Optional override of the expected args (by default a stringified version of arg_value)
+        :param arg_type: Either "args" or "kwargs (Schedule.args, Schedule.kwargs)
+        """
+        if expected_args is None:
+            expected_args = str(arg_value)
+
+        schedules = self._get_with({
+            schedule_name: {
+                "func": "print",
+                "cron": "* * * * *",
+                arg_type: arg_value,
+            }
+        })
+        schedule = schedules[schedule_name]
+        self.assertEqual(getattr(schedule, arg_type), expected_args, f"Schedule.{arg_type} != {expected_args}")
+
+    def test_no_args(self):
+        """args=tuple with primitives"""
+        self._test_schedule_arg_param("primitives", None, expected_args="()")
+
+    def test_tuple_args_primitives(self):
+        """args=tuple with primitives"""
+        self._test_schedule_arg_param("primitives", ("a", "b", "c", 1, 2, 3))
+
+    def test_tuple_args_lists(self):
+        """args=tuple with lists"""
+        self._test_schedule_arg_param("lists", (["a"], ["b"], ["c"]))
+
+    def test_tuple_args_dicts(self):
+        """args=tuple with dicts"""
+        self._test_schedule_arg_param("dicts", ({"a": 1}, {"b": 2}, {"c": 3}))
+
+    def test_tuple_args_mixed(self):
+        """args=tuple with mixed types"""
+        self._test_schedule_arg_param("mixed", ("a", ["b"], {"c": 3}))
+
+    def test_single_arg_primitive(self):
+        """args=primitive type"""
+        args = "hello"
+        self._test_schedule_arg_param("primitive", args, expected_args=str((args,)))
+
+    def test_single_arg_list(self):
+        """args=list type"""
+        args = ["a", "b", "c"]
+        self._test_schedule_arg_param("list", args, expected_args=str((args,)))
+
+    def test_single_arg_dict(self):
+        """args=dict type"""
+        args = dict(a=1, b=2, c=3)
+        self._test_schedule_arg_param("dict", args, expected_args=str((args,)))
+
+    def test_kwargs(self):
+        """kwargs are passed"""
+        self._test_schedule_arg_param("kwargs", dict(a=1, b=2, c=3), arg_type="kwargs")
+
+    def test_no_kwargs(self):
+        """args=tuple with primitives"""
+        self._test_schedule_arg_param("no_kwargs", None, expected_args="{}", arg_type="kwargs")