Repository: akvo/akvo-rsr

Issue:
A4A optimy script failing
The optimy script was failing with a `NonExistentTimeError`

Here's a recreation of the problem

```py
#!/usr/bin/env python3

from django.utils import timezone
from datetime import datetime
import pytz


tz = pytz.timezone("Europe/Stockholm")
dt = datetime(2021, 3, 28, 2, 10)
print(timezone.make_aware(dt, tz))
# Problem happens in this function of pytz
#print(tz.localize(dt, is_dst=None))

```

and the result

```
Traceback (most recent call last):
  File "/tmp/venv/test.py", line 10, in <module>
    print(timezone.make_aware(dt, tz))
  File "/tmp/venv/lib/python3.10/site-packages/django/utils/timezone.py", line 242, in make_aware
    return timezone.localize(value, is_dst=is_dst)
  File "/tmp/venv/lib/python3.10/site-packages/pytz/tzinfo.py", line 341, in localize
    raise NonExistentTimeError(dt)
```

# Investigation

From the documentation of `tzinfo.timezone.localize`

```py
"""
is_dst is also used to determine the correct timezone in the
wallclock times jumped over at the start of daylight saving time.

>>> pacific = timezone('US/Pacific')
>>> dt = datetime(2008, 3, 9, 2, 0, 0)
>>> ploc_dt1 = pacific.localize(dt, is_dst=True)
>>> ploc_dt2 = pacific.localize(dt, is_dst=False)
>>> ploc_dt1.strftime(fmt)
'2008-03-09 02:00:00 PDT (-0700)'
>>> ploc_dt2.strftime(fmt)
'2008-03-09 02:00:00 PST (-0800)'
>>> str(ploc_dt2 - ploc_dt1)
'1:00:00'

Use is_dst=None to raise a NonExistentTimeError for these skipped
times.

>>> try:
...     loc_dt1 = pacific.localize(dt, is_dst=None)
... except NonExistentTimeError:
...     print('Non-existent')
"""
```

The given time is right in the hour where Stockholm moves one hour forward in daylight savings and the time could be either CET or CEST.


Discussion:

Resolving patch:
diff --git a/akvo/rsr/management/commands/a4a_optimy_import.py b/akvo/rsr/management/commands/a4a_optimy_import.py
--- a/akvo/rsr/management/commands/a4a_optimy_import.py
+++ b/akvo/rsr/management/commands/a4a_optimy_import.py
@@ -11,7 +11,7 @@
     python manage.py a4a_optimy_import [--project-id <optimy_project_id>]
 
 """
-
+import logging
 from itertools import groupby
 from typing import Optional, Tuple
 
@@ -117,7 +117,7 @@ class ContractStatus:
 def programs_exist():
     program = Project.objects.filter(id=MASTER_PROGRAM_ID).first()
     if program is not None:
-        sub_programs = set(program.descendants(depth=1).values_list("pk", flat=True))
+        sub_programs = set(program.descendants(max_depth=1).values_list("pk", flat=True))
         program_ids = set(PROGRAM_IDS.values())
         return (sub_programs & program_ids) == program_ids
     return False
@@ -165,12 +165,13 @@ def get_project_answers(project_id):
 
 
 def get_answer(form_id: str, answers: dict, key: str, ans_key: str = "value"):
-    answer = answers.get(FORM_QUESTION_MAPPING[form_id][key], {}).get(ans_key)
+    answer = answers.get(FORM_QUESTION_MAPPING.get(form_id, {}).get(key), {}).get(ans_key)
     if not answer:
         print(f"Could not find answer for {key}")
     return answer
 
 
+@transaction.atomic()
 def create_project(optimy_project: dict, answers: dict) -> Tuple[Optional[Project], bool]:
     """
     Takes an Optimy project and upserts an RSR project in the DB
@@ -361,11 +362,14 @@ def _handle(self, *args, **options):
         if not programs_exist():
             raise CommandError("Not all programs are present in the DB")
         optimy_project_id = options["project_id"]
-        if not optimy_project_id:
-            print("Fetching projects from Optimy")
-            optimy_projects = get_optimy_projects()
-        else:
-            optimy_projects = [dict(id=optimy_project_id)]
+        print("Fetching projects from Optimy")
+        optimy_projects = get_optimy_projects()
+        if optimy_project_id:
+            optimy_projects = [
+                optimy_project
+                for optimy_project in optimy_projects
+                if optimy_project.get("id") == optimy_project_id
+            ]
 
         # Set program IDs
         set_program_iati_ids()
@@ -379,10 +383,28 @@ def _handle(self, *args, **options):
             "Operation",
             "Project Title",
         ]
+        failed_imported_projects = tablib.Dataset()
+        failed_imported_projects.headers = [
+            "Optimy ID",
+            "Contract Status",
+            "Project Title",
+        ]
+
         for optimy_project in optimy_projects:
             optimy_project_id = optimy_project["id"]
             answers = get_project_answers(optimy_project_id)
-            project, created = create_project(optimy_project, answers)
+            try:
+                project, created = create_project(optimy_project, answers)
+            except:  # noqa: E722
+                project = None
+                created = False
+                logging.warning("Couldn't create project %s", optimy_project_id, exc_info=True)
+                failed_imported_projects.append([
+                    optimy_project_id,
+                    CONTRACT_STATUS_IDS.get(optimy_project.get("status_id"), "UNKNOWN"),
+                    get_answer(optimy_project.get("form_id"), answers, "title"),
+                ])
+
             if project is None:
                 continue
             print(f"Imported {optimy_project_id} as {project.id} - {project.title}")
@@ -398,6 +420,8 @@ def _handle(self, *args, **options):
 
         print("Report:")
         print(imported_projects.export("csv"))
+        print("Failed:")
+        print(failed_imported_projects.export("csv"))
 
         if dry_run:
             raise InterruptedError()


Test patch: