Repository: akvo/akvo-rsr

Issue:
Adjust D2D report script
Here are the requested output metrics;

- \# of (active) partners using RSR
- \# of new partners using in year X
- \# of RSR (total)  projects
- \# of projects using the RSR results framework
- \# of indicator updates in RSR
- \# of partners reporting to IATI
- \# of projects reporting to IATI
- Organizations using RSR to report to IATI in year X
- \# of unique users that logged into RSR in year X


Discussion:

Resolving patch:
diff --git a/akvo/rsr/management/commands/report_d2d.py b/akvo/rsr/management/commands/report_d2d.py
--- a/akvo/rsr/management/commands/report_d2d.py
+++ b/akvo/rsr/management/commands/report_d2d.py
@@ -6,6 +6,7 @@
 
 
 import datetime
+import tablib
 
 from django.contrib.admin.models import LogEntry
 from django.core.management.base import BaseCommand
@@ -13,75 +14,105 @@
 
 from akvo.rsr.models import (
     IatiExport, Organisation, Project, IndicatorPeriodData,
-    IndicatorPeriodDataComment,
+    LoginLog, PublishingStatus,
 )
 
 
-def projects_using_results_framework(before_date):
-    """Return projects which had activity in the results framework before a date."""
-
-    projects_from_updates = set(
-        IndicatorPeriodData.objects
-        .filter(Q(created_at__lt=before_date) | Q(last_modified_at__lt=before_date))
-        .values_list('period__indicator__result__project__id', flat=True)
+def get_active_organisations(date_start, date_end):
+    projects_with_project_updates = Project.objects.filter(
+        Q(project_updates__created_at__range=(date_start, date_end))
+        | Q(project_updates__last_modified_at__range=(date_start, date_end))
+    ).distinct()
+    projects_with_indicator_updates = Project.objects.filter(
+        Q(results__indicators__periods__data__created_at__range=(date_start, date_end))
+        | Q(results__indicators__periods__data__last_modified_at__range=(date_start, date_end))
+    ).distinct()
+    active_projects = projects_with_project_updates.union(projects_with_indicator_updates)
+    org_ids_of_active_projects = active_projects.values_list('primary_organisation', flat=True)
+    return Organisation.objects.filter(id__in=set(org_ids_of_active_projects))
+
+
+def get_projects_with_active_results_framework(date_start, date_end):
+    log_entries = LogEntry.objects\
+        .filter(action_time__range=(date_start, date_end))\
+        .filter(Q(content_type__model='indicator') | Q(content_type__model='indicatorperiod') | Q(content_type__model='result'))\
+        .values_list('content_type__model', 'object_id')
+    result_ids = [r for _, r in filter(lambda x: x[0] == 'result', log_entries)]
+    indicator_ids = [i for _, i in filter(lambda x: x[0] == 'indicator', log_entries)]
+    period_ids = [p for _, p in filter(lambda x: x[0] == 'indicatorperiod', log_entries)]
+    projects_with_results = Project.objects.filter(results__in=result_ids).distinct()
+    projects_with_indicators = Project.objects.filter(results__indicators__in=indicator_ids).distinct()
+    projects_with_periods = Project.objects.filter(results__indicators__periods__in=period_ids).distinct()
+    projects_with_indicator_updates = Project.objects.filter(
+        Q(results__indicators__periods__data__created_at__range=(date_start, date_end))
+        | Q(results__indicators__periods__data__last_modified_at__range=(date_start, date_end))
+    ).distinct()
+    projects_with_indicator_update_comments = Project.objects.filter(
+        Q(results__indicators__periods__data__comments__created_at__range=(date_start, date_end))
+        | Q(results__indicators__periods__data__comments__last_modified_at__range=(date_start, date_end))
+    ).distinct()
+    return projects_with_results.union(
+        projects_with_indicators,
+        projects_with_periods,
+        projects_with_indicator_updates,
+        projects_with_indicator_update_comments,
     )
 
-    projects_from_comments = set(
-        IndicatorPeriodDataComment.objects
-        .filter(Q(created_at__lt=before_date) | Q(last_modified_at__lt=before_date))
-        .values_list('data__period__indicator__result__project__id', flat=True)
-    )
 
-    filter_ = (
-        Q(content_type__model='indicator')
-        | Q(content_type__model='indicatorperiod')
-        | Q(content_type__model='result')
-    )
-    log_entries = LogEntry.objects.filter(action_time__lt=before_date)\
-                                  .filter(filter_)\
-                                  .values_list('content_type__model', 'object_id')
-
-    indicators = [
-        id_ for _, id_ in
-        filter(lambda x: x[0] == 'indicator', log_entries)
-    ]
-    periods = [
-        id_ for _, id_ in
-        filter(lambda x: x[0] == 'indicatorperiod', log_entries)
-    ]
-    results = [
-        id_ for _, id_ in
-        filter(lambda x: x[0] == 'result', log_entries)
-    ]
-
-    projects_from_periods = set(
-        Project.objects
-        .filter(results__indicators__periods__in=periods)
-        .values_list('id', flat=True)
-    )
-    projects_from_indicators = set(
-        Project.objects
-        .filter(results__indicators__in=indicators)
-        .values_list('id', flat=True)
-    )
-    projects_from_results = set(
-        Project.objects.filter(results__in=results)
-        .values_list('id', flat=True)
-    )
+def get_unique_login_emails(date_start, date_end):
+    return LoginLog.objects\
+        .filter(success=True, created_at__range=(date_start, date_end))\
+        .values('email')\
+        .distinct()
+
+
+def get_published_projects(date_latest):
+    return Project.objects\
+        .filter(created_at__lt=date_latest, publishingstatus__status=PublishingStatus.STATUS_PUBLISHED)
+
+
+def get_indicator_updates(date_start, date_end):
+    return IndicatorPeriodData.objects\
+        .filter(Q(created_at__range=(date_start, date_end)) | Q(last_modified_at__range=(date_start, date_end)))
+
+
+def get_orgs_reporting_to_iati(date_start, date_end):
+    return Organisation.objects\
+        .filter(iati_exports__created_at__range=(date_start, date_end))\
+        .annotate(project_count=Count('iati_exports__projects'))\
+        .filter(project_count__gt=0)\
+        .distinct()
 
-    results_projects = (projects_from_comments
-                        | projects_from_updates
-                        | projects_from_periods
-                        | projects_from_indicators
-                        | projects_from_results)
 
-    return results_projects
+def get_projects_reporting_to_iati(date_start, date_end):
+    return Project.objects.filter(iatiexport__created_at__range=(date_start, date_end)).distinct()
+
+
+def make_orgs_reporting_to_iati_list(orgs, date_start, date_end):
+    orgs_list = tablib.Dataset()
+    orgs_list.headers = ['name', 'long_name', 'iati_org_type', 'report_count', 'project_count']
+    for org in orgs:
+        exports = IatiExport.objects.filter(
+            created_at__range=(date_start, date_end),
+            reporting_organisation=org
+        )
+        projects = set(exports.values_list('projects', flat=True))
+        if not projects:
+            continue
+        orgs_list.append([
+            org.name,
+            org.long_name,
+            org.iati_org_type_unicode(),
+            exports.count(),
+            len(projects),
+        ])
+    return orgs_list
 
 
 class Command(BaseCommand):
     help = """\
     Script to generate the D2D report
-    <script> <new_date:%Y-%m-%d> <old_date:%Y-%m-%d>
+    <script> <date_start:%Y-%m-%d> <date_end:%Y-%m-%d>
     """
 
     def add_arguments(self, parser):
@@ -97,70 +128,35 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
+        date_start = options['date_start']
+        date_end = options['date_end']
+
+        print(f'RSR Metrics Report ({date_start:%Y-%m-%d} - {date_end:%Y-%m-%d})')
+
+        active_partners = get_active_organisations(date_start, date_end)
+        print(f'# of active partners using RSR: {active_partners.count()}')
+
+        new_orgs = Organisation.objects.filter(created_at__range=(date_start, date_end))
+        print(f'# of new partners using RSR: {new_orgs.count()}')
+
+        unique_login_emails = get_unique_login_emails(date_start, date_end)
+        print(f'# of unique login: {unique_login_emails.count()}')
+
+        published_projects = get_published_projects(date_end)
+        print(f'# of published RSR projects (total): {published_projects.count()}')
+
+        active_projects = get_projects_with_active_results_framework(date_start, date_end)
+        print(f'# of projects using the RSR results framework: {active_projects.count()}')
+
+        indicator_updates = get_indicator_updates(date_start, date_end)
+        print(f'# of indicator updates in RSR: {indicator_updates.count()}')
+
+        orgs_reporting_to_iati = get_orgs_reporting_to_iati(date_start, date_end)
+        print(f'# of partners reporting to IATI: {orgs_reporting_to_iati.count()}')
+
+        projects_reporting_to_iati = get_projects_reporting_to_iati(date_start, date_end)
+        print(f'# of projects reporting to IATI: {projects_reporting_to_iati.count()}')
 
-        date_new = options['date_end']
-        date_old = options['date_start']
-
-        print('Metric parameter,{:%Y-%m-%d},{:%Y-%m-%d}'.format(date_new, date_old))
-
-        # Total number of organisations/partners on RSR
-        orgs_new = Organisation.objects.filter(created_at__lt=date_new).distinct()
-        orgs_old = Organisation.objects.filter(created_at__lt=date_old).distinct()
-        print('# of partners using RSR,{},{}'.format(orgs_new.count(), orgs_old.count()))
-
-        # Total number of projects on RSR
-        projects_new = Project.objects.filter(created_at__lt=date_new).distinct()
-        projects_old = Project.objects.filter(created_at__lt=date_old).distinct()
-        print('# of RSR projects,{},{}'.format(projects_new.count(), projects_old.count()))
-
-        # Total number of projects which had some activity going on in the results
-        # framework...
-        print('# of projects using the RSR results framework,{},{}'.format(
-            len(projects_using_results_framework(date_new)),
-            len(projects_using_results_framework(date_old)),
-        ))
-
-        # No. of Indicator Updates
-        updates_new = IndicatorPeriodData.objects.filter(created_at__lt=date_new).distinct()
-        updates_old = IndicatorPeriodData.objects.filter(created_at__lt=date_old).distinct()
-
-        print('# of indicator updates in RSR,{},{}'.format(
-            updates_new.count(), updates_old.count()
-        ))
-
-        # No. of organisations reporting to IATI
-        iati_orgs_new = IatiExport.objects.filter(created_at__lt=date_new)\
-                                          .annotate(project_count=Count('projects'))\
-                                          .filter(project_count__gt=0)\
-                                          .values_list('reporting_organisation_id', flat=True)\
-                                          .distinct()
-        iati_orgs_old = IatiExport.objects.filter(created_at__lt=date_old)\
-                                          .annotate(project_count=Count('projects'))\
-                                          .filter(project_count__gt=0)\
-                                          .values_list('reporting_organisation_id', flat=True)\
-                                          .distinct()
-        print('# of partners reporting to IATI,{},{}'.format(
-            iati_orgs_new.count(), iati_orgs_old.count()
-        ))
-
-        # No. of projects reporting to IATI
-        iati_projects_new = IatiExport.objects.filter(created_at__lt=date_new)\
-                                              .values_list('projects', flat=True)\
-                                              .distinct()
-        iati_projects_old = IatiExport.objects.filter(created_at__lt=date_old)\
-                                              .values_list('projects', flat=True)\
-                                              .distinct()
-        print('# of projects reporting to IATI,{},{}'.format(
-            iati_projects_new.count(), iati_projects_old.count()
-        ))
-
-        for org_id in sorted(set(iati_orgs_new)):
-            org = Organisation.objects.get(pk=org_id)
-            exports = IatiExport.objects.exclude(created_at__lte=date_new)\
-                                        .exclude(created_at__lt=date_old)\
-                                        .filter(reporting_organisation=org)
-            projects = set(exports.values_list('projects', flat=True))
-            if not projects:
-                continue
-            count = len(projects)
-            print(f"{org_id}\t{org.name}\t{count}")
+        print('')
+        orgs_reporting_to_iati_list = make_orgs_reporting_to_iati_list(orgs_reporting_to_iati, date_start, date_end)
+        print(orgs_reporting_to_iati_list.export('rst'))


Test patch: