Repository: learningequality/kolibri

Issue:
unhandled error in pingback

### Observed behavior

comparison between `None` and `datetime.datetime`

### Expected behavior

no unhandled errors

### User-facing consequences

Spurious errors in the logs

### Errors and logs

```
raceback (most recent call last):
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py", line 916, in _bootstrap_inner
    self.run()
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/utils/server.py", line 112, in run
    call_command("ping")
  File "/Users/d/Projects/learning_equality/repos/k012x/.venv/lib/python3.6/site-packages/django/core/management/__init__.py", line 131, in call_command
    return command.execute(*args, **defaults)
  File "/Users/d/Projects/learning_equality/repos/k012x/.venv/lib/python3.6/site-packages/django/core/management/base.py", line 330, in execute
    output = self.handle(*args, **options)
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/management/commands/ping.py", line 61, in handle
    self.perform_statistics(server, data["id"])
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/management/commands/ping.py", line 120, in perform_statistics
    facilities = [extract_facility_statistics(f) for f in Facility.objects.all()]
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/management/commands/ping.py", line 120, in <listcomp>
    facilities = [extract_facility_statistics(f) for f in Facility.objects.all()]
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/utils.py", line 68, in extract_facility_statistics
    first_interaction_timestamp = getattr(min(usersess_agg["first"], contsess_agg["first"]), 'strftime', None)
TypeError: '<' not supported between instances of 'NoneType' and 'datetime.datetime'
```

### Steps to reproduce

This happened when I ran `kolibri manage start` on my local dev setup

### Context

latest `develop`, macos
unhandled error in pingback

### Observed behavior

comparison between `None` and `datetime.datetime`

### Expected behavior

no unhandled errors

### User-facing consequences

Spurious errors in the logs

### Errors and logs

```
raceback (most recent call last):
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py", line 916, in _bootstrap_inner
    self.run()
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/utils/server.py", line 112, in run
    call_command("ping")
  File "/Users/d/Projects/learning_equality/repos/k012x/.venv/lib/python3.6/site-packages/django/core/management/__init__.py", line 131, in call_command
    return command.execute(*args, **defaults)
  File "/Users/d/Projects/learning_equality/repos/k012x/.venv/lib/python3.6/site-packages/django/core/management/base.py", line 330, in execute
    output = self.handle(*args, **options)
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/management/commands/ping.py", line 61, in handle
    self.perform_statistics(server, data["id"])
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/management/commands/ping.py", line 120, in perform_statistics
    facilities = [extract_facility_statistics(f) for f in Facility.objects.all()]
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/management/commands/ping.py", line 120, in <listcomp>
    facilities = [extract_facility_statistics(f) for f in Facility.objects.all()]
  File "/Users/d/Projects/learning_equality/repos/k012x/kolibri/core/analytics/utils.py", line 68, in extract_facility_statistics
    first_interaction_timestamp = getattr(min(usersess_agg["first"], contsess_agg["first"]), 'strftime', None)
TypeError: '<' not supported between instances of 'NoneType' and 'datetime.datetime'
```

### Steps to reproduce

This happened when I ran `kolibri manage start` on my local dev setup

### Context

latest `develop`, macos


Discussion:

Resolving patch:
diff --git a/kolibri/core/analytics/management/commands/ping.py b/kolibri/core/analytics/management/commands/ping.py
--- a/kolibri/core/analytics/management/commands/ping.py
+++ b/kolibri/core/analytics/management/commands/ping.py
@@ -8,6 +8,7 @@
 from django.core.management.base import BaseCommand
 from django.utils.six.moves.urllib.parse import urljoin
 from django.utils.timezone import get_current_timezone
+from django.utils.timezone import localtime
 from morango.models import InstanceIDModel
 from requests.exceptions import ConnectionError
 from requests.exceptions import RequestException
@@ -20,6 +21,7 @@
 from kolibri.core.auth.models import Facility
 from kolibri.core.content.models import ChannelMetadata
 from kolibri.core.device.models import DeviceSettings
+from kolibri.utils.server import installation_type
 from kolibri.utils.server import vacuum_db_lock
 
 logger = logging.getLogger(__name__)
@@ -100,6 +102,8 @@ def perform_ping(self, server):
             "language": language,
             "timezone": timezone,
             "uptime": int((datetime.now() - self.started).total_seconds() / 60),
+            "timestamp": localtime(),
+            "installer": installation_type(),
         }
 
         logger.debug("Pingback data: {}".format(data))
diff --git a/kolibri/core/analytics/utils.py b/kolibri/core/analytics/utils.py
--- a/kolibri/core/analytics/utils.py
+++ b/kolibri/core/analytics/utils.py
@@ -3,6 +3,7 @@
 import hashlib
 import json
 
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models import Count
 from django.db.models import Max
 from django.db.models import Min
@@ -34,7 +35,7 @@
 
 
 def dump_zipped_json(data):
-    jsondata = json.dumps(data)
+    jsondata = json.dumps(data, sort_keys=True, cls=DjangoJSONEncoder)
     try:
         # perform the import in here as zlib isn't available on some platforms
         import zlib
@@ -64,9 +65,13 @@ def extract_facility_statistics(facility):
                     .filter(start_timestamp__gt=datetime.datetime(2016, 1, 1))
                     .aggregate(first=Min("start_timestamp"), last=Max("end_timestamp")))
 
+    # extract the first and last times we've seen logs, ignoring any that are None
+    first_times = [d["first"] for d in [usersess_agg, contsess_agg] if d["first"]]
+    last_times = [d["last"] for d in [usersess_agg, contsess_agg] if d["last"]]
+
     # since newly provisioned devices won't have logs, we don't know whether we have an available datetime object
-    first_interaction_timestamp = getattr(min(usersess_agg["first"], contsess_agg["first"]), 'strftime', None)
-    last_interaction_timestamp = getattr(max(usersess_agg["last"], contsess_agg["last"]), 'strftime', None)
+    first_interaction_timestamp = getattr(min(first_times), 'strftime', None) if first_times else None
+    last_interaction_timestamp = getattr(max(last_times), 'strftime', None) if last_times else None
 
     sesslogs_by_kind = contsessions.order_by("kind").values("kind").annotate(count=Count("kind"))
     sesslogs_by_kind = {log["kind"]: log["count"] for log in sesslogs_by_kind}


Test patch:
diff --git a/kolibri/core/analytics/test/test_utils.py b/kolibri/core/analytics/test/test_utils.py
--- a/kolibri/core/analytics/test/test_utils.py
+++ b/kolibri/core/analytics/test/test_utils.py
@@ -219,6 +219,31 @@ def test_extract_facility_statistics(self):
         }
         assert actual == expected
 
+    def test_regression_4606_no_usersessions(self):
+        UserSessionLog.objects.all().delete()
+        facility = self.facilities[0]
+        # will raise an exception if we haven't addressed https://github.com/learningequality/kolibri/issues/4606
+        actual = extract_facility_statistics(facility)
+        assert actual["f"] == "2018-10-11"
+        assert actual["l"] == "2019-10-11"
+
+    def test_regression_4606_no_contentsessions(self):
+        ContentSessionLog.objects.all().delete()
+        facility = self.facilities[0]
+        # will raise an exception if we haven't addressed https://github.com/learningequality/kolibri/issues/4606
+        actual = extract_facility_statistics(facility)
+        assert actual["f"] == "2018-10-11"
+        assert actual["l"] == "2019-10-11"
+
+    def test_regression_4606_no_contentsessions_or_usersessions(self):
+        ContentSessionLog.objects.all().delete()
+        UserSessionLog.objects.all().delete()
+        facility = self.facilities[0]
+        # will raise an exception if we haven't addressed https://github.com/learningequality/kolibri/issues/4606
+        actual = extract_facility_statistics(facility)
+        assert actual["f"] is None
+        assert actual["l"] is None
+
 
 class ChannelStatisticsTestCase(BaseDeviceSetupMixin, TestCase):