Repository: learningequality/kolibri

Issue:
analytics error: 'bytes' is not JSON serializable

### Observed behavior

Error occurs while Kolibri is trying to send stats

### Expected behavior

No error

### User-facing consequences

Analytics not recorded

### Errors and logs

```
INFO:kolibri.core.analytics.management.commands.ping:Ping succeeded! (response: {'id': 87135})
Exception in thread Thread-1:
Traceback (most recent call last):
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py", line 916, in _bootstrap_inner
    self.run()
  File "/Users/d/Projects/learning_equality/repos/k011x/kolibri/utils/server.py", line 109, in run
    call_command("ping")
  File "/Users/d/Projects/learning_equality/repos/k011x/.venv/lib/python3.6/site-packages/django/core/management/__init__.py", line 131, in call_command
    return command.execute(*args, **defaults)
  File "/Users/d/Projects/learning_equality/repos/k011x/.venv/lib/python3.6/site-packages/django/core/management/base.py", line 330, in execute
    output = self.handle(*args, **options)
  File "/Users/d/Projects/learning_equality/repos/k011x/kolibri/core/analytics/management/commands/ping.py", line 58, in handle
    self.perform_statistics(server, data["id"])
  File "/Users/d/Projects/learning_equality/repos/k011x/kolibri/core/analytics/management/commands/ping.py", line 123, in perform_statistics
    jsondata = dump_zipped_json(data)
  File "/Users/d/Projects/learning_equality/repos/k011x/kolibri/core/analytics/utils.py", line 37, in dump_zipped_json
    jsondata = json.dumps(data)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/json/__init__.py", line 231, in dumps
    return _default_encoder.encode(obj)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/json/encoder.py", line 199, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/json/encoder.py", line 257, in iterencode
    return _iterencode(o, 0)
  File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/json/encoder.py", line 180, in default
    o.__class__.__name__)
TypeError: Object of type 'bytes' is not JSON serializable
```

### Steps to reproduce

Start kolibri

### Context

Mac, Python 3, latest `release-v0.11.x`


-----

cc @ralphiee22 @jamalex 




Discussion:
On it! We got automated tests working for Python 3, but it seems the serialization wasn't covered by the tests. Will add a test and fix.

Resolving patch:
diff --git a/kolibri/core/analytics/management/commands/ping.py b/kolibri/core/analytics/management/commands/ping.py
--- a/kolibri/core/analytics/management/commands/ping.py
+++ b/kolibri/core/analytics/management/commands/ping.py
@@ -39,12 +39,15 @@ def add_arguments(self, parser):
                             help='Number of minutes to wait between failed ping attempts.')
         parser.add_argument('--server', action='store', dest='server',
                             help='Base URL of the server to connect to.')
+        parser.add_argument('--once', action='store_true', dest='once',
+                            help='Only try to ping once, then exit')
 
     def handle(self, *args, **options):
 
         interval = float(options.get("interval") or DEFAULT_PING_INTERVAL)
         checkrate = float(options.get("checkrate") or DEFAULT_PING_CHECKRATE)
         server = options.get("server") or DEFAULT_SERVER_URL
+        once = options.get("once") or False
 
         self.started = datetime.now()
 
@@ -56,6 +59,8 @@ def handle(self, *args, **options):
                     logger.info("Ping succeeded! (response: {})".format(data))
                     if "id" in data:
                         self.perform_statistics(server, data["id"])
+                if once:
+                    break
                 logger.info("Sleeping for {} minutes.".format(interval))
                 time.sleep(interval * 60)
                 continue
@@ -65,6 +70,8 @@ def handle(self, *args, **options):
                 logger.warn("Ping failed (connection timed out). Trying again in {} minutes.".format(checkrate))
             except RequestException as e:
                 logger.warn("Ping failed ({})! Trying again in {} minutes.".format(e, checkrate))
+            if once:
+                break
             time.sleep(checkrate * 60)
 
     def perform_ping(self, server):
@@ -103,7 +110,7 @@ def perform_ping(self, server):
 
         response.raise_for_status()
 
-        return json.loads(response.content or "{}")
+        return json.loads(response.content.decode() or "{}")
 
     def perform_statistics(self, server, pingback_id):
 
@@ -126,4 +133,4 @@ def perform_statistics(self, server, pingback_id):
 
         response.raise_for_status()
 
-        return json.loads(response.content or "{}")
+        return json.loads(response.content.decode() or "{}")
diff --git a/kolibri/core/analytics/utils.py b/kolibri/core/analytics/utils.py
--- a/kolibri/core/analytics/utils.py
+++ b/kolibri/core/analytics/utils.py
@@ -78,7 +78,7 @@ def extract_facility_statistics(facility):
 
     return {
         # facility_id
-        "fi": base64.encodestring(hashlib.md5(facility.id.encode()).digest())[:10],
+        "fi": base64.encodestring(hashlib.md5(facility.id.encode()).digest())[:10].decode(),
         # settings
         "s": settings,
         # learners_count


Test patch:
diff --git a/kolibri/core/analytics/test/test_ping.py b/kolibri/core/analytics/test/test_ping.py
new file mode 100644
--- /dev/null
+++ b/kolibri/core/analytics/test/test_ping.py
@@ -0,0 +1,59 @@
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import json
+import zlib
+
+import mock
+from django.core.management import call_command
+from django.test import TestCase
+from requests.models import Response
+
+from .test_utils import BaseDeviceSetupMixin
+
+
+def load_zipped_json(data):
+    try:
+        data = zlib.decompress(data)
+    except Exception:
+        pass
+    return json.loads(data)
+
+
+def mocked_requests_post_wrapper(json_data, status_code):
+
+    def mocked_requests_post(*args, **kwargs):
+
+        class MockResponse(Response):
+
+            def __init__(self):
+                self.json_data = json_data
+                self.status_code = status_code
+                self._content = json.dumps(json_data).encode()
+                self.reason = ""
+                self.url = args[0]
+
+            def json(self):
+                return self.json_data
+
+        return MockResponse()
+
+    return mocked_requests_post
+
+
+class PingCommandTestCase(BaseDeviceSetupMixin, TestCase):
+
+    @mock.patch('requests.post', side_effect=mocked_requests_post_wrapper({"id": 17}, 200))
+    def test_ping_succeeds(self, post_mock):
+        call_command("ping", once=True)
+        assert len(post_mock.call_args_list) == 2
+        assert post_mock.call_args_list[0][0][0].endswith("/pingback")
+        assert post_mock.call_args_list[1][0][0].endswith("/statistics")
+        assert load_zipped_json(post_mock.call_args_list[1][1]["data"])["pi"] == 17
+
+    @mock.patch('requests.post', side_effect=mocked_requests_post_wrapper({}, 400))
+    def test_ping_fails(self, post_mock):
+        call_command("ping", once=True)
+        assert len(post_mock.call_args_list) == 1
+        assert post_mock.call_args_list[0][0][0].endswith("/pingback")
diff --git a/kolibri/core/analytics/test/test_utils.py b/kolibri/core/analytics/test/test_utils.py
--- a/kolibri/core/analytics/test/test_utils.py
+++ b/kolibri/core/analytics/test/test_utils.py
@@ -185,7 +185,7 @@ def test_extract_facility_statistics(self):
         actual = extract_facility_statistics(facility)
         facility_id_hash = actual.pop('fi')
         # just assert the beginning hex values of the facility id don't match
-        self.assertFalse(facility_id_hash.startswith(facility.id[:3].encode()))
+        self.assertFalse(facility_id_hash.startswith(facility.id[:3]))
         expected = {
             "s": {
                 "preset": facility_presets.default,