Repository: mozilla/bugbug

Issue:
Store compressed results in Redis
This would reduce the size of the data in Redis, and allow us to cache more results.

See also #1532.


Discussion:

Resolving patch:
diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py
--- a/http_service/bugbug_http/app.py
+++ b/http_service/bugbug_http/app.py
@@ -13,6 +13,7 @@
 
 import libmozdata
 import orjson
+import zstandard
 from apispec import APISpec
 from apispec.ext.marshmallow import MarshmallowPlugin
 from apispec_webframeworks.flask import FlaskPlugin
@@ -78,6 +79,7 @@
     + "/rest/bug"
 )
 
+dctx = zstandard.ZstdDecompressor()
 
 logging.basicConfig(level=logging.DEBUG)
 LOGGER = logging.getLogger()
@@ -296,7 +298,7 @@ def get_result(job):
 
     if result:
         LOGGER.debug(f"Found {result}")
-        return orjson.loads(result)
+        return orjson.loads(dctx.decompress(result))
 
     return None
 
diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py
--- a/http_service/bugbug_http/models.py
+++ b/http_service/bugbug_http/models.py
@@ -11,6 +11,7 @@
 
 import orjson
 import requests
+import zstandard
 from redis import Redis
 
 from bugbug import bugzilla, repository
@@ -40,9 +41,13 @@
 )
 MODEL_CACHE.start_ttl_thread()
 
+cctx = zstandard.ZstdCompressor(level=10)
 
-def setkey(key: str, value: bytes) -> None:
+
+def setkey(key: str, value: bytes, compress: bool = False) -> None:
     LOGGER.debug(f"Storing data at {key}: {value!r}")
+    if compress:
+        value = cctx.compress(value)
     redis.set(key, value)
     redis.expire(key, DEFAULT_EXPIRATION_TTL)
 
@@ -94,7 +99,7 @@ def classify_bug(model_name: str, bug_ids: Collection[int], bugzilla_token: str)
         }
 
         job = JobInfo(classify_bug, model_name, bug_id)
-        setkey(job.result_key, orjson.dumps(data))
+        setkey(job.result_key, orjson.dumps(data), compress=True)
 
         # Save the bug last change
         setkey(job.change_time_key, bugs[bug_id]["last_change_time"].encode())
@@ -167,6 +172,6 @@ def schedule_tests(branch: str, rev: str) -> str:
         "reduced_tasks_higher": {t: c for t, c in tasks.items() if t in reduced_higher},
         "known_tasks": get_known_tasks(),
     }
-    setkey(job.result_key, orjson.dumps(data))
+    setkey(job.result_key, orjson.dumps(data), compress=True)
 
     return "OK"


Test patch:
diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py
--- a/http_service/tests/conftest.py
+++ b/http_service/tests/conftest.py
@@ -14,9 +14,11 @@
 
 import hglib
 import numpy as np
+import orjson
 import py.path
 import pytest
 import responses
+import zstandard
 from _pytest.monkeypatch import MonkeyPatch
 from rq.exceptions import NoSuchJobError
 
@@ -141,7 +143,8 @@ def add_result():
 
     def inner(key, data, change_time=None):
         result_key = f"bugbug:job_result:{key}"
-        app.redis_conn.set(result_key, json.dumps(data))
+        cctx = zstandard.ZstdCompressor(level=10)
+        app.redis_conn.set(result_key, cctx.compress(orjson.dumps(data)))
 
     return inner
 
diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py
--- a/http_service/tests/test_schedule_tests.py
+++ b/http_service/tests/test_schedule_tests.py
@@ -8,6 +8,7 @@
 
 import hglib
 import pytest
+import zstandard
 
 from bugbug_http import models
 
@@ -94,7 +95,9 @@ def test_simple_schedule(
 
     # Assert the test selection result is stored in Redis.
     result = json.loads(
-        models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}")
+        zstandard.ZstdDecompressor().decompress(
+            models.redis.get(f"bugbug:job_result:schedule_tests:mozilla-central_{rev}")
+        )
     )
     assert len(result) == 6
     assert result["tasks"] == labels_to_choose