Repository: mozilla/bugbug

Issue:
Same request is handled multiple times without ever returning 200
Extract from the log:
```
2020-03-04T17:01:37.739066+00:00 app[web.1]: 2020-03-04 17:01:37,738:INFO:root:Request with API TOKEN 'autonag'
2020-03-04T17:01:38.293559+00:00 heroku[router]: at=info method=POST path="//component/predict/batch" host=bugbug.herokuapp.com request_id=5f7a8e49-8ee8-4e43-af29-dec9bf95d924 fwd="95.235.139.29" dyno=web.1 connect=0ms service=553ms status=202 bytes=592 protocol=https
2020-03-04T17:01:38.286828+00:00 app[worker.1]: 2020-03-04 17:01:38,286:INFO:rq.worker:default: bugbug_http.models.classify_bug('component', [1598744, 1615281, 1566486, 1600164, 1616722, 1608725, 1608978, 1610169, 1569287, 1569289, 1602463, 1618941, 1545829, 1619699, 1571773], None) (cf380d479e774498added8c080d1cfda)
2020-03-04T17:01:39.949460+00:00 app[worker.1]: 2020-03-04 17:01:39,949:INFO:rq.worker:default: Job OK (cf380d479e774498added8c080d1cfda)
2020-03-04T17:01:39.949731+00:00 app[worker.1]: 2020-03-04 17:01:39,949:INFO:rq.worker:Result is kept for 500 seconds
2020-03-04T17:01:42.399853+00:00 heroku[worker.1]: Process running mem=2752M(104.7%)
2020-03-04T17:01:42.399853+00:00 heroku[worker.1]: Error R14 (Memory quota exceeded)
2020-03-04T17:01:45.955452+00:00 app[web.1]: 2020-03-04 17:01:45,955:INFO:root:Request with API TOKEN 'autonag'
2020-03-04T17:01:46.182893+00:00 heroku[router]: at=info method=POST path="//component/predict/batch" host=bugbug.herokuapp.com request_id=5fb46497-6af2-4f22-97ce-8cd8e1db24b4 fwd="95.235.139.29" dyno=web.1 connect=0ms service=228ms status=202 bytes=592 protocol=https
2020-03-04T17:01:46.184089+00:00 app[worker.1]: 2020-03-04 17:01:46,183:INFO:rq.worker:default: bugbug_http.models.classify_bug('component', [1598744, 1615281, 1566486, 1600164, 1616722, 1608725, 1608978, 1610169, 1569287, 1569289, 1602463, 1618941, 1545829, 1619699, 1571773], None) (7d404d5bc1d342a7a455a8a7dbfbc014)
2020-03-04T17:01:47.849524+00:00 app[worker.1]: 2020-03-04 17:01:47,849:INFO:rq.worker:default: Job OK (7d404d5bc1d342a7a455a8a7dbfbc014)
2020-03-04T17:01:47.849830+00:00 app[worker.1]: 2020-03-04 17:01:47,849:INFO:rq.worker:Result is kept for 500 seconds
```


Discussion:
Probably a regression from bb9c2fbf71752206e03420ef6f20e64f70057f72.
Looks like it schedules a new job, so there's probably some difference in the keys being generated for redis between the two.

I assume that "Memory quota exceeded" is unrelated?

Resolving patch:
diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py
--- a/http_service/bugbug_http/app.py
+++ b/http_service/bugbug_http/app.py
@@ -261,7 +261,7 @@ def clean_prediction_cache(job):
 
 
 def get_result(job):
-    LOGGER.debug(f"Checking for existing results for {job}")
+    LOGGER.debug(f"Checking for existing results at {job.result_key}")
     result = redis_conn.get(job.result_key)
 
     if result:
diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py
--- a/http_service/bugbug_http/models.py
+++ b/http_service/bugbug_http/models.py
@@ -42,6 +42,9 @@
 MODEL_CACHE: Dict[str, Model] = {}
 
 
+redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://localhost/0"))
+
+
 def get_model(model_name):
     if model_name not in MODEL_CACHE:
         print("Recreating the %r model in cache" % model_name)
@@ -74,9 +77,15 @@ def preload_models():
         get_model(model)
 
 
-def classify_bug(
-    model_name, bug_ids, bugzilla_token, expiration=DEFAULT_EXPIRATION_TTL
-):
+def setkey(key, value, expiration=DEFAULT_EXPIRATION_TTL):
+    LOGGER.debug(f"Storing data at {key}: {value}")
+    redis.set(key, value)
+
+    if expiration > 0:
+        redis.expire(key, expiration)
+
+
+def classify_bug(model_name, bug_ids, bugzilla_token):
     from bugbug_http.app import JobInfo
 
     # This should be called in a process worker so it should be safe to set
@@ -85,19 +94,14 @@ def classify_bug(
     bugzilla.set_token(bugzilla_token)
     bugs = bugzilla.get(bug_ids)
 
-    redis_url = os.environ.get("REDIS_URL", "redis://localhost/0")
-    redis = Redis.from_url(redis_url)
-
     missing_bugs = bug_ids_set.difference(bugs.keys())
 
     for bug_id in missing_bugs:
-        redis_key = f"result_{model_name}_{bug_id}"
+        job = JobInfo(classify_bug, model_name, bug_id)
 
         # TODO: Find a better error format
         encoded_data = json.dumps({"available": False})
-
-        redis.set(redis_key, encoded_data)
-        redis.expire(redis_key, expiration)
+        setkey(job.result_key, encoded_data)
 
     if not bugs:
         return "NOK"
@@ -131,18 +135,15 @@ def classify_bug(
         encoded_data = json.dumps(data)
 
         job = JobInfo(classify_bug, model_name, bug_id)
-        redis_key = job.result_key
-
-        redis.set(redis_key, encoded_data)
-        redis.expire(redis_key, expiration)
+        setkey(job.result_key, encoded_data)
 
         # Save the bug last change
-        redis.set(job.change_time_key, bugs[bug_id]["last_change_time"])
+        setkey(job.change_time_key, bugs[bug_id]["last_change_time"], expiration=0)
 
     return "OK"
 
 
-def schedule_tests(branch, rev, expiration=DEFAULT_EXPIRATION_TTL):
+def schedule_tests(branch, rev):
     from bugbug_http.app import JobInfo
 
     job = JobInfo(schedule_tests, branch, rev)
@@ -173,12 +174,6 @@ def get_data(revset):
 
     data = get_data(revset)
     encoded_data = json.dumps(data)
-
-    redis_url = os.environ.get("REDIS_URL", "redis://localhost/0")
-
-    redis_key = job.result_key
-    redis = Redis.from_url(redis_url)
-    redis.set(redis_key, encoded_data)
-    redis.expire(redis_key, expiration)
+    setkey(job.result_key, encoded_data)
 
     return "OK"


Test patch: