Repository: mozilla/bugbug

Issue:
Don't fill queue with requests while the worker is booting
Here's an example failure of the integration test:
[live_backing.txt](https://github.com/mozilla/bugbug/files/4268644/live_backing.txt)

I think what's happening is that we enqueue a lot of requests while the workers are starting, and so we spend a lot of time doing them all.

We should queue jobs only once while the workers are not running.


Discussion:
It should in theory already be working with the `is_running` check, but apparently it isn't (or at least I think it isn't, let's see what happens with https://community-tc.services.mozilla.com/tasks/FdozjNfCRRCy_Gtk5qEFqg).
I ran into this too, I fixed it by returning true if status is "queued":
https://github.com/mozilla/bugbug/pull/1319/commits/7262584827fa94041adec7cfa24f14d04b711063#diff-596c6d06758c5d9b7ca20ffba2196bb2R197

I'll roll that change out into a separate PR that fixes this issue when I get a chance.
As part of this, we should also move https://github.com/mozilla/bugbug/commit/791c16ebe0f8a607ab194d477ce602d8ecd7d80b#diff-aec5ad9e6cf04fcf90061c24d3443d57R26 and https://github.com/mozilla/bugbug/commit/791c16ebe0f8a607ab194d477ce602d8ecd7d80b#diff-aec5ad9e6cf04fcf90061c24d3443d57R49 back to 1 second.

Resolving patch:
diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py
--- a/http_service/bugbug_http/app.py
+++ b/http_service/bugbug_http/app.py
@@ -118,26 +118,26 @@ def schedule_bug_classification(model_name, bug_ids):
     q.enqueue(classify_bug, model_name, bug_ids, BUGZILLA_TOKEN, job_id=job_id)
 
 
-def is_running(model_name, bug_id):
+def is_pending(model_name, bug_id):
     # Check if there is a job
     mapping_key = get_mapping_key(model_name, bug_id)
 
     job_id = redis_conn.get(mapping_key)
 
     if not job_id:
-        LOGGER.debug("No job ID mapping %s, False", job_id)
+        LOGGER.debug(f"No job ID mapping {job_id}, False")
         return False
 
     try:
         job = Job.fetch(job_id.decode("utf-8"), connection=redis_conn)
     except NoSuchJobError:
-        LOGGER.debug("No job in DB for %s, False", job_id)
+        LOGGER.debug(f"No job in DB for {job_id}, False")
         # The job might have expired from redis
         return False
 
     job_status = job.get_status()
-    if job_status == "started":
-        LOGGER.debug("Job running %s, True", job_id)
+    if job_status in ("started", "queued"):
+        LOGGER.debug(f"Job {job_id} has status {job_status}, True")
         return True
 
     # Enforce job timeout as RQ doesn't seems to do it https://github.com/rq/rq/issues/758
@@ -148,11 +148,11 @@ def is_running(model_name, bug_id):
         job.cancel()
         job.cleanup()
 
-        LOGGER.debug("Job timeout %s, False", job_id)
+        LOGGER.debug(f"Job timeout {job_id}, False")
 
         return False
 
-    LOGGER.debug("Job status %s, False", job_status)
+    LOGGER.debug(f"Job {job_id} has status {job_status}, False")
 
     return False
 
@@ -274,7 +274,7 @@ def model_prediction(model_name, bug_id):
     data = get_bug_classification(model_name, bug_id)
 
     if not data:
-        if not is_running(model_name, bug_id):
+        if not is_pending(model_name, bug_id):
             schedule_bug_classification(model_name, [bug_id])
         status_code = 202
         data = {"ready": False}
@@ -438,7 +438,7 @@ def batch_prediction(model_name):
 
         data[str(bug_id)] = get_bug_classification(model_name, bug_id)
         if not data[str(bug_id)]:
-            if not is_running(model_name, bug_id):
+            if not is_pending(model_name, bug_id):
                 missing_bugs.append(bug_id)
             status_code = 202
             data[str(bug_id)] = {"ready": False}


Test patch:
diff --git a/http_service/tests/test_integration.py b/http_service/tests/test_integration.py
--- a/http_service/tests/test_integration.py
+++ b/http_service/tests/test_integration.py
@@ -13,7 +13,8 @@
 
 # Test classifying a single bug.
 def integration_test_single():
-    for _ in range(600):
+    timeout = 600
+    for _ in range(timeout):
         response = requests.get(
             f"{BUGBUG_HTTP_SERVER}/defectenhancementtask/predict/1376406",
             headers={"X-Api-Key": "integration_test_single"},
@@ -22,12 +23,12 @@ def integration_test_single():
         if response.status_code == 200:
             break
 
-        time.sleep(30)
+        time.sleep(1)
 
     response_json = response.json()
 
     if not response.ok:
-        raise Exception(f"Couldn't get an answer in 600 seconds: {response_json}")
+        raise Exception(f"Couldn't get an answer in {timeout} seconds: {response_json}")
 
     print("Response for bug 1376406", response_json)
     assert response_json["class"] is not None
@@ -35,7 +36,8 @@ def integration_test_single():
 
 # Test classifying a batch of bugs.
 def integration_test_batch():
-    for _ in range(100):
+    timeout = 100
+    for _ in range(timeout):
         response = requests.post(
             f"{BUGBUG_HTTP_SERVER}/defectenhancementtask/predict/batch",
             headers={"X-Api-Key": "integration_test_batch"},
@@ -45,12 +47,12 @@ def integration_test_batch():
         if response.status_code == 200:
             break
 
-        time.sleep(30)
+        time.sleep(1)
 
     response_json = response.json()
 
     if not response.ok:
-        raise Exception(f"Couldn't get an answer in 100 seconds: {response_json}")
+        raise Exception(f"Couldn't get an answer in {timeout} seconds: {response_json}")
 
     response_1376544 = response_json["bugs"]["1376544"]
     print("Response for bug 1376544", response_1376544)