Repository: mozilla/bugbug

Issue:
Add the scaffolding needed to add an API call to select tests
This issue is just about adding an API call (with a hardcoded result) to the http service, preparing the ground for the actual API.

It should be similar to the bug classification API call (`model_prediction`), but its input will be a repository and a commit hash.

As part of this issue, we should:
- add the API definition (in http_service/app.py);
- the implementation of queuing the classification from the web dyno to the worker dyno using rq (again, in http_service/app.py, like we do for bugs).
- a `select_tests` function in http_service/models.py which for now returns a hardcoded result.

Note: the implementation should be simpler than the one for bugs, as we don't need (actually, we don't want to!) cache results.

Blocks #632.


Discussion:
@La0 would it help if I took a look at this issue while you worked on #1299 ? Or do you think you would have this finished in a couple days anyway?
I'm on PTO tommorow, will start looking into these issues on Wednesday.
I *should* have something by the end of the week, if that's too slow, feel free to re-assign the issues.
Ok I don't have permission to re-assign, but I'm going to take a look at getting a head start here. If I'm not done by the time you're ready to work on it.. I can upload what I have and you can take over again if you like.
@ahal I've assigned the issue to you, as I see some commits from your fork linked to this issue.

Resolving patch:
diff --git a/http_service/bugbug_http/app.py b/http_service/bugbug_http/app.py
--- a/http_service/bugbug_http/app.py
+++ b/http_service/bugbug_http/app.py
@@ -7,7 +7,9 @@
 import logging
 import os
 import uuid
+from dataclasses import dataclass, field
 from datetime import datetime, timedelta
+from typing import Any, Callable, List
 
 from apispec import APISpec
 from apispec.ext.marshmallow import MarshmallowPlugin
@@ -22,7 +24,7 @@
 from rq.job import Job
 
 from bugbug import get_bugbug_version
-from bugbug_http.models import MODELS_NAMES, change_time_key, classify_bug, result_key
+from bugbug_http.models import MODELS_NAMES, classify_bug, schedule_tests
 from bugbug_http.utils import get_bugzilla_http_client
 
 API_TOKEN = "X-Api-Key"
@@ -73,7 +75,7 @@ class BugPrediction(Schema):
     extra_data = fields.Dict()
 
 
-class BugPredictionNotAvailableYet(Schema):
+class NotAvailableYet(Schema):
     ready = fields.Boolean(enum=[False])
 
 
@@ -85,44 +87,101 @@ class UnauthorizedError(Schema):
     message = fields.Str(default="Error, missing X-API-KEY")
 
 
+class BranchName(Schema):
+    branch = fields.Str(example="autoland")
+
+
+class Schedules(Schema):
+    tasks = fields.List(fields.Str)
+    groups = fields.List(fields.Str)
+
+
 spec.components.schema(BugPrediction.__name__, schema=BugPrediction)
-spec.components.schema(
-    BugPredictionNotAvailableYet.__name__, schema=BugPredictionNotAvailableYet
-)
+spec.components.schema(NotAvailableYet.__name__, schema=NotAvailableYet)
 spec.components.schema(ModelName.__name__, schema=ModelName)
 spec.components.schema(UnauthorizedError.__name__, schema=UnauthorizedError)
+spec.components.schema(BranchName.__name__, schema=BranchName)
+spec.components.schema(Schedules.__name__, schema=Schedules)
 
 
 api_key_scheme = {"type": "apiKey", "in": "header", "name": "X-API-Key"}
 spec.components.security_scheme("api_key", api_key_scheme)
 
 
+@dataclass(init=False, frozen=True)
+class JobInfo:
+    func: Callable[..., str]
+    args: List[Any] = field(default_factory=list)
+
+    def __init__(self, func, *args):
+        # Custom __init__ is needed to support *args, and object.__setattr__ is
+        # needed for 'frozen=True'.
+        object.__setattr__(self, "func", func)
+        object.__setattr__(self, "args", args)
+
+    def __str__(self):
+        return f"{self.func.__name__}:{'_'.join(map(str, self.args))}"
+
+    @property
+    def mapping_key(self):
+        """The mapping key for this job.
+
+        Returns:
+            (str) A key to be used to for job ids.
+        """
+        return f"bugbug:job_id:{self}"
+
+    @property
+    def result_key(self):
+        """The result key for this job.
+
+        Returns:
+            (str) A key to be used to for job results.
+        """
+        return f"bugbug:job_result:{self}"
+
+    @property
+    def change_time_key(self):
+        """The change time key for this job.
+
+        Returns:
+            (str) A key to be used to for job change time.
+        """
+        return f"bugbug:change_time:{self}"
+
+
 def get_job_id():
     return uuid.uuid4().hex
 
 
-def get_mapping_key(model_name, bug_id):
-    return f"bugbug:mapping_{model_name}_{bug_id}"
+def schedule_job(job, job_id=None):
+    job_id = job_id or get_job_id()
+
+    redis_conn.mset({job.mapping_key: job_id})
+    q.enqueue(job.func, *job.args, job_id=job_id)
 
 
 def schedule_bug_classification(model_name, bug_ids):
     """ Schedule the classification of a bug_id list
     """
-
     job_id = get_job_id()
 
     # Set the mapping before queuing to avoid some race conditions
-    job_id_mapping = {get_mapping_key(model_name, bug_id): job_id for bug_id in bug_ids}
+    job_id_mapping = {}
+    for bug_id in bug_ids:
+        key = JobInfo(classify_bug, model_name, bug_id).mapping_key
+        job_id_mapping[key] = job_id
+
     redis_conn.mset(job_id_mapping)
 
-    q.enqueue(classify_bug, model_name, bug_ids, BUGZILLA_TOKEN, job_id=job_id)
+    schedule_job(
+        JobInfo(classify_bug, model_name, bug_ids, BUGZILLA_TOKEN), job_id=job_id
+    )
 
 
-def is_pending(model_name, bug_id):
+def is_pending(job):
     # Check if there is a job
-    mapping_key = get_mapping_key(model_name, bug_id)
-
-    job_id = redis_conn.get(mapping_key)
+    job_id = redis_conn.get(job.mapping_key)
 
     if not job_id:
         LOGGER.debug(f"No job ID mapping {job_id}, False")
@@ -178,16 +237,14 @@ def get_bugs_last_change_time(bug_ids):
     return bugs
 
 
-def is_prediction_invalidated(model_name, bug_id, change_time):
+def is_prediction_invalidated(job, change_time):
     # First get the saved change time
-    change_key = change_time_key(model_name, bug_id)
-
-    saved_change_time = redis_conn.get(change_key)
+    saved_change_time = redis_conn.get(job.change_time_key)
 
     # If we have no last changed time, the bug was not classified yet or the bug was classified by an old worker
     if not saved_change_time:
         # We can have a result without a cache time
-        if redis_conn.exists(result_key(model_name, bug_id)):
+        if redis_conn.exists(job.result_key):
             return True
 
         return False
@@ -195,19 +252,20 @@ def is_prediction_invalidated(model_name, bug_id, change_time):
     return saved_change_time.decode("utf-8") != change_time
 
 
-def clean_prediction_cache(model_name, bug_id):
+def clean_prediction_cache(job):
     # If the bug was modified since last time we classified it, clear the cache to avoid stale answer
-    LOGGER.debug("Cleaning results for bug id %s and model %s", bug_id, model_name)
+    LOGGER.debug(f"Cleaning results for {job}")
 
-    redis_conn.delete(result_key(model_name, bug_id))
-    redis_conn.delete(change_time_key(model_name, bug_id))
+    redis_conn.delete(job.result_key)
+    redis_conn.delete(job.change_time_key)
 
 
-def get_bug_classification(model_name, bug_id):
-    redis_key = f"result_{model_name}_{bug_id}"
-    result = redis_conn.get(redis_key)
+def get_result(job):
+    LOGGER.debug(f"Checking for existing results for {job}")
+    result = redis_conn.get(job.result_key)
 
     if result:
+        LOGGER.debug(f"Found {result}")
         return json.loads(result)
 
     return None
@@ -240,12 +298,7 @@ def model_prediction(model_name, bug_id):
           description: A temporary answer for the bug being processed
           content:
             application/json:
-              schema:
-                type: object
-                properties:
-                  ready:
-                    type: boolean
-                    enum: [False]
+              schema: NotAvailableYet
         401:
           description: API key is missing
           content:
@@ -266,15 +319,16 @@ def model_prediction(model_name, bug_id):
     bug = get_bugs_last_change_time([bug_id])
 
     # Change time could be None if it's a security bug
+    job = JobInfo(classify_bug, model_name, bug_id)
     bug_change_time = bug.get(bug_id, None)
-    if bug_change_time and is_prediction_invalidated(model_name, bug_id, bug[bug_id]):
-        clean_prediction_cache(model_name, bug_id)
+    if bug_change_time and is_prediction_invalidated(job, bug[bug_id]):
+        clean_prediction_cache(job)
 
     status_code = 200
-    data = get_bug_classification(model_name, bug_id)
+    data = get_result(job)
 
     if not data:
-        if not is_pending(model_name, bug_id):
+        if not is_pending(job):
             schedule_bug_classification(model_name, [bug_id])
         status_code = 202
         data = {"ready": False}
@@ -430,15 +484,16 @@ def batch_prediction(model_name):
     bug_change_dates = get_bugs_last_change_time(bugs)
 
     for bug_id in bugs:
+        job = JobInfo(classify_bug, model_name, bug_id)
 
         change_time = bug_change_dates.get(int(bug_id), None)
         # Change time could be None if it's a security bug
-        if change_time and is_prediction_invalidated(model_name, bug_id, change_time):
-            clean_prediction_cache(model_name, bug_id)
+        if change_time and is_prediction_invalidated(job, change_time):
+            clean_prediction_cache(job)
 
-        data[str(bug_id)] = get_bug_classification(model_name, bug_id)
+        data[str(bug_id)] = get_result(job)
         if not data[str(bug_id)]:
-            if not is_pending(model_name, bug_id):
+            if not is_pending(job):
                 missing_bugs.append(bug_id)
             status_code = 202
             data[str(bug_id)] = {"ready": False}
@@ -452,6 +507,65 @@ def batch_prediction(model_name):
     return jsonify({"bugs": data}), status_code
 
 
+@application.route("/push/<path:branch>/<rev>/schedules")
+@cross_origin()
+def push_schedules(branch, rev):
+    """
+    ---
+    get:
+      description: Determine which tests and tasks a push should schedule.
+      summary: Get which tests and tasks to schedule.
+      parameters:
+      - name: branch
+        in: path
+        schema:
+          BranchName
+      - name: rev
+        in: path
+        schema:
+          type: str
+          example: 76383a875678
+      responses:
+        200:
+          description: A dict of tests and tasks to schedule.
+          content:
+            application/json:
+              schema: Schedules
+        202:
+          description: Request is still being processed.
+          content:
+            application/json:
+              schema: NotAvailableYet
+        401:
+          description: API key is missing
+          content:
+            application/json:
+              schema: UnauthorizedError
+    """
+    headers = request.headers
+    redis_conn.ping()
+
+    auth = headers.get(API_TOKEN)
+
+    if not auth:
+        return jsonify(UnauthorizedError().dump({})), 401
+    else:
+        LOGGER.info("Request with API TOKEN %r", auth)
+
+    # Support the string 'autoland' for convenience.
+    if branch == "autoland":
+        branch = "integration/autoland"
+
+    job = JobInfo(schedule_tests, branch, rev)
+    data = get_result(job)
+    if data:
+        return jsonify(data), 200
+
+    if not is_pending(job):
+        schedule_job(job)
+    return jsonify({"ready": False}), 202
+
+
 @application.route("/swagger")
 @cross_origin()
 def swagger():
diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py
--- a/http_service/bugbug_http/models.py
+++ b/http_service/bugbug_http/models.py
@@ -9,6 +9,7 @@
 from datetime import datetime
 from typing import Dict
 
+import requests
 from dateutil.relativedelta import relativedelta
 from redis import Redis
 
@@ -41,14 +42,6 @@
 MODEL_CACHE: Dict[str, Model] = {}
 
 
-def result_key(model_name, bug_id):
-    return f"result_{model_name}_{bug_id}"
-
-
-def change_time_key(model_name, bug_id):
-    return f"bugbug:change_time_{model_name}_{bug_id}"
-
-
 def get_model(model_name):
     if model_name not in MODEL_CACHE:
         print("Recreating the %r model in cache" % model_name)
@@ -84,6 +77,8 @@ def preload_models():
 def classify_bug(
     model_name, bug_ids, bugzilla_token, expiration=DEFAULT_EXPIRATION_TTL
 ):
+    from bugbug_http.app import JobInfo
+
     # This should be called in a process worker so it should be safe to set
     # the token here
     bug_ids_set = set(map(int, bug_ids))
@@ -135,13 +130,55 @@ def classify_bug(
 
         encoded_data = json.dumps(data)
 
-        redis_key = result_key(model_name, bug_id)
+        job = JobInfo(classify_bug, model_name, bug_id)
+        redis_key = job.result_key
 
         redis.set(redis_key, encoded_data)
         redis.expire(redis_key, expiration)
 
         # Save the bug last change
-        change_key = change_time_key(model_name, bug_id)
-        redis.set(change_key, bugs[bug_id]["last_change_time"])
+        redis.set(job.change_time_key, bugs[bug_id]["last_change_time"])
+
+    return "OK"
+
+
+def schedule_tests(branch, rev, expiration=DEFAULT_EXPIRATION_TTL):
+    from bugbug_http.app import JobInfo
+
+    job = JobInfo(schedule_tests, branch, rev)
+    LOGGER.debug("Processing {job}")
+
+    url = f"https://hg.mozilla.org/{branch}/json-automationrelevance/{rev}"
+    r = requests.get(url)
+
+    if r.status_code == 404:
+        LOGGER.warning(f"Push not found at {url}!")
+        return "NOK"
+
+    first_rev = r.json()["changesets"][0]["node"]
+    if first_rev != rev:
+        revset = f"{first_rev}::{rev}"
+    else:
+        revset = rev
+
+    # TODO Return real data based on 'revset'
+    def get_data(revset):
+        return {
+            "tasks": ["test-macosx1014-64/debug-gtest-1proc"],
+            "groups": [
+                "caps/test/unit/xpcshell.ini",
+                "dom/indexedDB/test/mochitest.ini",
+            ],
+        }
+
+    data = get_data(revset)
+    encoded_data = json.dumps(data)
+
+    redis_url = os.environ.get("REDIS_URL", "redis://localhost/0")
+
+    redis_key = job.result_key
+    redis = Redis.from_url(redis_url)
+    redis.set(redis_key, encoded_data)
+    redis.expire(redis_key, expiration)
 
     return "OK"


Test patch:
diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/http_service/tests/conftest.py
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pytest
+
+from bugbug_http.app import application
+
+
+@pytest.fixture
+def client():
+    yield application.test_client()
diff --git a/http_service/tests/test_app.py b/http_service/tests/test_app.py
--- a/http_service/tests/test_app.py
+++ b/http_service/tests/test_app.py
@@ -5,14 +5,7 @@
 
 import json
 
-import pytest
-
-from bugbug_http.app import API_TOKEN, application
-
-
-@pytest.fixture
-def client():
-    yield application.test_client()
+from bugbug_http.app import API_TOKEN
 
 
 def test_empty_batch(client):
diff --git a/http_service/tests/test_push_schedules.py b/http_service/tests/test_push_schedules.py
new file mode 100644
--- /dev/null
+++ b/http_service/tests/test_push_schedules.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import logging
+import time
+
+import pytest
+from rq.exceptions import NoSuchJobError
+
+from bugbug_http import app
+
+# { <job id>: <result key> }
+JOBS = {}
+
+
+class RedisMock:
+    """Mock class to mimic a Redis database."""
+
+    def __init__(self):
+        self.data = {}
+
+    def set(self, k, v):
+        self.data[k] = v.encode("utf-8")
+
+    def mset(self, d):
+        for k, v in d.items():
+            self.set(k, v)
+
+    def get(self, k):
+        return self.data.get(k)
+
+    def exists(self, k):
+        return k in self.data
+
+    def delete(self, k):
+        del self.data[k]
+
+    def ping(self):
+        pass
+
+
+class QueueMock:
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def enqueue(self, func, *args, job_id=None):
+        job = app.JobInfo(func, *args)
+        JOBS[job_id] = job.result_key
+
+
+class JobMock:
+    def __init__(self, job_id):
+        self.job_id = job_id
+        self.enqueued_at = time.time()
+        self.timeout = 10
+
+    @staticmethod
+    def fetch(job_id, **kwargs):
+        if job_id in JOBS:
+            return JobMock(job_id)
+
+        raise NoSuchJobError
+
+    def get_status(self):
+        if self.job_id not in JOBS:
+            raise NoSuchJobError
+
+        result_key = JOBS[self.job_id]
+        if result_key in app.redis_conn.data:
+            return "finished"
+        return "started"
+
+    def cancel(self):
+        if self.job_id in JOBS:
+            del JOBS[self.job_id]
+
+    def cleanup(self):
+        pass
+
+
+@pytest.fixture(autouse=True)
+def patch_resources(monkeypatch):
+    global JOBS
+    JOBS = {}
+
+    app.LOGGER.setLevel(logging.DEBUG)
+    monkeypatch.setattr(app, "redis_conn", RedisMock())
+    monkeypatch.setattr(app, "q", QueueMock())
+    monkeypatch.setattr(app, "Job", JobMock)
+
+
+@pytest.fixture
+def add_result():
+    """Fixture that can be called to simulate a worker finishing a job."""
+
+    def inner(job_id, data):
+        result_key = JOBS.pop(job_id)
+        app.redis_conn.set(result_key, json.dumps(data))
+
+    return inner
+
+
+def test_queue_job_valid(client, add_result):
+    # schedule job
+    rv = client.get("/push/autoland/abcdef/schedules", headers={app.API_TOKEN: "test"},)
+
+    assert rv.status_code == 202
+    assert rv.json == {"ready": False}
+
+    # still not ready
+    rv = client.get("/push/autoland/abcdef/schedules", headers={app.API_TOKEN: "test"},)
+
+    assert rv.status_code == 202
+    assert rv.json == {"ready": False}
+
+    # job done
+    result = {
+        "groups": ["foo/mochitest.ini", "bar/xpcshell.ini"],
+        "tasks": ["test-linux/opt-mochitest-1"],
+    }
+    job_id = list(JOBS.keys())[0]
+    add_result(job_id, result)
+
+    rv = client.get("/push/autoland/abcdef/schedules", headers={app.API_TOKEN: "test"},)
+    assert rv.status_code == 200
+    assert rv.json == result
+
+
+def test_no_api_key(client):
+    rv = client.get("/push/autoland/foobar/schedules")
+
+    assert rv.status_code == 401
+    assert rv.json == {"message": "Error, missing X-API-KEY"}