Repository: mozilla/bugbug

Issue:
In the API to select tests, apply the model to select tests
Depends on #1301.

We should:
1) Analyze the patches relevant for the try push (the non public ones? or the ones coming from json-automationrelevance?), as https://github.com/mozilla/bugbug/blob/cb51deac6235477e7f2dbc13352a2627d129451c/scripts/commit_classifier.py#L575-L580;
2) Apply the model to select tests, as https://github.com/mozilla/bugbug/blob/cb51deac6235477e7f2dbc13352a2627d129451c/scripts/commit_classifier.py#L614-L694.

Blocks #632.


Discussion:

Resolving patch:
diff --git a/bugbug/repository.py b/bugbug/repository.py
--- a/bugbug/repository.py
+++ b/bugbug/repository.py
@@ -1048,6 +1048,9 @@ def get_base():
 
         return "tip"
 
+    def first_in_stack():
+        return hg.log(revrange=f"-{len(stack)}")[0].node.decode("utf-8")
+
     # Start by cleaning the repo, without pulling
     clean(repo_dir, pull=False)
 
@@ -1062,7 +1065,7 @@ def get_base():
         # Apply all the patches in the stack on current base
         if apply_patches(base, patches):
             logger.info(f"Stack applied successfully on {base}")
-            return
+            return first_in_stack()
 
         # We tried to apply on the valid parent and failed: cannot try another revision
         if base != "tip":
@@ -1081,6 +1084,8 @@ def get_base():
 
         logger.info(f"Stack applied successfully on {new_base}")
 
+        return first_in_stack()
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py
--- a/http_service/bugbug_http/models.py
+++ b/http_service/bugbug_http/models.py
@@ -9,14 +9,14 @@
 from datetime import datetime
 from typing import Dict
 
+import numpy as np
 import requests
 from dateutil.relativedelta import relativedelta
 from redis import Redis
 
-from bugbug import bugzilla
+from bugbug import bugzilla, commit_features, repository, test_scheduling
 from bugbug.model import Model
 from bugbug.models import load_model
-from bugbug.repository import apply_stack
 from bugbug_http import ALLOW_MISSING_MODELS
 from bugbug_http.utils import get_hgmo_stack
 
@@ -157,29 +157,47 @@ def schedule_tests(branch, rev):
 
     # Apply the stack on the local repository
     try:
-        apply_stack(REPO_DIR, stack, branch)
+        first_rev = repository.apply_stack(REPO_DIR, stack, branch)
     except Exception as e:
         LOGGER.warning(f"Failed to apply stack {branch} @ {rev}: {e}")
         return "NOK"
 
-    first_rev = stack[0]["node"]
-    if first_rev != rev:
-        revset = f"{first_rev}::{rev}"
-    else:
-        revset = rev
-
-    # TODO Return real data based on 'revset'
-    def get_data(revset):
-        return {
-            "tasks": ["test-macosx1014-64/debug-gtest-1proc"],
-            "groups": [
-                "caps/test/unit/xpcshell.ini",
-                "dom/indexedDB/test/mochitest.ini",
-            ],
-        }
+    test_selection_threshold = float(
+        os.environ.get("TEST_SELECTION_CONFIDENCE_THRESHOLD", 0.5)
+    )
+
+    # Analyze patches.
+    commits = repository.download_commits(REPO_DIR, rev_start=first_rev, save=False)
+
+    commit_data = commit_features.merge_commits(commits)
+
+    def get_runnables(granularity):
+        past_failures_data = test_scheduling.get_past_failures(granularity)
+
+        push_num = past_failures_data["push_num"]
+        all_runnables = past_failures_data["all_runnables"]
+
+        commit_tests = []
+        for data in test_scheduling.generate_data(
+            past_failures_data, commit_data, push_num, all_runnables, [], []
+        ):
+            if granularity == "label" and not data["name"].startswith("test-"):
+                continue
+
+            commit_test = commit_data.copy()
+            commit_test["test_job"] = data
+            commit_tests.append(commit_test)
+
+        probs = get_model(f"test{granularity}select").classify(
+            commit_tests, probabilities=True
+        )
+        selected_indexes = np.argwhere(probs[:, 1] > test_selection_threshold)[:, 0]
+        return [commit_tests[i]["test_job"]["name"] for i in selected_indexes]
 
-    data = get_data(revset)
-    encoded_data = json.dumps(data)
-    setkey(job.result_key, encoded_data)
+    data = {
+        "tasks": get_runnables("label"),
+        "groups": get_runnables("group"),
+    }
+    setkey(job.result_key, json.dumps(data))
 
     return "OK"


Test patch:
diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py
--- a/http_service/tests/conftest.py
+++ b/http_service/tests/conftest.py
@@ -11,12 +11,14 @@
 from datetime import datetime
 
 import hglib
+import numpy as np
 import pytest
 import responses
 from rq.exceptions import NoSuchJobError
 
 import bugbug_http
 import bugbug_http.models
+from bugbug import test_scheduling
 from bugbug_http import app
 
 FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures")
@@ -220,6 +222,8 @@ def mock_repo(tmpdir, monkeypatch):
     # Create the repo
     hglib.init(str(local_dir))
 
+    (local_dir / ".hg-annotate-ignore-revs").write_text("", encoding="ascii")
+
     # Add several commits on a test file to create some history
     test_file = local_dir / "test.txt"
     with hglib.open(str(local_dir)) as repo:
@@ -242,4 +246,69 @@ def mock_repo(tmpdir, monkeypatch):
         repo.add([str(remote).encode("utf-8")])
         repo.commit("Pulled from remote", user="bugbug")
 
+    # Allow using the local code analysis server.
+    responses.add_passthru("http://127.0.0.1")
+
     return local_dir, remote_dir
+
+
+@pytest.fixture
+def mock_component_taskcluster_artifact():
+    responses.add(
+        responses.HEAD,
+        "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.source.source-bugzilla-info/artifacts/public/components.json",
+        status=200,
+        headers={"ETag": "100"},
+    )
+
+    responses.add(
+        responses.GET,
+        "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.source.source-bugzilla-info/artifacts/public/components.json",
+        status=200,
+        json={},
+    )
+
+
+@pytest.fixture
+def mock_schedule_tests_classify(tmpdir, monkeypatch):
+    os.chdir(tmpdir)
+    os.makedirs("data")
+
+    # Initialize a mock past failures DB.
+    for granularity in ("label", "group"):
+        past_failures_data = test_scheduling.get_past_failures(granularity)
+        past_failures_data["push_num"] = 1
+        past_failures_data["all_runnables"] = [
+            f"test-{granularity}1",
+            f"test-{granularity}2",
+        ]
+        past_failures_data.close()
+
+    def do_mock(labels_to_choose, groups_to_choose):
+        # Add a mock test selection model.
+        class Model:
+            def __init__(self, name):
+                self.name = name
+
+            def classify(self, items, probabilities=False):
+                assert probabilities
+                results = []
+                for item in items:
+                    if self.name == "testlabelselect":
+                        if item["test_job"]["name"] in labels_to_choose:
+                            results.append([0.1, 0.9])
+                        else:
+                            results.append([0.9, 0.1])
+                    elif self.name == "testgroupselect":
+                        if item["test_job"]["name"] in groups_to_choose:
+                            results.append([0.1, 0.9])
+                        else:
+                            results.append([0.9, 0.1])
+                return np.array(results)
+
+        def mock_get_model(modelname):
+            return Model(modelname)
+
+        monkeypatch.setattr(bugbug_http.models, "get_model", mock_get_model)
+
+    return do_mock
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/localOnRemote.diff b/http_service/tests/fixtures/hgmo_integration-autoland/localOnRemote.diff
--- a/http_service/tests/fixtures/hgmo_integration-autoland/localOnRemote.diff
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/localOnRemote.diff
@@ -1,4 +1,4 @@
-Based on remote
+Bug 123 - Based on remote
 
 diff --git a/remote.txt b/remote.txt
 --- a/remote.txt
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff b/http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff
--- a/http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/localParent123.diff
@@ -1,4 +1,4 @@
-Parent 123
+Bug 123 - Parent 123
 
 diff --git a/hello.txt b/hello.txt
 new file mode 100644
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.diff b/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.diff
--- a/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.diff
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.diff
@@ -1,4 +1,4 @@
-On top of remote + local
+Bug 123 - On top of remote + local
 
 diff --git a/remote.txt b/remote.txt
 --- a/remote.txt
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff b/http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff
--- a/http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/normal123.diff
@@ -1,4 +1,4 @@
-Target patch
+Bug 123 - Target patch
 
 diff --git a/hello.txt b/hello.txt
 --- a/hello.txt
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.diff b/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.diff
--- a/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.diff
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.diff
@@ -1,4 +1,4 @@
-Orphan 123
+Bug 123 - Orphan 123
 
 diff --git a/hello.txt b/hello.txt
 new file mode 100644
diff --git a/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff b/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff
--- a/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff
+++ b/http_service/tests/fixtures/hgmo_mozilla-central/12345deadbeef.diff
@@ -1,4 +1,4 @@
-Bug XYZ - Dummy patch
+Bug 123 - Dummy patch
 
 diff --git a/test.txt b/test.txt
 --- a/test.txt
diff --git a/http_service/tests/fixtures/hgmo_try/bad123.diff b/http_service/tests/fixtures/hgmo_try/bad123.diff
--- a/http_service/tests/fixtures/hgmo_try/bad123.diff
+++ b/http_service/tests/fixtures/hgmo_try/bad123.diff
@@ -1,4 +1,4 @@
-This patch will not apply cleanly
+Bug 123 - This patch will not apply cleanly
 
 diff --git a/hello.txt b/hello.txt
 --- a/hello.txt
diff --git a/http_service/tests/fixtures/hgmo_try/localParent456.diff b/http_service/tests/fixtures/hgmo_try/localParent456.diff
--- a/http_service/tests/fixtures/hgmo_try/localParent456.diff
+++ b/http_service/tests/fixtures/hgmo_try/localParent456.diff
@@ -1,4 +1,4 @@
-Parent 456
+Bug 123 - Parent 456
 
 diff --git a/hello.txt b/hello.txt
 new file mode 100644
diff --git a/http_service/tests/fixtures/hgmo_try/needRemote.diff b/http_service/tests/fixtures/hgmo_try/needRemote.diff
--- a/http_service/tests/fixtures/hgmo_try/needRemote.diff
+++ b/http_service/tests/fixtures/hgmo_try/needRemote.diff
@@ -1,4 +1,4 @@
-Depends on remote
+Bug 123 - Depends on remote
 
 diff --git a/remote.txt b/remote.txt
 --- a/remote.txt
diff --git a/http_service/tests/fixtures/hgmo_try/normal456.diff b/http_service/tests/fixtures/hgmo_try/normal456.diff
--- a/http_service/tests/fixtures/hgmo_try/normal456.diff
+++ b/http_service/tests/fixtures/hgmo_try/normal456.diff
@@ -1,4 +1,4 @@
-Target patch
+Bug 123 - Target patch
 
 diff --git a/hello.txt b/hello.txt
 --- a/hello.txt
diff --git a/http_service/tests/fixtures/hgmo_try/orphan456.diff b/http_service/tests/fixtures/hgmo_try/orphan456.diff
--- a/http_service/tests/fixtures/hgmo_try/orphan456.diff
+++ b/http_service/tests/fixtures/hgmo_try/orphan456.diff
@@ -1,4 +1,4 @@
-Orphan 456
+Bug 123 - Orphan 456
 
 diff --git a/hello.txt b/hello.txt
 new file mode 100644
diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py
--- a/http_service/tests/test_schedule_tests.py
+++ b/http_service/tests/test_schedule_tests.py
@@ -3,14 +3,36 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import json
+
 import hglib
 import pytest
 
-from bugbug_http.models import schedule_tests
-
+from bugbug_http import models
 
-def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
 
+@pytest.mark.parametrize(
+    "labels_to_choose, groups_to_choose",
+    [
+        # one from label, one from group
+        (["test-label1"], ["test-group2"],),
+        # one from label, none from group
+        (["test-label1"], [],),
+        # none from label, one from group
+        ([], ["test-group1"],),
+        # two from label, one from group
+        (["test-label1", "test-label2"], ["test-group2"],),
+    ],
+)
+def test_simple_schedule(
+    labels_to_choose,
+    groups_to_choose,
+    monkeypatch,
+    mock_hgmo,
+    mock_repo,
+    mock_component_taskcluster_artifact,
+    mock_schedule_tests_classify,
+):
     # The repo should be almost empty at first
     repo_dir, _ = mock_repo
     repo = hglib.open(str(repo_dir))
@@ -19,13 +41,22 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
     assert test_txt.exists()
     assert test_txt.read_text("utf-8") == "Version 3"
 
+    mock_schedule_tests_classify(labels_to_choose, groups_to_choose)
+
     # Scheduling a test on a revision should apply changes in the repo
-    assert schedule_tests("mozilla-central", "12345deadbeef") == "OK"
+    assert models.schedule_tests("mozilla-central", "12345deadbeef") == "OK"
 
     # Check changes have been applied
     assert len(repo.log()) == 5
     assert test_txt.read_text("utf-8") == "Version 3\nThis is a new line\n"
 
+    # Assert the test selection result is stored in Redis.
+    assert json.loads(
+        models.redis.get(
+            "bugbug:job_result:schedule_tests:mozilla-central_12345deadbeef"
+        )
+    ) == {"tasks": labels_to_choose, "groups": groups_to_choose,}
+
 
 @pytest.mark.parametrize(
     "branch, revision, result, final_log",
@@ -35,7 +66,7 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             "integration/autoland",
             "normal123",
             "OK",
-            ["Target patch", "Parent 123", "Base history 0"],
+            ["Bug 123 - Target patch", "Bug 123 - Parent 123", "Base history 0"],
         ),
         # patch from autoland where parent is not available
         # so the patch is applied on top of tip
@@ -44,7 +75,7 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             "orphan123",
             "OK",
             [
-                "Orphan 123",
+                "Bug 123 - Orphan 123",
                 "Base history 3",
                 "Base history 2",
                 "Base history 1",
@@ -57,8 +88,8 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             "needRemote",
             "OK",
             [
-                "On top of remote + local",
-                "Based on remote",
+                "Bug 123 - On top of remote + local",
+                "Bug 123 - Based on remote",
                 "Pulled from remote",
                 "Base history 3",
                 "Base history 2",
@@ -71,7 +102,12 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             "try",
             "normal456",
             "OK",
-            ["Target patch", "Parent 456", "Base history 1", "Base history 0"],
+            [
+                "Bug 123 - Target patch",
+                "Bug 123 - Parent 456",
+                "Base history 1",
+                "Base history 0",
+            ],
         ),
         # patch from try where parent is not available
         # so the patch is applied on top of tip
@@ -80,7 +116,7 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             "orphan456",
             "OK",
             [
-                "Orphan 456",
+                "Bug 123 - Orphan 456",
                 "Base history 3",
                 "Base history 2",
                 "Base history 1",
@@ -102,7 +138,7 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             "needRemote",
             "OK",
             [
-                "Depends on remote",
+                "Bug 123 - Depends on remote",
                 "Pulled from remote",
                 "Base history 3",
                 "Base history 2",
@@ -113,7 +149,15 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
     ],
 )
 def test_schedule(
-    branch, revision, result, final_log, patch_resources, mock_hgmo, mock_repo
+    branch,
+    revision,
+    result,
+    final_log,
+    patch_resources,
+    mock_hgmo,
+    mock_repo,
+    mock_component_taskcluster_artifact,
+    mock_schedule_tests_classify,
 ):
 
     # The repo should only have the base commits
@@ -128,8 +172,16 @@ def test_schedule(
         "Base history 0",
     ]
 
+    mock_schedule_tests_classify(["test-label1"], ["test-group2"])
+
     # Schedule tests for parametrized revision
-    assert schedule_tests(branch, revision) == result
+    assert models.schedule_tests(branch, revision) == result
 
     # Now check the log has evolved
     assert final_log == [l.desc.decode("utf-8") for l in repo.log(follow=True)]
+
+    if result == "OK":
+        # Assert the test selection result is stored in Redis.
+        assert json.loads(
+            models.redis.get(f"bugbug:job_result:schedule_tests:{branch}_{revision}")
+        ) == {"tasks": ["test-label1"], "groups": ["test-group2"],}