Repository: mozilla/bugbug

Issue:
Try to apply patches without pulling first
In most cases, we'd be able to apply patches without pulling first, so we should:
- If the parent is available, checkout to the parent
- If the parent is not available, checkout to "tip"
- Try to apply the patch
- If the patch fails to apply and we had updated to "tip", pull from the remote repo and restart from the first step (if the parent still doesn't exist, we can directly raise an exception).
- If the patch fails to apply and we had updated to the corrent parent, raise an exception.
Try to apply patches without pulling first
In most cases, we'd be able to apply patches without pulling first, so we should:
- If the parent is available, checkout to the parent
- If the parent is not available, checkout to "tip"
- Try to apply the patch
- If the patch fails to apply and we had updated to "tip", pull from the remote repo and restart from the first step (if the parent still doesn't exist, we can directly raise an exception).
- If the patch fails to apply and we had updated to the corrent parent, raise an exception.


Discussion:
As part of this, we should add the test suggested at https://github.com/mozilla/bugbug/pull/1369/files#r388303811.
As part of this, we should add the test suggested at https://github.com/mozilla/bugbug/pull/1369/files#r388303811.

Resolving patch:
diff --git a/bugbug/repository.py b/bugbug/repository.py
--- a/bugbug/repository.py
+++ b/bugbug/repository.py
@@ -960,7 +960,7 @@ def download_commits(repo_dir, rev_start=0, save=True):
     return commits
 
 
-def clean(repo_dir):
+def clean(repo_dir, pull=True):
     with hglib.open(repo_dir) as hg:
         hg.revert(repo_dir.encode("utf-8"), all=True)
 
@@ -974,9 +974,10 @@ def clean(repo_dir):
                 raise
 
         # Pull and update.
-        logger.info(f"Pulling and updating {repo_dir}")
-        hg.pull(update=True)
-        logger.info(f"{repo_dir} pulled and updated")
+        if pull:
+            logger.info(f"Pulling and updating {repo_dir}")
+            hg.pull(update=True)
+            logger.info(f"{repo_dir} pulled and updated")
 
 
 def clone(repo_dir, url="https://hg.mozilla.org/mozilla-central"):
@@ -1010,13 +1011,10 @@ def clone(repo_dir, url="https://hg.mozilla.org/mozilla-central"):
     clean(repo_dir)
 
 
-def apply_stack(repo_dir, stack, branch, default_base):
+def apply_stack(repo_dir, stack, branch):
     """Apply a stack of patches on a repository"""
     assert len(stack) > 0, "Empty stack"
 
-    # Start by updating the repository
-    clean(repo_dir)
-
     def has_revision(revision):
         try:
             hg.identify(revision)
@@ -1024,32 +1022,64 @@ def has_revision(revision):
         except hglib.error.CommandError:
             return False
 
-    with hglib.open(repo_dir) as hg:
+    def apply_patches(base, patches):
+
+        # Update to base revision
+        logger.info(f"Updating {repo_dir} to {base}")
+        hg.update(base, clean=True)
 
-        # Find the base revision to apply all the patches onto
-        # Use first parent from first patch if all its parents are available
-        # Otherwise fallback on tip
+        # Then apply each patch in the stack
+        try:
+            for node, patch in patches:
+                hg.import_(patches=io.BytesIO(patch.encode("utf-8")), user="bugbug")
+        except hglib.error.CommandError as e:
+            logger.warning(f"Failed to apply patch {node}: {e.err}")
+            return False
+        return True
+
+    # Find the base revision to apply all the patches onto
+    # Use first parent from first patch if all its parents are available
+    # Otherwise fallback on tip
+    def get_base():
         parents = stack[0]["parents"]
         assert len(parents) > 0, "No parents found for first patch"
         if all(map(has_revision, parents)):
-            base = parents[0]
-        else:
-            # Some repositories need to have the exact parent to apply
-            if default_base is None:
-                raise Exception("Parents are not available, cannot apply this stack")
+            return parents[0]
 
-            base = default_base
+        return "tip"
 
-        # Update to base revision
-        logger.info(f"Will apply stack on {base}")
-        hg.update(base, clean=True)
+    # Start by cleaning the repo, without pulling
+    clean(repo_dir, pull=False)
+
+    with hglib.open(repo_dir) as hg:
+        # Get initial base revision
+        base = get_base()
+
+        # Load all the patches in the stack
+        patches = [(rev["node"], get_hgmo_patch(branch, rev["node"])) for rev in stack]
+        logger.info(f"Loaded {len(patches)} patches for the stack")
+
+        # Apply all the patches in the stack on current base
+        if apply_patches(base, patches):
+            logger.info(f"Stack applied successfully on {base}")
+            return
+
+        # We tried to apply on the valid parent and failed: cannot try another revision
+        if base != "tip":
+            raise Exception(f"Failed to apply on valid parent {base}")
+
+        # We tried to apply on tip, let's try to find the valid parent after pulling
+        clean(repo_dir, pull=True)
+
+        # Check if the valid base is now available
+        new_base = get_base()
+        if base == new_base:
+            raise Exception("No valid parent found for the stack")
+
+        if not apply_patches(new_base, patches):
+            raise Exception("Failed to apply stack on second try")
 
-        # Apply all the patches in the stack
-        for rev in stack:
-            node = rev["node"]
-            logger.info(f"Applying patch for {node}")
-            patch = get_hgmo_patch(branch, node)
-            hg.import_(patches=io.BytesIO(patch.encode("utf-8")), user="bugbug")
+        logger.info(f"Stack applied successfully on {new_base}")
 
 
 if __name__ == "__main__":
diff --git a/http_service/bugbug_http/models.py b/http_service/bugbug_http/models.py
--- a/http_service/bugbug_http/models.py
+++ b/http_service/bugbug_http/models.py
@@ -156,10 +156,8 @@ def schedule_tests(branch, rev):
         return "NOK"
 
     # Apply the stack on the local repository
-    # Autoland should always rebase on top of parents, never on tip
-    default_base = "tip" if branch != "integration/autoland" else None
     try:
-        apply_stack(REPO_DIR, stack, branch, default_base)
+        apply_stack(REPO_DIR, stack, branch)
     except Exception as e:
         LOGGER.warning(f"Failed to apply stack {branch} @ {rev}: {e}")
         return "NOK"


Test patch:
diff --git a/http_service/tests/conftest.py b/http_service/tests/conftest.py
--- a/http_service/tests/conftest.py
+++ b/http_service/tests/conftest.py
@@ -15,7 +15,6 @@
 import responses
 from rq.exceptions import NoSuchJobError
 
-import bugbug.repository
 import bugbug_http
 import bugbug_http.models
 from bugbug_http import app
@@ -186,11 +185,12 @@ def fake_json_relevance(request):
         with open(mock_path) as f:
             content = f.read()
 
-        # Patch the hardcoded revisions
-        for log in mock_repo[1].log():
-            log_id = log.rev.decode("utf-8")
-            node = log.node.decode("utf-8")
-            content = content.replace(f"BASE_HISTORY_{log_id}", node)
+        # Patch the hardcoded revisions using the remote repo
+        with hglib.open(str(mock_repo[1])) as repo:
+            for log in repo.log():
+                desc = log.desc.decode("utf-8")
+                node = log.node.decode("utf-8")
+                content = content.replace(desc.replace(" ", "_").upper(), node)
 
         return (200, {"Content-Type": "application/json"}, content)
 
@@ -211,23 +211,35 @@ def fake_json_relevance(request):
 @pytest.fixture
 def mock_repo(tmpdir, monkeypatch):
     """Create an empty mercurial repo"""
-    repo_dir = tmpdir / "repo"
+    local_dir = tmpdir / "local"
+    remote_dir = tmpdir / "remote"
 
     # Setup the worker env to use that repo dir
-    monkeypatch.setattr(bugbug_http, "REPO_DIR", str(repo_dir))
-
-    # Silence the clean method
-    monkeypatch.setattr(bugbug.repository, "clean", lambda repo_dir: True)
+    monkeypatch.setattr(bugbug_http, "REPO_DIR", str(local_dir))
 
     # Create the repo
-    hglib.init(str(repo_dir))
+    hglib.init(str(local_dir))
 
     # Add several commits on a test file to create some history
-    test_file = repo_dir / "test.txt"
-    repo = hglib.open(str(repo_dir))
-    for i in range(4):
-        test_file.write_text(f"Version {i}", encoding="utf-8")
-        repo.add([str(test_file).encode("utf-8")])
-        repo.commit(f"Base history {i}", user="bugbug")
-
-    return repo_dir, repo
+    test_file = local_dir / "test.txt"
+    with hglib.open(str(local_dir)) as repo:
+        for i in range(4):
+            test_file.write_text(f"Version {i}", encoding="utf-8")
+            repo.add([str(test_file).encode("utf-8")])
+            repo.commit(f"Base history {i}", user="bugbug")
+
+    # Copy initialized repo as remote
+    local_dir.copy(remote_dir)
+
+    # Configure remote on local
+    hgrc = local_dir / ".hg" / "hgrc"
+    hgrc.write_text("\n".join(["[paths]", f"default = {remote_dir}"]), "utf-8")
+
+    # Add extra commit on remote
+    with hglib.open(str(remote_dir)) as repo:
+        remote = remote_dir / "remote.txt"
+        remote.write_text("New remote file !", encoding="utf-8")
+        repo.add([str(remote).encode("utf-8")])
+        repo.commit("Pulled from remote", user="bugbug")
+
+    return local_dir, remote_dir
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/localOnRemote.diff b/http_service/tests/fixtures/hgmo_integration-autoland/localOnRemote.diff
new file mode 100644
--- /dev/null
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/localOnRemote.diff
@@ -0,0 +1,8 @@
+Based on remote
+
+diff --git a/remote.txt b/remote.txt
+--- a/remote.txt
++++ b/remote.txt
+@@ -1,1 +1,2 @@
+ New remote file !
++On top of remote - autoland
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.diff b/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.diff
new file mode 100644
--- /dev/null
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.diff
@@ -0,0 +1,9 @@
+On top of remote + local
+
+diff --git a/remote.txt b/remote.txt
+--- a/remote.txt
++++ b/remote.txt
+@@ -1,2 +1,3 @@
+ New remote file !
+ On top of remote - autoland
++This is the new top !
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.json b/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.json
new file mode 100644
--- /dev/null
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/needRemote.json
@@ -0,0 +1,18 @@
+{
+    "changesets": [
+        {
+            "node": "localOnRemote",
+            "parents": [
+                "PULLED_FROM_REMOTE"
+            ]
+        },
+        {
+            "node": "needRemote",
+            "parents": [
+                "localOnRemote"
+            ]
+        }
+    ],
+    "visible": true
+}
+
diff --git a/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.diff b/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.diff
new file mode 100644
--- /dev/null
+++ b/http_service/tests/fixtures/hgmo_integration-autoland/orphan123.diff
@@ -0,0 +1,8 @@
+Orphan 123
+
+diff --git a/hello.txt b/hello.txt
+new file mode 100644
+--- /dev/null
++++ b/hello.txt
+@@ -0,0 +1,1 @@
++Hello from autoland - single orphan
diff --git a/http_service/tests/fixtures/hgmo_try/needRemote.diff b/http_service/tests/fixtures/hgmo_try/needRemote.diff
new file mode 100644
--- /dev/null
+++ b/http_service/tests/fixtures/hgmo_try/needRemote.diff
@@ -0,0 +1,8 @@
+Depends on remote
+
+diff --git a/remote.txt b/remote.txt
+--- a/remote.txt
++++ b/remote.txt
+@@ -1,1 +1,2 @@
+ New remote file !
++This patch will apply when remote is fetched
diff --git a/http_service/tests/fixtures/hgmo_try/needRemote.json b/http_service/tests/fixtures/hgmo_try/needRemote.json
new file mode 100644
--- /dev/null
+++ b/http_service/tests/fixtures/hgmo_try/needRemote.json
@@ -0,0 +1,11 @@
+{
+    "changesets": [
+        {
+            "node": "needRemote",
+            "parents": [
+                "PULLED_FROM_REMOTE"
+            ]
+        }
+    ],
+    "visible": true
+}
diff --git a/http_service/tests/test_schedule_tests.py b/http_service/tests/test_schedule_tests.py
--- a/http_service/tests/test_schedule_tests.py
+++ b/http_service/tests/test_schedule_tests.py
@@ -3,6 +3,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import hglib
 import pytest
 
 from bugbug_http.models import schedule_tests
@@ -11,7 +12,8 @@
 def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
 
     # The repo should be almost empty at first
-    repo_dir, repo = mock_repo
+    repo_dir, _ = mock_repo
+    repo = hglib.open(str(repo_dir))
     assert len(repo.log()) == 4
     test_txt = repo_dir / "test.txt"
     assert test_txt.exists()
@@ -36,13 +38,33 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             ["Target patch", "Parent 123", "Base history 0"],
         ),
         # patch from autoland where parent is not available
-        # so the patch is rejected as we need the parents on autoland
-        # and no changes is made on the repository
+        # so the patch is applied on top of tip
         (
             "integration/autoland",
             "orphan123",
-            "NOK",
-            ["Base history 3", "Base history 2", "Base history 1", "Base history 0"],
+            "OK",
+            [
+                "Orphan 123",
+                "Base history 3",
+                "Base history 2",
+                "Base history 1",
+                "Base history 0",
+            ],
+        ),
+        # patch on autoland that only applies after a pull has been done
+        (
+            "integration/autoland",
+            "needRemote",
+            "OK",
+            [
+                "On top of remote + local",
+                "Based on remote",
+                "Pulled from remote",
+                "Base history 3",
+                "Base history 2",
+                "Base history 1",
+                "Base history 0",
+            ],
         ),
         # patch from try based on local parent n°1
         (
@@ -74,6 +96,20 @@ def test_simple_schedule(patch_resources, mock_hgmo, mock_repo):
             "NOK",
             ["Base history 2", "Base history 1", "Base history 0"],
         ),
+        # patch on try that only applies after a pull has been done
+        (
+            "try",
+            "needRemote",
+            "OK",
+            [
+                "Depends on remote",
+                "Pulled from remote",
+                "Base history 3",
+                "Base history 2",
+                "Base history 1",
+                "Base history 0",
+            ],
+        ),
     ],
 )
 def test_schedule(
@@ -81,7 +117,8 @@ def test_schedule(
 ):
 
     # The repo should only have the base commits
-    repo_dir, repo = mock_repo
+    repo_dir, _ = mock_repo
+    repo = hglib.open(str(repo_dir))
     logs = repo.log(follow=True)
     assert len(logs) == 4
     assert [l.desc.decode("utf-8") for l in logs] == [