Repository: mozilla/bugbug

Issue:
Ignore "# ignore-this-changeset" commits
When "# ignore-this-changeset" is in the desc, we can ignore the commit.


Discussion:
https://hg.mozilla.org/hgcustom/version-control-tools/file/b85b908567bfecf9146d6e2ff6634ea194ca8beb/hgext/configwizard/__init__.py#l1102
We can't do this because of #502.

Resolving patch:
diff --git a/bugbug/repository.py b/bugbug/repository.py
--- a/bugbug/repository.py
+++ b/bugbug/repository.py
@@ -60,6 +60,13 @@ def __init__(
         self.file_copies = file_copies
         self.reviewers = reviewers
 
+    def __eq__(self, other):
+        assert isinstance(other, Commit)
+        return self.node == other.node
+
+    def __hash__(self):
+        return hash(self.node)
+
     def set_experience(
         self, exp_type, commit_type, timespan, exp_sum, exp_max, exp_min
     ):
@@ -157,13 +164,11 @@ def is_test(path):
 
 
 def _transform(commit):
-    desc = commit.desc.decode("utf-8")
-
     obj = {
         "node": commit.node,
         "author": commit.author,
         "reviewers": commit.reviewers,
-        "desc": desc,
+        "desc": commit.desc,
         "date": str(commit.date),
         "pushdate": str(commit.pushdate),
         "bug_id": int(commit.bug.decode("ascii")) if commit.bug else None,
@@ -293,7 +298,7 @@ def hg_log(hg, revs):
             Commit(
                 node=sys.intern(rev[0].decode("ascii")),
                 author=sys.intern(rev[1].decode("utf-8")),
-                desc=rev[2],
+                desc=rev[2].decode("utf-8"),
                 date=date,
                 pushdate=pushdate,
                 bug=rev[4],
@@ -365,7 +370,7 @@ def __setitem__(self, day, value):
         assert day == self.last_day
 
 
-def calculate_experiences(commits):
+def calculate_experiences(commits, commits_to_ignore):
     print(f"Analyzing experiences from {len(commits)} commits...")
 
     first_commit_time = {}
@@ -523,9 +528,6 @@ def update_complex_experiences(experience_type, day, items):
         day = (commit.pushdate - first_pushdate).days
         assert day >= 0
 
-        update_experiences("author", day, [commit.author])
-        update_experiences("reviewer", day, commit.reviewers)
-
         # When a file is moved/copied, copy original experience values to the copied path.
         if len(commit.file_copies) > 0:
             for orig, copied in commit.file_copies.items():
@@ -555,19 +557,49 @@ def update_complex_experiences(experience_type, day, items):
                         experiences["file"][commit_type][orig]
                     )
 
-        update_complex_experiences("file", day, commit.files)
+        if commit not in commits_to_ignore:
+            update_experiences("author", day, [commit.author])
+            update_experiences("reviewer", day, commit.reviewers)
 
-        update_complex_experiences("directory", day, get_directories(commit.files))
+            update_complex_experiences("file", day, commit.files)
 
-        components = list(
-            set(
-                path_to_component[path]
-                for path in commit.files
-                if path in path_to_component
+            update_complex_experiences("directory", day, get_directories(commit.files))
+
+            components = list(
+                set(
+                    path_to_component[path]
+                    for path in commit.files
+                    if path in path_to_component
+                )
             )
-        )
 
-        update_complex_experiences("component", day, components)
+            update_complex_experiences("component", day, components)
+
+
+def get_commits_to_ignore(repo_dir, commits):
+    # Skip commits which are in .hg-annotate-ignore-revs or which have
+    # 'ignore-this-changeset' in their description (mostly consisting of very
+    # large and not meaningful formatting changes).
+    with open(os.path.join(repo_dir, ".hg-annotate-ignore-revs"), "r") as f:
+        ignore_revs = set(l[:40] for l in f)
+
+    backouts = set(commit.backedoutby for commit in commits if commit.backedoutby != "")
+
+    def should_ignore(commit):
+        if commit.node in ignore_revs or "ignore-this-changeset" in commit.desc:
+            return True
+
+        # Don't analyze backouts.
+        if commit.node in backouts:
+            return True
+
+        # Don't analyze commits that are not linked to a bug.
+        if commit.bug == b"":
+            return True
+
+        return False
+
+    return set(commit for commit in commits if should_ignore(commit))
 
 
 def download_commits(repo_dir, date_from):
@@ -581,13 +613,6 @@ def download_commits(repo_dir, date_from):
 
     hg.close()
 
-    # Skip commits which are in .hg-annotate-ignore-revs (mostly consisting of very
-    # large and not meaningful formatting changes).
-    with open(os.path.join(repo_dir, ".hg-annotate-ignore-revs"), "rb") as f:
-        ignore_revs = set(l[:40] for l in f)
-
-    revs = [rev for rev in revs if rev not in ignore_revs]
-
     processes = multiprocessing.cpu_count()
 
     print(f"Mining {len(revs)} commits using {processes} processes...")
@@ -602,13 +627,6 @@ def download_commits(repo_dir, date_from):
         commits = tqdm(commits, total=len(revs_groups))
         commits = list(itertools.chain.from_iterable(commits))
 
-    # Don't analyze backouts.
-    backouts = set(commit.backedoutby for commit in commits if commit.backedoutby != "")
-    commits = [commit for commit in commits if commit.node not in backouts]
-
-    # Don't analyze commits that are not linked to a bug.
-    commits = [commit for commit in commits if commit.bug != b""]
-
     print("Downloading file->component mapping...")
 
     global path_to_component
@@ -621,7 +639,13 @@ def download_commits(repo_dir, date_from):
         path: "::".join(component) for path, component in path_to_component.items()
     }
 
-    calculate_experiences(commits)
+    commits_to_ignore = get_commits_to_ignore(repo_dir, commits)
+    print(f"{len(commits_to_ignore)} commits to ignore")
+
+    calculate_experiences(commits, commits_to_ignore)
+
+    # Exclude commits to ignore.
+    commits = [commit for commit in commits if commit not in commits_to_ignore]
 
     # Exclude commits outside the range we care about.
     commits = [commit for commit in commits if commit.pushdate > date_from]


Test patch:
diff --git a/tests/test_repository.py b/tests/test_repository.py
--- a/tests/test_repository.py
+++ b/tests/test_repository.py
@@ -166,6 +166,53 @@ def test_exp_queue():
     assert q[12] == 1
 
 
+def test_get_commits_to_ignore(tmpdir):
+    tmp_path = tmpdir.strpath
+
+    with open(os.path.join(tmp_path, ".hg-annotate-ignore-revs"), "w") as f:
+        f.write("commit1\ncommit2\n8ba995b74e18334ab3707f27e9eb8f4e37ba3d29\n")
+
+    def create_commit(node, desc, bug, backedoutby):
+        return repository.Commit(
+            node=node,
+            author="author",
+            desc=desc,
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2019, 1, 1),
+            bug=bug,
+            backedoutby=backedoutby,
+            author_email="author@mozilla.org",
+            files=["dom/file1.cpp"],
+            file_copies={},
+            reviewers=("reviewer1", "reviewer2"),
+        )
+
+    commits = [
+        create_commit("commit", "", 123, ""),
+        create_commit("commit_backout", "", 123, ""),
+        create_commit("commit_backedout", "", 123, "commit_backout"),
+        create_commit("commit_no_bug", "", b"", ""),
+        create_commit(
+            "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29",
+            "commit in .hg-annotate-ignore-revs",
+            123,
+            "",
+        ),
+        create_commit(
+            "commit_with_ignore_in_desc", "prova\nignore-this-changeset\n", 123, ""
+        ),
+    ]
+
+    leftovers = repository.get_commits_to_ignore(tmp_path, commits)
+    assert len(leftovers) == 4
+    assert set(commit.node for commit in leftovers) == {
+        "commit_backout",
+        "commit_no_bug",
+        "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29",
+        "commit_with_ignore_in_desc",
+    }
+
+
 def test_calculate_experiences():
     commits = {
         "commit1": repository.Commit(
@@ -207,6 +254,19 @@ def test_calculate_experiences():
             file_copies={},
             reviewers=("reviewer1",),
         ),
+        "commit2refactoring": repository.Commit(
+            node="commit2refactoring",
+            author="author2",
+            desc="commit2refactoring",
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2019, 1, 1),
+            bug="123",
+            backedoutby="",
+            author_email="author2@mozilla.org",
+            files=["dom/file1.cpp"],
+            file_copies={},
+            reviewers=("reviewer1",),
+        ),
         "commit3": repository.Commit(
             node="commit3",
             author="author1",
@@ -269,7 +329,9 @@ def test_calculate_experiences():
         "apps/file2.jsm": "Firefox::Boh",
     }
 
-    repository.calculate_experiences(list(commits.values()))
+    repository.calculate_experiences(
+        list(commits.values()), {commits["commit2refactoring"]}
+    )
 
     assert commits["commit1"].seniority_author == 0
     assert commits["commitbackedout"].seniority_author == 0