Repository: mozilla/bugbug

Issue:
Don't get file_copies from hg log
It's pretty slow.
We need it for experience calculation, so we could move this step after the parse patch step.


Discussion:
Probably to be done after #371 to avoid making that harder.
In fact we should move it in _transform and so move xp after _transform.
In order to compute modified files, hg gets some data from the disk and put them in memory.
Since hg export and hg cat need to have almost the same info, if we move file_* stuff in _transform, then all the info for hg things will be there in memory so it should be faster.
And since we're analyzing patch of different sizes in the different processes, it will remove almost all the IO blocking we can see currently (in hg_log)

Resolving patch:
diff --git a/bugbug/repository.py b/bugbug/repository.py
--- a/bugbug/repository.py
+++ b/bugbug/repository.py
@@ -9,6 +9,7 @@
 import itertools
 import json
 import logging
+import math
 import os
 import pickle
 import re
@@ -24,7 +25,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 hg_servers = list()
 hg_servers_lock = threading.Lock()
 thread_local = threading.local()
@@ -51,24 +51,34 @@ def __init__(
         desc,
         date,
         pushdate,
-        bug,
+        bug_id,
         backedoutby,
         author_email,
-        files,
-        file_copies,
         reviewers,
+        ignored=False,
     ):
         self.node = node
         self.author = author
+        self.bug_id = bug_id
         self.desc = desc
         self.date = date
         self.pushdate = pushdate
-        self.bug = bug
         self.backedoutby = backedoutby
+        self.ever_backedout = backedoutby != ""
         self.author_email = author_email
-        self.files = files
-        self.file_copies = file_copies
         self.reviewers = reviewers
+        self.ignored = ignored
+        self.added = 0
+        self.test_added = 0
+        self.deleted = 0
+        self.test_deleted = 0
+        self.types = set()
+        self.seniority_author = 0.0
+        self.total_file_size = 0
+        self.average_file_size = 0
+        self.maximum_file_size = 0
+        self.minimum_file_size = 0
+        self.files_modified_num = 0
 
     def __eq__(self, other):
         assert isinstance(other, Commit)
@@ -77,6 +87,15 @@ def __eq__(self, other):
     def __hash__(self):
         return hash(self.node)
 
+    def set_files(self, files, file_copies):
+        self.files = files
+        self.file_copies = file_copies
+        self.components = list(
+            set(path_to_component[path] for path in files if path in path_to_component)
+        )
+        self.directories = get_directories(files)
+        return self
+
     def set_experience(
         self, exp_type, commit_type, timespan, exp_sum, exp_max, exp_min
     ):
@@ -88,6 +107,15 @@ def set_experience(
             setattr(self, f"{exp_str}max", exp_max)
             setattr(self, f"{exp_str}min", exp_min)
 
+    def to_json(self):
+        d = self.__dict__
+        for f in ["backedoutby", "ignored", "file_copies"]:
+            del d[f]
+        d["types"] = list(d["types"])
+        d["pushdate"] = str(d["pushdate"])
+        d["date"] = str(d["date"])
+        return d
+
 
 # This is only a temporary hack: Should be removed after the template issue with reviewers (https://bugzilla.mozilla.org/show_bug.cgi?id=1528938)
 # gets fixed. Most of this code is copied from https://github.com/mozilla/version-control-tools/blob/2c2812d4a41b690203672a183b1dd85ca8b39e01/pylib/mozautomation/mozautomation/commitparser.py#L129
@@ -180,30 +208,38 @@ def is_test(path):
     )
 
 
-def _transform(commit):
-    obj = {
-        "node": commit.node,
-        "author": commit.author,
-        "reviewers": commit.reviewers,
-        "desc": commit.desc,
-        "date": str(commit.date),
-        "pushdate": str(commit.pushdate),
-        "bug_id": int(commit.bug.decode("ascii")) if commit.bug else None,
-        "ever_backedout": commit.backedoutby != "",
-        "added": 0,
-        "test_added": 0,
-        "deleted": 0,
-        "test_deleted": 0,
-        "types": set(),
-        "author_email": commit.author_email.decode("utf-8"),
-    }
+def hg_modified_files(hg, commit):
+    template = '{join(files,"|")}\\0{join(file_copies,"|")}\\0'
+    args = hglib.util.cmdbuilder(
+        b"log",
+        template=template,
+        no_merges=True,
+        rev=commit.node.encode("ascii"),
+        branch="central",
+    )
+    x = hg.rawcommand(args)
+    files_str, file_copies_str = x.split(b"\x00")[:-1]
+
+    file_copies = {}
+    for file_copy in file_copies_str.decode("utf-8").split("|"):
+        if not file_copy:
+            continue
+
+        parts = file_copy.split(" (")
+        copied = parts[0]
+        orig = parts[1][:-1]
+        file_copies[sys.intern(orig)] = sys.intern(copied)
+
+    commit.set_files(
+        [sys.intern(f) for f in files_str.decode("utf-8").split("|")], file_copies
+    )
+
 
-    # Copy all experience fields.
-    for attr, value in commit.__dict__.items():
-        if attr.startswith(f"touched_prev"):
-            obj[attr] = value
+def _transform(commit):
+    hg_modified_files(HG, commit)
 
-    obj["seniority_author"] = commit.seniority_author
+    if commit.ignored:
+        return commit
 
     sizes = []
 
@@ -211,17 +247,17 @@ def _transform(commit):
     patch_data = rs_parsepatch.get_counts(patch)
     for stats in patch_data:
         if stats["binary"]:
-            obj["types"].add("binary")
+            commit.types.add("binary")
             continue
 
         path = stats["filename"]
 
         if is_test(path):
-            obj["test_added"] += stats["added_lines"]
-            obj["test_deleted"] += stats["deleted_lines"]
+            commit.test_added += stats["added_lines"]
+            commit.test_deleted += stats["deleted_lines"]
         else:
-            obj["added"] += stats["added_lines"]
-            obj["deleted"] += stats["deleted_lines"]
+            commit.added += stats["added_lines"]
+            commit.deleted += stats["deleted_lines"]
 
         ext = os.path.splitext(path)[1]
         if ext in [".js", ".jsm"]:
@@ -247,7 +283,7 @@ def _transform(commit):
             type_ = "Rust"
         else:
             type_ = ext
-        obj["types"].add(type_)
+        commit.types.add(type_)
 
         if not stats["deleted"]:
             try:
@@ -257,33 +293,20 @@ def _transform(commit):
                 if b"no such file in rev" not in e.err:
                     raise
 
-    obj["total_file_size"] = sum(sizes)
-    obj["average_file_size"] = (
-        obj["total_file_size"] / len(sizes) if len(sizes) > 0 else 0
+    commit.total_file_size = sum(sizes)
+    commit.average_file_size = (
+        commit.total_file_size / len(sizes) if len(sizes) > 0 else 0
     )
-    obj["maximum_file_size"] = max(sizes, default=0)
-    obj["minimum_file_size"] = min(sizes, default=0)
+    commit.maximum_file_size = max(sizes, default=0)
+    commit.minimum_file_size = min(sizes, default=0)
 
-    obj["files_modified_num"] = len(patch_data)
+    commit.files_modified_num = len(patch_data)
 
-    # Covert to a list, as a set is not JSON-serializable.
-    obj["types"] = list(obj["types"])
-
-    obj["components"] = list(
-        set(
-            path_to_component[path]
-            for path in commit.files
-            if path in path_to_component
-        )
-    )
-    obj["directories"] = get_directories(commit.files)
-    obj["files"] = commit.files
-
-    return obj
+    return commit
 
 
 def hg_log(hg, revs):
-    template = '{node}\\0{author}\\0{desc}\\0{date|hgdate}\\0{bug}\\0{backedoutby}\\0{author|email}\\0{join(files,"|")}\\0{join(file_copies,"|")}\\0{pushdate|hgdate}\\0'
+    template = "{node}\\0{author}\\0{desc}\\0{date|hgdate}\\0{bug}\\0{backedoutby}\\0{author|email}\\0{pushdate|hgdate}\\0"
 
     args = hglib.util.cmdbuilder(
         b"log",
@@ -300,22 +323,14 @@ def hg_log(hg, revs):
         assert b" " in rev[3]
         date = datetime.utcfromtimestamp(float(rev[3].split(b" ", 1)[0]))
 
-        assert b" " in rev[9]
-        pushdate_timestamp = rev[9].split(b" ", 1)[0]
+        assert b" " in rev[7]
+        pushdate_timestamp = rev[7].split(b" ", 1)[0]
         if pushdate_timestamp != b"0":
             pushdate = datetime.utcfromtimestamp(float(pushdate_timestamp))
         else:
             pushdate = datetime.utcnow()
 
-        file_copies = {}
-        for file_copy in rev[8].decode("utf-8").split("|"):
-            if not file_copy:
-                continue
-
-            parts = file_copy.split(" (")
-            copied = parts[0]
-            orig = parts[1][:-1]
-            file_copies[sys.intern(orig)] = sys.intern(copied)
+        bug_id = int(rev[4].decode("ascii")) if rev[4] else None
 
         revs.append(
             Commit(
@@ -324,11 +339,9 @@ def hg_log(hg, revs):
                 desc=rev[2].decode("utf-8"),
                 date=date,
                 pushdate=pushdate,
-                bug=rev[4],
+                bug_id=bug_id,
                 backedoutby=rev[5].decode("ascii"),
-                author_email=rev[6],
-                files=[sys.intern(f) for f in rev[7].decode("utf-8").split("|")],
-                file_copies=file_copies,
+                author_email=rev[6].decode("utf-8"),
                 reviewers=tuple(
                     set(sys.intern(r) for r in get_reviewers(rev[2].decode("utf-8")))
                 ),
@@ -409,7 +422,7 @@ def __setitem__(self, day, value):
         assert day == self.last_day
 
 
-def calculate_experiences(commits, commits_to_ignore, first_pushdate, save=True):
+def calculate_experiences(commits, first_pushdate, save=True):
     print(f"Analyzing experiences from {len(commits)} commits...")
 
     try:
@@ -483,9 +496,9 @@ def update_experiences(experience_type, day, items):
             # We don't want to consider backed out commits when calculating normal experiences.
             if (
                 commit_type == ""
-                and not commit.backedoutby
+                and not commit.ever_backedout
                 or commit_type == "backout"
-                and commit.backedoutby
+                and commit.ever_backedout
             ):
                 for i, item in enumerate(items):
                     experiences[experience_type][commit_type][item][day] = (
@@ -556,9 +569,9 @@ def update_complex_experiences(experience_type, day, items):
             # We don't want to consider backed out commits when calculating normal experiences.
             if (
                 commit_type == ""
-                and not commit.backedoutby
+                and not commit.ever_backedout
                 or commit_type == "backout"
-                and commit.backedoutby
+                and commit.ever_backedout
             ):
                 for i, item in enumerate(items):
                     experiences[experience_type][commit_type][item][
@@ -570,34 +583,26 @@ def update_complex_experiences(experience_type, day, items):
         assert day >= 0
 
         # When a file is moved/copied, copy original experience values to the copied path.
-        for orig, copied in commit.file_copies.items():
-            for commit_type in ["", "backout"]:
-                if orig in experiences["file"][commit_type]:
-                    experiences["file"][commit_type][copied] = copy.deepcopy(
-                        experiences["file"][commit_type][orig]
-                    )
-                else:
-                    print(
-                        f"Experience missing for file {orig}, type '{commit_type}', on commit {commit.node}"
-                    )
-
-        if commit not in commits_to_ignore:
+        if "file" in experiences:
+            xp_file = experiences["file"]
+            for orig, copied in commit.file_copies.items():
+                for commit_type in ["", "backout"]:
+                    if orig in xp_file[commit_type]:
+                        xp_file[commit_type][copied] = copy.deepcopy(
+                            xp_file[commit_type][orig]
+                        )
+                    else:
+                        print(
+                            f"Experience missing for file {orig}, type '{commit_type}', on commit {commit.node}"
+                        )
+
+        if not commit.ignored:
             update_experiences("author", day, [commit.author])
             update_experiences("reviewer", day, commit.reviewers)
 
             update_complex_experiences("file", day, commit.files)
-
-            update_complex_experiences("directory", day, get_directories(commit.files))
-
-            components = list(
-                set(
-                    path_to_component[path]
-                    for path in commit.files
-                    if path in path_to_component
-                )
-            )
-
-            update_complex_experiences("component", day, components)
+            update_complex_experiences("directory", day, commit.directories)
+            update_complex_experiences("component", day, commit.components)
 
     if save:
         with open("data/commit_experiences.pickle", "wb") as f:
@@ -606,14 +611,14 @@ def update_complex_experiences(experience_type, day, items):
             )
 
 
-def get_commits_to_ignore(repo_dir, commits):
+def set_commits_to_ignore(repo_dir, commits):
     # Skip commits which are in .hg-annotate-ignore-revs or which have
     # 'ignore-this-changeset' in their description (mostly consisting of very
     # large and not meaningful formatting changes).
     with open(os.path.join(repo_dir, ".hg-annotate-ignore-revs"), "r") as f:
         ignore_revs = set(l[:40] for l in f)
 
-    backouts = set(commit.backedoutby for commit in commits if commit.backedoutby != "")
+    backouts = set(commit.backedoutby for commit in commits if commit.ever_backedout)
 
     def should_ignore(commit):
         if commit.node in ignore_revs or "ignore-this-changeset" in commit.desc:
@@ -624,12 +629,13 @@ def should_ignore(commit):
             return True
 
         # Don't analyze commits that are not linked to a bug.
-        if commit.bug == b"":
+        if commit.bug_id is None:
             return True
 
         return False
 
-    return set(commit for commit in commits if should_ignore(commit))
+    for commit in commits:
+        commit.ignored = should_ignore(commit)
 
 
 def download_component_mapping():
@@ -652,11 +658,16 @@ def hg_log_multi(repo_dir, revs):
     cwd = os.getcwd()
     os.chdir(repo_dir)
 
-    CHUNK_SIZE = 256
-    revs_groups = [revs[i : (i + CHUNK_SIZE)] for i in range(0, len(revs), CHUNK_SIZE)]
+    threads_num = os.cpu_count() + 1
+    CHUNK_SIZE = int(math.ceil(len(revs) / threads_num))
+    REVS_COUNT = len(revs)
+    revs_groups = [
+        (revs[i], revs[min(i + CHUNK_SIZE, REVS_COUNT) - 1])
+        for i in range(0, REVS_COUNT, CHUNK_SIZE)
+    ]
 
     with concurrent.futures.ThreadPoolExecutor(
-        initializer=_init_thread, max_workers=os.cpu_count() + 1
+        initializer=_init_thread, max_workers=threads_num
     ) as executor:
         commits = executor.map(_hg_log, revs_groups)
         commits = tqdm(commits, total=len(revs_groups))
@@ -671,7 +682,7 @@ def hg_log_multi(repo_dir, revs):
     return commits
 
 
-def download_commits(repo_dir, rev_start=0, ret=False, save=True):
+def download_commits(repo_dir, rev_start=0, save=True):
     with hglib.open(repo_dir) as hg:
         revs = get_revs(hg, rev_start)
         if len(revs) == 0:
@@ -688,13 +699,7 @@ def download_commits(repo_dir, rev_start=0, ret=False, save=True):
 
     download_component_mapping()
 
-    commits_to_ignore = get_commits_to_ignore(repo_dir, commits)
-    print(f"{len(commits_to_ignore)} commits to ignore")
-
-    calculate_experiences(commits, commits_to_ignore, first_pushdate, save)
-
-    # Exclude commits to ignore.
-    commits = [commit for commit in commits if commit not in commits_to_ignore]
+    set_commits_to_ignore(repo_dir, commits)
 
     commits_num = len(commits)
 
@@ -708,15 +713,16 @@ def download_commits(repo_dir, rev_start=0, ret=False, save=True):
     ) as executor:
         commits = executor.map(_transform, commits, chunksize=64)
         commits = tqdm(commits, total=commits_num)
+        commits = list(commits)
 
-        if ret:
-            commits = list(commits)
+    calculate_experiences(commits, first_pushdate, save)
 
-        if save:
-            db.append(COMMITS_DB, commits)
+    commits = [commit for commit in commits if not commit.ignored]
 
-        if ret:
-            return commits
+    if save:
+        db.append(COMMITS_DB, (commit.to_json() for commit in commits))
+
+    return commits
 
 
 def clean(repo_dir):
diff --git a/scripts/regressor_finder.py b/scripts/regressor_finder.py
--- a/scripts/regressor_finder.py
+++ b/scripts/regressor_finder.py
@@ -171,11 +171,11 @@ def get_commits_to_ignore(self):
 
         commits = repository.hg_log_multi(self.mercurial_repo_dir, revs)
 
-        ignore_set = repository.get_commits_to_ignore(self.mercurial_repo_dir, commits)
+        repository.set_commits_to_ignore(self.mercurial_repo_dir, commits)
         commits_to_ignore = []
 
         for commit in commits:
-            if commit in ignore_set or commit.backedoutby:
+            if commit.ignored or commit.backedoutby:
                 commits_to_ignore.append(
                     {
                         "rev": commit.node,


Test patch:
diff --git a/tests/test_repository.py b/tests/test_repository.py
--- a/tests/test_repository.py
+++ b/tests/test_repository.py
@@ -119,6 +119,51 @@ def test_get_revs(fake_hg_repo):
     assert revs[1].decode("ascii") == revision3
 
 
+def test_hg_modified_files(fake_hg_repo):
+    hg, local, remote = fake_hg_repo
+
+    add_file(hg, local, "f1", "1\n2\n3\n4\n5\n6\n7\n")
+    revision1 = commit(hg, date=datetime(1991, 4, 16, tzinfo=timezone.utc))
+
+    add_file(hg, local, "f2", "1\n2\n3\n4\n5\n6\n7\n")
+    revision2 = commit(hg, "Bug 123 - Prova. r=moz,rev2")
+
+    hg.copy(
+        bytes(os.path.join(local, "f2"), "ascii"),
+        bytes(os.path.join(local, "f2copy"), "ascii"),
+    )
+    revision3 = commit(hg, "Copy")
+
+    hg.move(
+        bytes(os.path.join(local, "f2copy"), "ascii"),
+        bytes(os.path.join(local, "f2copymove"), "ascii"),
+    )
+    revision4 = commit(hg, "Move")
+
+    hg.push(dest=bytes(remote, "ascii"))
+    revs = repository.get_revs(hg, revision1)
+    commits = repository.hg_log(hg, revs)
+
+    for c in commits:
+        repository.hg_modified_files(hg, c)
+
+    assert commits[0].node == revision1
+    assert commits[0].files == ["f1"]
+    assert commits[0].file_copies == {}
+
+    assert commits[1].node == revision2
+    assert commits[1].files == ["f2"]
+    assert commits[1].file_copies == {}
+
+    assert commits[2].node == revision3
+    assert commits[2].files == ["f2copy"]
+    assert commits[2].file_copies == {"f2": "f2copy"}
+
+    assert commits[3].node == revision4
+    assert commits[3].files == ["f2copy", "f2copymove"]
+    assert commits[3].file_copies == {"f2copy": "f2copymove"}
+
+
 def test_hg_log(fake_hg_repo):
     hg, local, remote = fake_hg_repo
 
@@ -181,11 +226,9 @@ def test_hg_log(fake_hg_repo):
         <= commits[0].pushdate
         <= first_push_date + relativedelta(seconds=1)
     )
-    assert commits[0].bug == b""
+    assert commits[0].bug_id is None
     assert commits[0].backedoutby == ""
-    assert commits[0].author_email == b"milla@mozilla.org"
-    assert commits[0].files == ["file1"]
-    assert commits[0].file_copies == {}
+    assert commits[0].author_email == "milla@mozilla.org"
     assert commits[0].reviewers == tuple()
 
     assert commits[1].node == revision2
@@ -197,11 +240,9 @@ def test_hg_log(fake_hg_repo):
         <= commits[1].pushdate
         <= second_push_date + relativedelta(seconds=1)
     )
-    assert commits[1].bug == b"123"
+    assert commits[1].bug_id == 123
     assert commits[1].backedoutby == ""
-    assert commits[1].author_email == b"milla@mozilla.org"
-    assert commits[1].files == ["file2"]
-    assert commits[1].file_copies == {}
+    assert commits[1].author_email == "milla@mozilla.org"
     assert set(commits[1].reviewers) == {"moz", "rev2"}
 
     assert commits[2].node == revision3
@@ -213,11 +254,9 @@ def test_hg_log(fake_hg_repo):
         <= commits[2].pushdate
         <= second_push_date + relativedelta(seconds=1)
     )
-    assert commits[2].bug == b""
+    assert commits[2].bug_id is None
     assert commits[2].backedoutby == ""
-    assert commits[2].author_email == b"milla@mozilla.org"
-    assert commits[2].files == ["file2copy"]
-    assert commits[2].file_copies == {"file2": "file2copy"}
+    assert commits[2].author_email == "milla@mozilla.org"
     assert commits[2].reviewers == tuple()
 
     assert commits[3].node == revision4
@@ -229,11 +268,9 @@ def test_hg_log(fake_hg_repo):
         <= commits[3].pushdate
         <= second_push_date + relativedelta(seconds=1)
     )
-    assert commits[3].bug == b""
+    assert commits[3].bug_id is None
     assert commits[3].backedoutby == revision5
-    assert commits[3].author_email == b"milla@mozilla.org"
-    assert commits[3].files == ["file2copy", "file2copymove"]
-    assert commits[3].file_copies == {"file2copy": "file2copymove"}
+    assert commits[3].author_email == "milla@mozilla.org"
     assert commits[3].reviewers == tuple()
 
     assert commits[4].node == revision5
@@ -245,11 +282,9 @@ def test_hg_log(fake_hg_repo):
         <= commits[4].pushdate
         <= second_push_date + relativedelta(seconds=1)
     )
-    assert commits[4].bug == b""
+    assert commits[4].bug_id is None
     assert commits[4].backedoutby == ""
-    assert commits[4].author_email == b"sheriff"
-    assert commits[4].files == ["file2copy", "file2copymove"]
-    assert commits[4].file_copies == {"file2copymove": "file2copy"}
+    assert commits[4].author_email == "sheriff"
     assert commits[4].reviewers == tuple()
 
     assert commits[5].node == revision6
@@ -261,11 +296,9 @@ def test_hg_log(fake_hg_repo):
         <= commits[5].pushdate
         <= hg_log_date + relativedelta(seconds=1)
     )
-    assert commits[5].bug == b""
+    assert commits[5].bug_id is None
     assert commits[5].backedoutby == ""
-    assert commits[5].author_email == b"milla@mozilla.org"
-    assert commits[5].files == ["file3"]
-    assert commits[5].file_copies == {}
+    assert commits[5].author_email == "milla@mozilla.org"
     assert commits[5].reviewers == tuple()
 
     commits = repository.hg_log(hg, [revs[1], revs[3]])
@@ -364,7 +397,7 @@ def test_download_commits(fake_hg_repo):
     hg.push(dest=bytes(remote, "ascii"))
     copy_pushlog_database(remote, local)
 
-    commits = repository.download_commits(local, ret=True)
+    commits = repository.download_commits(local)
     assert len(commits) == 0
     commits = list(repository.get_commits())
     assert len(commits) == 0
@@ -377,7 +410,7 @@ def test_download_commits(fake_hg_repo):
     hg.push(dest=bytes(remote, "ascii"))
     copy_pushlog_database(remote, local)
 
-    commits = repository.download_commits(local, ret=True)
+    commits = repository.download_commits(local)
     assert len(commits) == 1
     commits = list(repository.get_commits())
     assert len(commits) == 1
@@ -393,7 +426,7 @@ def test_download_commits(fake_hg_repo):
     hg.push(dest=bytes(remote, "ascii"))
     copy_pushlog_database(remote, local)
 
-    commits = repository.download_commits(local, revision3, ret=True)
+    commits = repository.download_commits(local, revision3)
     assert len(commits) == 1
     commits = list(repository.get_commits())
     assert len(commits) == 2
@@ -406,14 +439,13 @@ def test_download_commits(fake_hg_repo):
 
     os.remove("data/commits.json")
     os.remove("data/commit_experiences.pickle")
-    commits = repository.download_commits(local, f"children({revision2})", ret=True)
+    commits = repository.download_commits(local, f"children({revision2})")
     assert len(commits) == 1
     assert len(list(repository.get_commits())) == 1
 
     os.remove("data/commits.json")
     os.remove("data/commit_experiences.pickle")
-    commits = repository.download_commits(local, ret=False)
-    assert commits is None
+    commits = repository.download_commits(local)
     assert len(list(repository.get_commits())) == 2
 
 
@@ -503,32 +535,30 @@ def test_exp_queue():
     assert q[12] == 1
 
 
-def test_get_commits_to_ignore(tmpdir):
+def test_set_commits_to_ignore(tmpdir):
     tmp_path = tmpdir.strpath
 
     with open(os.path.join(tmp_path, ".hg-annotate-ignore-revs"), "w") as f:
         f.write("commit1\ncommit2\n8ba995b74e18334ab3707f27e9eb8f4e37ba3d29\n")
 
-    def create_commit(node, desc, bug, backedoutby):
+    def create_commit(node, desc, bug_id, backedoutby):
         return repository.Commit(
             node=node,
             author="author",
             desc=desc,
             date=datetime(2019, 1, 1),
             pushdate=datetime(2019, 1, 1),
-            bug=bug,
+            bug_id=bug_id,
             backedoutby=backedoutby,
             author_email="author@mozilla.org",
-            files=["dom/file1.cpp"],
-            file_copies={},
             reviewers=("reviewer1", "reviewer2"),
-        )
+        ).set_files(["dom/file1.cpp"], {})
 
     commits = [
         create_commit("commit", "", 123, ""),
         create_commit("commit_backout", "", 123, ""),
         create_commit("commit_backedout", "", 123, "commit_backout"),
-        create_commit("commit_no_bug", "", b"", ""),
+        create_commit("commit_no_bug", "", None, ""),
         create_commit(
             "8ba995b74e18334ab3707f27e9eb8f4e37ba3d29",
             "commit in .hg-annotate-ignore-revs",
@@ -540,7 +570,8 @@ def create_commit(node, desc, bug, backedoutby):
         ),
     ]
 
-    leftovers = repository.get_commits_to_ignore(tmp_path, commits)
+    repository.set_commits_to_ignore(tmp_path, commits)
+    leftovers = [commit for commit in commits if commit.ignored]
     assert len(leftovers) == 4
     assert set(commit.node for commit in leftovers) == {
         "commit_backout",
@@ -551,6 +582,14 @@ def create_commit(node, desc, bug, backedoutby):
 
 
 def test_calculate_experiences():
+    repository.path_to_component = {
+        "dom/file1.cpp": "Core::DOM",
+        "dom/file1copied.cpp": "Core::DOM",
+        "dom/file2.cpp": "Core::Layout",
+        "apps/file1.jsm": "Firefox::Boh",
+        "apps/file2.jsm": "Firefox::Boh",
+    }
+
     commits = {
         "commit1": repository.Commit(
             node="commit1",
@@ -558,117 +597,92 @@ def test_calculate_experiences():
             desc="commit1",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2019, 1, 1),
-            bug="123",
+            bug_id=123,
             backedoutby="",
             author_email="author1@mozilla.org",
-            files=["dom/file1.cpp", "apps/file1.jsm"],
-            file_copies={},
             reviewers=("reviewer1", "reviewer2"),
-        ),
+        ).set_files(["dom/file1.cpp", "apps/file1.jsm"], {}),
         "commitbackedout": repository.Commit(
             node="commitbackedout",
             author="author1",
             desc="commitbackedout",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2019, 1, 1),
-            bug="123",
+            bug_id=123,
             backedoutby="commitbackout",
             author_email="author1@mozilla.org",
-            files=["dom/file1.cpp", "apps/file1.jsm"],
-            file_copies={},
             reviewers=("reviewer1", "reviewer2"),
-        ),
+        ).set_files(["dom/file1.cpp", "apps/file1.jsm"], {}),
         "commit2": repository.Commit(
             node="commit2",
             author="author2",
             desc="commit2",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2019, 1, 1),
-            bug="123",
+            bug_id=123,
             backedoutby="",
             author_email="author2@mozilla.org",
-            files=["dom/file1.cpp"],
-            file_copies={},
             reviewers=("reviewer1",),
-        ),
+        ).set_files(["dom/file1.cpp"], {}),
         "commit2refactoring": repository.Commit(
             node="commit2refactoring",
             author="author2",
             desc="commit2refactoring",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2019, 1, 1),
-            bug="123",
+            bug_id=123,
             backedoutby="",
             author_email="author2@mozilla.org",
-            files=["dom/file1.cpp"],
-            file_copies={},
             reviewers=("reviewer1",),
-        ),
+            ignored=True,
+        ).set_files(["dom/file1.cpp"], {}),
         "commit3": repository.Commit(
             node="commit3",
             author="author1",
             desc="commit3",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2019, 1, 1),
-            bug="123",
+            bug_id=123,
             backedoutby="",
             author_email="author1@mozilla.org",
-            files=["dom/file2.cpp", "apps/file1.jsm"],
-            file_copies={},
             reviewers=("reviewer2",),
-        ),
+        ).set_files(["dom/file2.cpp", "apps/file1.jsm"], {}),
         "commit4": repository.Commit(
             node="commit4",
             author="author2",
             desc="commit4",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2020, 1, 1),
-            bug="123",
+            bug_id=123,
             backedoutby="",
             author_email="author2@mozilla.org",
-            files=["dom/file1.cpp", "apps/file2.jsm"],
-            file_copies={},
             reviewers=("reviewer1", "reviewer2"),
-        ),
+        ).set_files(["dom/file1.cpp", "apps/file2.jsm"], {}),
         "commit5": repository.Commit(
             node="commit5",
             author="author3",
             desc="commit5",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2020, 1, 2),
-            bug="123",
+            bug_id=123,
             backedoutby="",
             author_email="author3@mozilla.org",
-            files=["dom/file1.cpp"],
-            file_copies={"dom/file1.cpp": "dom/file1copied.cpp"},
             reviewers=("reviewer3",),
-        ),
+        ).set_files(["dom/file1.cpp"], {"dom/file1.cpp": "dom/file1copied.cpp"}),
         "commit6": repository.Commit(
             node="commit6",
             author="author3",
             desc="commit6",
             date=datetime(2019, 1, 1),
             pushdate=datetime(2020, 1, 3),
-            bug="123",
+            bug_id=123,
             backedoutby="",
             author_email="author3@mozilla.org",
-            files=["dom/file1.cpp", "dom/file1copied.cpp"],
-            file_copies={},
             reviewers=("reviewer3",),
-        ),
+        ).set_files(["dom/file1.cpp", "dom/file1copied.cpp"], {}),
     }
 
-    repository.path_to_component = {
-        "dom/file1.cpp": "Core::DOM",
-        "dom/file1copied.cpp": "Core::DOM",
-        "dom/file2.cpp": "Core::Layout",
-        "apps/file1.jsm": "Firefox::Boh",
-        "apps/file2.jsm": "Firefox::Boh",
-    }
-
-    repository.calculate_experiences(
-        list(commits.values()), {commits["commit2refactoring"]}, datetime(2019, 1, 1)
-    )
+    repository.calculate_experiences(commits.values(), datetime(2019, 1, 1))
 
     assert commits["commit1"].seniority_author == 0
     assert commits["commitbackedout"].seniority_author == 0