Repository: mozilla/bugbug

Issue:
Add tests for repository.calculate_experiences

Add tests for repository.calculate_experiences



Discussion:

Resolving patch:
diff --git a/bugbug/repository.py b/bugbug/repository.py
--- a/bugbug/repository.py
+++ b/bugbug/repository.py
@@ -228,7 +228,7 @@ def _transform(commit):
     return obj
 
 
-def _hg_log(revs):
+def hg_log(hg, revs):
     template = '{node}\\0{author}\\0{desc}\\0{date}\\0{bug}\\0{backedoutby}\\0{author|email}\\0{join(files,"|")}\\0{join(file_copies,"|")}\\0{pushdate}\\0'
 
     args = hglib.util.cmdbuilder(
@@ -238,7 +238,7 @@ def _hg_log(revs):
         rev=revs[0] + b":" + revs[-1],
         branch="central",
     )
-    x = HG.rawcommand(args)
+    x = hg.rawcommand(args)
     out = x.split(b"\x00")[:-1]
 
     revs = []
@@ -276,6 +276,10 @@ def _hg_log(revs):
     return revs
 
 
+def _hg_log(revs):
+    return hg_log(HG, revs)
+
+
 def get_revs(hg):
     args = hglib.util.cmdbuilder(
         b"log", template="{node}\n", no_merges=True, branch="central", rev="0:tip"
@@ -298,65 +302,11 @@ def get_directories(files):
     return list(directories)
 
 
-def download_commits(repo_dir, date_from):
-    hg = hglib.open(repo_dir)
-
-    revs = get_revs(hg)
-
-    commits_num = len(revs)
-
-    assert (
-        commits_num > 0
-    ), "There should definitely be more than 0 commits, something is wrong"
-
-    hg.close()
-
-    processes = multiprocessing.cpu_count()
-
-    print(f"Mining {commits_num} commits using {processes} processes...")
-
-    CHUNK_SIZE = 256
-    revs_groups = [revs[i : (i + CHUNK_SIZE)] for i in range(0, len(revs), CHUNK_SIZE)]
-
-    with concurrent.futures.ProcessPoolExecutor(
-        initializer=_init, initargs=(repo_dir,)
-    ) as executor:
-        commits = executor.map(_hg_log, revs_groups, chunksize=20)
-        commits = tqdm(commits, total=len(revs_groups))
-        commits = list(itertools.chain.from_iterable(commits))
-
-    # Don't analyze backouts.
-    backouts = set(
-        commit.backedoutby for commit in commits if commit.backedoutby != b""
-    )
-    commits = [commit for commit in commits if commit.node not in backouts]
-
-    # Don't analyze commits that are not linked to a bug.
-    commits = [commit for commit in commits if commit.bug != b""]
-
-    # Skip commits which are in .hg-annotate-ignore-revs (mostly consisting of very
-    # large and not meaningful formatting changes).
-    with open(os.path.join(repo_dir, ".hg-annotate-ignore-revs"), "r") as f:
-        ignore_revs = set(l[:40].encode("utf-8") for l in f)
-
-    commits = [commit for commit in commits if commit.node not in ignore_revs]
-
-    commits_num = len(commits)
-
-    print(f"Analyzing {commits_num} patches...")
+def calculate_experiences(commits):
+    print(f"Analyzing experiences from {len(commits)} commits...")
 
     global experiences_by_commit
 
-    global path_to_component
-    r = requests.get(
-        "https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central.latest.source.source-bugzilla-info/artifacts/public/components.json"
-    )
-    r.raise_for_status()
-    path_to_component = r.json()
-    path_to_component = {
-        path: "::".join(component) for path, component in path_to_component.items()
-    }
-
     first_pushdate = commits[0].pushdate
 
     experiences = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
@@ -379,7 +329,7 @@ def update_experiences(experience_type, day, items):
             if not commit.backedoutby:
                 experiences[day][experience_type][item] += 1
 
-    def update_complex_experiences(experience_type, day, items):
+    def update_complex_experiences(experience_type, day, items, self_node):
         all_commits = set()
         before_timespan_commits = set()
         for item in items:
@@ -393,19 +343,27 @@ def update_complex_experiences(experience_type, day, items):
             if not commit.backedoutby:
                 complex_experiences[day][experience_type][item].add(commit.node)
 
+        # If a commit changes two files in the same component, we shouldn't increase the exp by two.
+        all_commits.discard(self_node)
+
         experiences_by_commit["total"][experience_type][commit.node] = len(all_commits)
         experiences_by_commit[EXPERIENCE_TIMESPAN_TEXT][experience_type][
             commit.node
         ] = len(all_commits - before_timespan_commits)
 
+    prev_days = 0
+
     for commit in tqdm(commits):
         days = (commit.pushdate - first_pushdate).days
         assert days >= 0
 
-        if days not in experiences:
+        if days not in experiences and days != prev_days:
             assert days not in complex_experiences
-            experiences[days] = copy.deepcopy(experiences[days - 1])
-            complex_experiences[days] = copy.deepcopy(complex_experiences[days - 1])
+            for day in range(prev_days + 1, days + 1):
+                experiences[day] = copy.deepcopy(experiences[day - 1])
+                complex_experiences[day] = copy.deepcopy(complex_experiences[day - 1])
+
+        prev_days = days
 
         update_experiences("author", days, [commit.author])
         update_experiences("reviewer", days, commit.reviewers)
@@ -439,9 +397,11 @@ def update_complex_experiences(experience_type, day, items):
                             copied_directory
                         ] = complex_experiences[prev_day]["directory"][orig_directory]
 
-        update_complex_experiences("file", days, commit.files)
+        update_complex_experiences("file", days, commit.files, commit.node)
 
-        update_complex_experiences("directory", days, get_directories(commit.files))
+        update_complex_experiences(
+            "directory", days, get_directories(commit.files), commit.node
+        )
 
         components = list(
             set(
@@ -451,14 +411,74 @@ def update_complex_experiences(experience_type, day, items):
             )
         )
 
-        update_complex_experiences("component", days, components)
+        update_complex_experiences("component", days, components, commit.node)
 
         # TODO: We can delete anything older than 90 days at this point.
 
+
+def download_commits(repo_dir, date_from):
+    hg = hglib.open(repo_dir)
+
+    revs = get_revs(hg)
+
+    commits_num = len(revs)
+
+    assert (
+        commits_num > 0
+    ), "There should definitely be more than 0 commits, something is wrong"
+
+    hg.close()
+
+    processes = multiprocessing.cpu_count()
+
+    print(f"Mining {commits_num} commits using {processes} processes...")
+
+    CHUNK_SIZE = 256
+    revs_groups = [revs[i : (i + CHUNK_SIZE)] for i in range(0, len(revs), CHUNK_SIZE)]
+
+    with concurrent.futures.ProcessPoolExecutor(
+        initializer=_init, initargs=(repo_dir,)
+    ) as executor:
+        commits = executor.map(_hg_log, revs_groups, chunksize=20)
+        commits = tqdm(commits, total=len(revs_groups))
+        commits = list(itertools.chain.from_iterable(commits))
+
+    # Don't analyze backouts.
+    backouts = set(
+        commit.backedoutby for commit in commits if commit.backedoutby != b""
+    )
+    commits = [commit for commit in commits if commit.node not in backouts]
+
+    # Don't analyze commits that are not linked to a bug.
+    commits = [commit for commit in commits if commit.bug != b""]
+
+    # Skip commits which are in .hg-annotate-ignore-revs (mostly consisting of very
+    # large and not meaningful formatting changes).
+    with open(os.path.join(repo_dir, ".hg-annotate-ignore-revs"), "r") as f:
+        ignore_revs = set(l[:40].encode("utf-8") for l in f)
+
+    commits = [commit for commit in commits if commit.node not in ignore_revs]
+
+    print("Downloading file->component mapping...")
+
+    global path_to_component
+    r = requests.get(
+        "https://index.taskcluster.net/v1/task/gecko.v2.mozilla-central.latest.source.source-bugzilla-info/artifacts/public/components.json"
+    )
+    r.raise_for_status()
+    path_to_component = r.json()
+    path_to_component = {
+        path: "::".join(component) for path, component in path_to_component.items()
+    }
+
+    calculate_experiences(commits)
+
     # Exclude commits outside the range we care about.
     commits = [commit for commit in commits if commit.pushdate > date_from]
 
-    print(f"Mining commits using {multiprocessing.cpu_count()} processes...")
+    commits_num = len(commits)
+
+    print(f"Mining {commits_num} commits using {processes} processes...")
 
     with concurrent.futures.ProcessPoolExecutor(
         initializer=_init, initargs=(repo_dir,)


Test patch:
diff --git a/tests/test_repository.py b/tests/test_repository.py
new file mode 100644
--- /dev/null
+++ b/tests/test_repository.py
@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+from datetime import datetime
+
+import hglib
+import pytest
+import responses
+
+from bugbug import repository
+
+
+@pytest.fixture
+def fake_hg_repo(tmpdir):
+    tmp_path = tmpdir.strpath
+    dest = os.path.join(tmp_path, "repos")
+    local = os.path.join(dest, "local")
+    remote = os.path.join(dest, "remote")
+    for d in [local, remote]:
+        os.makedirs(d)
+        hglib.init(d)
+
+    os.environ["USER"] = "app"
+    hg = hglib.open(local)
+
+    hg.branch(b"central")
+
+    responses.add_passthru("http://localhost:8000")
+
+    yield hg, local, remote
+
+    hg.close()
+
+
+def copy_pushlog_database(remote, local):
+    shutil.copyfile(
+        os.path.join(remote, ".hg/pushlog2.db"), os.path.join(local, ".hg/pushlog2.db")
+    )
+
+
+def add_file(hg, repo_dir, name, contents):
+    path = os.path.join(repo_dir, name)
+
+    with open(path, "w") as f:
+        f.write(contents)
+
+    hg.add(files=[bytes(path, "ascii")])
+
+
+def commit(hg):
+    commit_message = "Commit {}".format(hg.status())
+
+    i, revision = hg.commit(message=commit_message, user="Moz Illa <milla@mozilla.org>")
+
+    return str(revision, "ascii")
+
+
+def test_get_revs(fake_hg_repo):
+    hg, local, remote = fake_hg_repo
+
+    add_file(hg, local, "file1", "1\n2\n3\n4\n5\n6\n7\n")
+    revision1 = commit(hg)
+
+    revs = repository.get_revs(hg)
+
+    assert len(revs) == 1, "There should be one revision now"
+    assert revs[0].decode("ascii") == revision1
+
+    add_file(hg, local, "file2", "1\n2\n3\n4\n5\n6\n7\n")
+    revision2 = commit(hg)
+
+    revs = repository.get_revs(hg)
+
+    assert len(revs) == 2, "There should be two revisions now"
+    assert revs[0].decode("ascii") == revision1
+    assert revs[1].decode("ascii") == revision2
+
+
+def test_get_directories():
+    assert repository.get_directories("") == []
+    assert repository.get_directories("Makefile") == []
+    assert repository.get_directories("dom/aFile.jsm") == ["dom"]
+    assert set(
+        repository.get_directories("tools/code-coverage/CodeCoverageHandler.cpp")
+    ) == {"tools", "tools/code-coverage"}
+
+    assert repository.get_directories([""]) == []
+    assert repository.get_directories(["Makefile"]) == []
+    assert repository.get_directories(["dom/aFile.jsm"]) == ["dom"]
+    assert set(
+        repository.get_directories(["tools/code-coverage/CodeCoverageHandler.cpp"])
+    ) == {"tools", "tools/code-coverage"}
+    assert set(
+        repository.get_directories(
+            ["dom/aFile.jsm", "tools/code-coverage/CodeCoverageHandler.cpp"]
+        )
+    ) == {"dom", "tools", "tools/code-coverage"}
+
+
+def test_calculate_experiences():
+    commits = [
+        repository.Commit(
+            node="commit1",
+            author="author1",
+            desc="commit1",
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2019, 1, 1),
+            bug="123",
+            backedoutby="",
+            author_email="author1@mozilla.org",
+            files=["dom/file1.cpp", "apps/file1.jsm"],
+            file_copies={},
+            reviewers=("reviewer1", "reviewer2"),
+        ),
+        repository.Commit(
+            node="commit2",
+            author="author2",
+            desc="commit2",
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2019, 1, 1),
+            bug="123",
+            backedoutby="",
+            author_email="author2@mozilla.org",
+            files=["dom/file1.cpp"],
+            file_copies={},
+            reviewers=("reviewer1",),
+        ),
+        repository.Commit(
+            node="commit3",
+            author="author1",
+            desc="commit3",
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2019, 1, 1),
+            bug="123",
+            backedoutby="",
+            author_email="author1@mozilla.org",
+            files=["dom/file2.cpp", "apps/file1.jsm"],
+            file_copies={},
+            reviewers=("reviewer2",),
+        ),
+        repository.Commit(
+            node="commit4",
+            author="author2",
+            desc="commit4",
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2020, 1, 1),
+            bug="123",
+            backedoutby="",
+            author_email="author2@mozilla.org",
+            files=["dom/file1.cpp", "apps/file2.jsm"],
+            file_copies={},
+            reviewers=("reviewer1", "reviewer2"),
+        ),
+        repository.Commit(
+            node="commit5",
+            author="author3",
+            desc="commit5",
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2020, 1, 2),
+            bug="123",
+            backedoutby="",
+            author_email="author3@mozilla.org",
+            files=["dom/file1.cpp"],
+            file_copies={"dom/file1.cpp": "dom/file1copied.cpp"},
+            reviewers=("reviewer3",),
+        ),
+        repository.Commit(
+            node="commit6",
+            author="author3",
+            desc="commit6",
+            date=datetime(2019, 1, 1),
+            pushdate=datetime(2020, 1, 3),
+            bug="123",
+            backedoutby="",
+            author_email="author3@mozilla.org",
+            files=["dom/file1.cpp", "dom/file1copied.cpp"],
+            file_copies={},
+            reviewers=("reviewer3",),
+        ),
+    ]
+
+    repository.path_to_component = {
+        "dom/file1.cpp": "Core::DOM",
+        "dom/file1copied.cpp": "Core::DOM",
+        "dom/file2.cpp": "Core::Layout",
+        "apps/file1.jsm": "Firefox::Boh",
+        "apps/file2.jsm": "Firefox::Boh",
+    }
+
+    repository.calculate_experiences(commits)
+
+    assert repository.experiences_by_commit["total"]["author"]["commit1"] == 0
+    assert repository.experiences_by_commit["total"]["author"]["commit2"] == 0
+    assert repository.experiences_by_commit["total"]["author"]["commit3"] == 1
+    assert repository.experiences_by_commit["total"]["author"]["commit4"] == 1
+    assert repository.experiences_by_commit["total"]["author"]["commit5"] == 0
+    assert repository.experiences_by_commit["total"]["author"]["commit6"] == 1
+
+    assert repository.experiences_by_commit["90_days"]["author"]["commit1"] == 0
+    assert repository.experiences_by_commit["90_days"]["author"]["commit2"] == 0
+    assert repository.experiences_by_commit["90_days"]["author"]["commit3"] == 1
+    assert repository.experiences_by_commit["90_days"]["author"]["commit4"] == 0
+    assert repository.experiences_by_commit["90_days"]["author"]["commit5"] == 0
+    assert repository.experiences_by_commit["90_days"]["author"]["commit6"] == 1
+
+    assert repository.experiences_by_commit["total"]["reviewer"]["commit1"] == 0
+    assert repository.experiences_by_commit["total"]["reviewer"]["commit2"] == 1
+    assert repository.experiences_by_commit["total"]["reviewer"]["commit3"] == 1
+    assert repository.experiences_by_commit["total"]["reviewer"]["commit4"] == 4
+    assert repository.experiences_by_commit["total"]["reviewer"]["commit5"] == 0
+    assert repository.experiences_by_commit["total"]["reviewer"]["commit6"] == 1
+
+    assert repository.experiences_by_commit["90_days"]["reviewer"]["commit1"] == 0
+    assert repository.experiences_by_commit["90_days"]["reviewer"]["commit2"] == 1
+    assert repository.experiences_by_commit["90_days"]["reviewer"]["commit3"] == 1
+    assert repository.experiences_by_commit["90_days"]["reviewer"]["commit4"] == 0
+    assert repository.experiences_by_commit["90_days"]["reviewer"]["commit5"] == 0
+    assert repository.experiences_by_commit["90_days"]["reviewer"]["commit6"] == 1
+
+    assert repository.experiences_by_commit["total"]["file"]["commit1"] == 0
+    assert repository.experiences_by_commit["total"]["file"]["commit2"] == 1
+    assert repository.experiences_by_commit["total"]["file"]["commit3"] == 1
+    assert repository.experiences_by_commit["total"]["file"]["commit4"] == 2
+    assert repository.experiences_by_commit["total"]["file"]["commit5"] == 3
+    assert repository.experiences_by_commit["total"]["file"]["commit6"] == 4
+
+    assert repository.experiences_by_commit["90_days"]["file"]["commit1"] == 0
+    assert repository.experiences_by_commit["90_days"]["file"]["commit2"] == 1
+    assert repository.experiences_by_commit["90_days"]["file"]["commit3"] == 1
+    assert repository.experiences_by_commit["90_days"]["file"]["commit4"] == 0
+    assert repository.experiences_by_commit["90_days"]["file"]["commit5"] == 1
+    assert repository.experiences_by_commit["90_days"]["file"]["commit6"] == 2
+
+    assert repository.experiences_by_commit["total"]["directory"]["commit1"] == 0
+    assert repository.experiences_by_commit["total"]["directory"]["commit2"] == 1
+    assert repository.experiences_by_commit["total"]["directory"]["commit3"] == 2
+    assert repository.experiences_by_commit["total"]["directory"]["commit4"] == 3
+    assert repository.experiences_by_commit["total"]["directory"]["commit5"] == 4
+    assert repository.experiences_by_commit["total"]["directory"]["commit6"] == 5
+
+    assert repository.experiences_by_commit["90_days"]["directory"]["commit1"] == 0
+    assert repository.experiences_by_commit["90_days"]["directory"]["commit2"] == 1
+    assert repository.experiences_by_commit["90_days"]["directory"]["commit3"] == 2
+    assert repository.experiences_by_commit["90_days"]["directory"]["commit4"] == 0
+    assert repository.experiences_by_commit["90_days"]["directory"]["commit5"] == 1
+    assert repository.experiences_by_commit["90_days"]["directory"]["commit6"] == 2
+
+    assert repository.experiences_by_commit["total"]["component"]["commit1"] == 0
+    assert repository.experiences_by_commit["total"]["component"]["commit2"] == 1
+    assert repository.experiences_by_commit["total"]["component"]["commit3"] == 1
+    assert repository.experiences_by_commit["total"]["component"]["commit4"] == 3
+    assert repository.experiences_by_commit["total"]["component"]["commit5"] == 3
+    assert repository.experiences_by_commit["total"]["component"]["commit6"] == 4
+
+    assert repository.experiences_by_commit["90_days"]["component"]["commit1"] == 0
+    assert repository.experiences_by_commit["90_days"]["component"]["commit2"] == 1
+    assert repository.experiences_by_commit["90_days"]["component"]["commit3"] == 1
+    assert repository.experiences_by_commit["90_days"]["component"]["commit4"] == 0
+    assert repository.experiences_by_commit["90_days"]["component"]["commit5"] == 1
+    assert repository.experiences_by_commit["90_days"]["component"]["commit6"] == 2