Repository: huggingface/transformers
Pull request: further reducing flakiness in `utils/check_bad_commit.py` (#41658) 

# What does this PR do?

In #41690, we run each test (on each git bisect commit) 4 times to avoid the false positive being reported to specific team member.

However, when we check if a test passes on the commit of the current CI run (in the last job of checking bad commit) that failed in the same current run in the usual job for that model, we only needs 1 pass among 4 runs to conclude it's flaky.

Prior to this PR, it requires the 4 runs all passing to say "it pass on the check time" and conclude it's flaky which is incorrect, and this leads the false positive being still reported (even with less chance).

Changed files:
filename: utils/check_bad_commit.py
patch: @@ -97,7 +97,17 @@ def is_bad_commit(target_test, commit):
     # Restore to original commit
     repo.git.checkout(original_head)
 
-    return result.returncode != 0
+    n_passed = 0
+    o = re.findall(r"====.* (\d+) passed", result.stdout)
+    if len(o) > 0:
+        n_passed = int(o[0])
+
+    n_failed = 0
+    o = re.findall(r"====.* (\d+) failed", result.stdout)
+    if len(o) > 0:
+        n_failed = int(o[0])
+
+    return result.returncode != 0, n_failed, n_passed
 
 
 def find_bad_commit(target_test, start_commit, end_commit):
@@ -113,7 +123,8 @@ def find_bad_commit(target_test, start_commit, end_commit):
     """
 
     # check if `end_commit` fails the test
-    failed_before = is_bad_commit(target_test, end_commit)
+    # (we only need one failure to conclude the test is flaky on the previous run with `end_commit`)
+    failed_before, _, _ = is_bad_commit(target_test, end_commit)
     if failed_before:
         return (
             None,
@@ -130,8 +141,9 @@ def find_bad_commit(target_test, start_commit, end_commit):
 
     # Now, we are (almost) sure `target_test` is not failing at `end_commit`
     # check if `start_commit` fail the test
-    failed_now = is_bad_commit(target_test, start_commit)
-    if not failed_now:
+    # **IMPORTANT** we only need one pass to conclude the test is flaky on the current run with `start_commit`!
+    _, n_failed, n_passed = is_bad_commit(target_test, start_commit)
+    if n_passed > 0:
         # failed on CI run, but not reproducible here --> don't report
         return None, f"flaky: test fails on the current CI run (commit: {start_commit}) but passes during the check."
 
@@ -194,12 +206,13 @@ def get_commit_info(commit):
         if pr_for_commit["merged_by"] is not None:
             merged_author = pr_for_commit["merged_by"]["login"]
 
+    url = f"https://api.github.com/repos/huggingface/transformers/commits/{commit}"
+    commit_info = requests.get(url).json()
+    parent = commit_info["parents"][0]["sha"]
     if author is None:
-        url = f"https://api.github.com/repos/huggingface/transformers/commits/{commit}"
-        commit_info = requests.get(url).json()
         author = commit_info["author"]["login"]
 
-    return {"commit": commit, "pr_number": pr_number, "author": author, "merged_by": merged_author}
+    return {"commit": commit, "pr_number": pr_number, "author": author, "merged_by": merged_author, "parent": parent}
 
 
 if __name__ == "__main__":