Repository: huggingface/transformers
Pull request: Improve new failures reporting

# What does this PR do?

Clean up the `new_failures` reports, also adding the "trace".

It looks like 

https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/blob/main/2026-01-30/runs/1596-21518380210/ci_results_run_models_gpu/new_failures.json

now

```json
{
    "vit": {
        "failures": {
            "multi-gpu": [
                {
                    "line": "tests/models/vit/test_modeling_vit.py::ViTModelTest::test_config",
                    "trace": "(line 223)  assert 1 == 2"
                },
                {
                    "line": "tests/models/vit/test_modeling_vit.py::ViTModelTest::test_foo",
                    "trace": "(line 220)  assert 1 == 3"
                }
            ],
            "single-gpu": [
                {
                    "line": "tests/models/vit/test_modeling_vit.py::ViTModelTest::test_config",
                    "trace": "(line 223)  assert 1 == 2"
                },
                {
                    "line": "tests/models/vit/test_modeling_vit.py::ViTModelTest::test_foo",
                    "trace": "(line 220)  assert 1 == 3"
                }
            ]
        },
        "job_link": {
            "multi": "https://github.com/huggingface/transformers/actions/runs/21518380210/job/62003150802",
            "single": "https://github.com/huggingface/transformers/actions/runs/21518380210/job/62003150793"
        }
    }
}

```

Changed files:
filename: utils/check_bad_commit.py
patch: @@ -246,12 +246,23 @@ def get_commit_info(commit):
             reports = json.load(fp)
 
         for model in reports:
+            # We change the format of "new_failures.json" in PR #XXXXX, let's handle both formats for a few weeks.
+            if "failures" in reports[model]:
+                if "job_link" in reports[model]:
+                    for device, device_failures in reports[model]["failures"].items():
+                        if device in reports[model]["job_link"]:
+                            for failure in device_failures:
+                                failure["job_link"] = reports[model]["job_link"][device]
+                    del reports[model]["job_link"]
+                reports[model] = reports[model]["failures"]
+
             # TODO: make this script able to deal with both `single-gpu` and `multi-gpu` via a new argument.
             reports[model].pop("multi-gpu", None)
             failed_tests = reports[model]["single-gpu"]
 
             failed_tests_with_bad_commits = []
-            for test in failed_tests:
+            for failure in failed_tests:
+                test, trace = failure["line"], failure["trace"]
                 commit, status = find_bad_commit(
                     target_test=test, start_commit=args.start_commit, end_commit=args.end_commit
                 )
@@ -264,6 +275,7 @@ def get_commit_info(commit):
                     commit_info_cache[commit] = commit_info
 
                 info.update(commit_info)
+                info["trace"] = trace
                 failed_tests_with_bad_commits.append(info)
 
             # If no single-gpu test failures, remove the key

filename: utils/notification_service.py
patch: @@ -583,61 +583,19 @@ def payload(self) -> str:
         if len(self.selected_warnings) > 0:
             blocks.append(self.warnings)
 
-        new_failure_blocks = []
         for idx, (prev_workflow_run_id, prev_ci_artifacts) in enumerate(
             [self.prev_ci_artifacts] + self.other_ci_artifacts
         ):
-            if idx == 0:
-                # This is the truncated version to show on slack. For now.
-                new_failure_blocks = self.get_new_model_failure_blocks(
-                    prev_ci_artifacts=prev_ci_artifacts, with_header=False
-                )
-
-            # To save the list of new model failures and uploaed to hub repositories
-            extra_blocks = self.get_new_model_failure_blocks(prev_ci_artifacts=prev_ci_artifacts, to_truncate=False)
-            if extra_blocks:
+            # To save the dict of new failures and upload to hub repositories
+            new_failures = self.get_new_failures(prev_ci_artifacts=prev_ci_artifacts)
+            if new_failures:
                 filename = "new_failures"
                 if idx > 0:
                     filename = f"{filename}_against_{prev_workflow_run_id}"
 
-                failure_text = extra_blocks[-1]["text"]["text"]
-                file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/{filename}.txt")
-                with open(file_path, "w", encoding="UTF-8") as fp:
-                    fp.write(failure_text)
-
-                # upload results to Hub dataset
-                file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/{filename}.txt")
-                _ = api.upload_file(
-                    path_or_fileobj=file_path,
-                    path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/{filename}.txt",
-                    repo_id=report_repo_id,
-                    repo_type="dataset",
-                    token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
-                )
-
-                # extra processing to save to json format
-                new_failed_tests = {}
-                nb_new_failed_tests = 0
-                for line in failure_text.split():
-                    if "https://github.com/huggingface/transformers/actions/runs" in line:
-                        pattern = r"<(https://github.com/huggingface/transformers/actions/runs/.+?/job/.+?)\|(.+?)>"
-                        items = re.findall(pattern, line)
-                    elif "tests/" in line:
-                        # TODO: Improve the condition here.
-                        if "tests/models/" in line or (
-                            "tests/quantization/" in line and job_name == "run_quantization_torch_gpu"
-                        ):
-                            model = line.split("/")[2]
-                        else:
-                            model = line.split("/")[1]
-                        if model not in new_failed_tests:
-                            new_failed_tests[model] = {"single-gpu": [], "multi-gpu": []}
-                        for _, device in items:
-                            new_failed_tests[model][f"{device}-gpu"].append(line)
-                            nb_new_failed_tests += 1
                 file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/{filename}.json")
                 with open(file_path, "w", encoding="UTF-8") as fp:
-                    json.dump(new_failed_tests, fp, ensure_ascii=False, indent=4)
+                    json.dump(new_failures, fp, ensure_ascii=False, indent=4)
 
                 # upload results to Hub dataset
                 file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/{filename}.json")
@@ -650,6 +608,12 @@ def payload(self) -> str:
                 )
                 new_failures_url = f"https://huggingface.co/datasets/{report_repo_id}/raw/{commit_info.oid}/{report_repo_folder}/ci_results_{job_name}/{filename}.json"
 
+                nb_new_failed_tests = 0
+                for results in new_failures.values():
+                    if "failures" in results:
+                        for failures_list in results["failures"].values():
+                            nb_new_failed_tests += len(failures_list)
+
                 if idx == 0:
                     block = {
                         "type": "section",
@@ -695,9 +659,6 @@ def payload(self) -> str:
             }
             blocks.append(block)
 
-        if len(new_failure_blocks) > 0:
-            blocks.extend(new_failure_blocks)
-
         return json.dumps(blocks)
 
     @staticmethod
@@ -817,10 +778,22 @@ def get_reply_blocks(self, job_name, job_result, failures, device, text):
             {"type": "section", "text": {"type": "mrkdwn", "text": failure_text}},
         ]
 
-    def get_new_model_failure_blocks(self, prev_ci_artifacts, with_header=True, to_truncate=True):
+    def get_new_failures(self, prev_ci_artifacts, include_all=False):
+        """
+        Get new model failures compared to previous CI artifacts.
+
+        Args:
+            prev_ci_artifacts: Previous CI artifacts to compare against
+            include_all: If True, include all failures without checking prev_ci_artifacts
+
+        Returns:
+            Dictionary with new failures, or empty dict if no failures
+        """
         if prev_ci_artifacts is None:
-            return []
+            # Not compare against a previous run
+            return {}
 
+        # Get target results
         if len(self.model_results) > 0:
             target_results = self.model_results
         else:
@@ -831,68 +804,75 @@ def get_new_model_failure_blocks(self, prev_ci_artifacts, with_header=True, to_t
             target_results = {job_name: target_results}
         sorted_dict = sorted(target_results.items(), key=lambda t: t[0])
 
-        job = job_to_test_map[job_name]
-        prev_model_results = {}
-        if (
-            f"ci_results_{job_name}" in prev_ci_artifacts
-            and f"{test_to_result_name[job]}_results.json" in prev_ci_artifacts[f"ci_results_{job_name}"]
-        ):
-            prev_model_results = json.loads(
-                prev_ci_artifacts[f"ci_results_{job_name}"][f"{test_to_result_name[job]}_results.json"]
-            )
-            # Make the format uniform between `model_results` and `additional_results[XXX]`
-            if "failures" in prev_model_results:
-                prev_model_results = {job_name: prev_model_results}
+        # Get previous model results
+        prev_results = {}
+        if not include_all and prev_ci_artifacts is not None:
+            job = job_to_test_map[job_name]
+            if (
+                f"ci_results_{job_name}" in prev_ci_artifacts
+                and f"{test_to_result_name[job]}_results.json" in prev_ci_artifacts[f"ci_results_{job_name}"]
+            ):
+                prev_results = json.loads(
+                    prev_ci_artifacts[f"ci_results_{job_name}"][f"{test_to_result_name[job]}_results.json"]
+                )
+                # Make the format uniform between `model_results` and `additional_results[XXX]`
+                if "failures" in prev_results:
+                    prev_results = {job_name: prev_results}
 
-        all_failure_lines = {}
+        # Extract new failures
+        new_failures = {}
         for job, job_result in sorted_dict:
-            if len(job_result["failures"]):
-                devices = sorted(job_result["failures"].keys(), reverse=True)
-                for device in devices:
-                    failures = job_result["failures"][device]
-                    prev_error_lines = {}
-                    if job in prev_model_results and device in prev_model_results[job]["failures"]:
-                        prev_error_lines = {error["line"] for error in prev_model_results[job]["failures"][device]}
-
-                    url = None
-                    if job_result["job_link"] is not None and job_result["job_link"][device] is not None:
-                        url = job_result["job_link"][device]
-
-                    for idx, error in enumerate(failures):
-                        if error["line"] in prev_error_lines:
-                            continue
-
-                        new_text = f"{error['line']}\n\n"
-
-                        if new_text not in all_failure_lines:
-                            all_failure_lines[new_text] = []
-
-                        all_failure_lines[new_text].append(f"<{url}|{device}>" if url is not None else device)
-
-        MAX_ERROR_TEXT = 3000 - len("[Truncated]") - len("```New failures```\n\n")
-        if not to_truncate:
-            MAX_ERROR_TEXT = float("inf")
-        failure_text = ""
-        for line, devices in all_failure_lines.items():
-            new_text = failure_text + f"{'|'.join(devices)} gpu\n{line}"
-            if len(new_text) > MAX_ERROR_TEXT:
-                # `failure_text` here has length <= 3000
-                failure_text = failure_text + "[Truncated]"
-                break
-            # `failure_text` here has length <= MAX_ERROR_TEXT
-            failure_text = new_text
-
-        blocks = []
-        if failure_text:
-            if with_header:
-                blocks.append(
-                    {"type": "header", "text": {"type": "plain_text", "text": "New failures", "emoji": True}}
-                )
-            else:
-                failure_text = f"{failure_text}"
-            blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": failure_text}})
+            # Skip if no failures in current results
+            if "failures" not in job_result or not job_result["failures"]:
+                continue
 
-        return blocks
+            # Get previous failures for this model (if exists)
+            prev_failures = {}
+            if not include_all and job in prev_results:
+                prev_model_data = prev_results[job]
+                if "failures" in prev_model_data:
+                    prev_failures = prev_model_data["failures"]
+
+            # Build set of previous failure lines for quick lookup by device type
+            prev_failure_lines = {}
+            if not include_all:
+                for device_type, failures_list in prev_failures.items():
+                    if isinstance(failures_list, list):
+                        prev_failure_lines[device_type] = {f["line"] for f in failures_list if "line" in f}
+
+            # Check each device type (single, multi)
+            job_new_failures = {}
+            for device_type, failures_list in job_result["failures"].items():
+                if not isinstance(failures_list, list):
+                    continue
+
+                # Filter to only new failures
+                new_failures_for_device = []
+                for failure in failures_list:
+                    if "line" not in failure:
+                        continue
+
+                    # Include if flag is set, or if not in previous results
+                    if (
+                        include_all
+                        or device_type not in prev_failure_lines
+                        or failure["line"] not in prev_failure_lines[device_type]
+                    ):
+                        new_failures_for_device.append(failure)
+
+                # Only add device type if there are new failures
+                if new_failures_for_device:
+                    job_new_failures[f"{device_type}-gpu"] = new_failures_for_device
+
+            # Only add model if there are new failures
+            if job_new_failures:
+                job = job.replace("models_", "").replace("quantization_", "")
+                new_failures[job] = {"failures": job_new_failures}
+                # Add job_link if it exists
+                if "job_link" in job_result:
+                    new_failures[job]["job_link"] = job_result["job_link"]
+
+        return new_failures
 
     def post_reply(self):
         if self.thread_ts is None: