Repository: huggingface/transformers
Pull request: Better new failures reporting for PR comment CI

# What does this PR do?

Provide more information like

- the link to the hub repository containing the whole set of failed tests caused by the PR

Also clean up some internal logic

## CI Results
[Workflow Run ⚙️](https://github.com/huggingface/transformers/actions/runs/21529052863)

### Model CI Report

❌ **[2 new failed tests from this PR](https://huggingface.co/datasets/hf-internal-testing/transformers_pr_ci/raw/5951a00357bff86fe3d851da257fde38d4121a46/2026-01-30/runs/27894-21529052863/ci_results_run_models_gpu/new_failures_with_bad_commit.json)** 😭

- [vit](https://github.com/huggingface/transformers/actions/runs/21529052863/job/62040267894):
    tests/models/vit/test_modeling_vit.py::ViTModelTest::test_config
    tests/models/vit/test_modeling_vit.py::ViTModelTest::test_foo


Changed files:
filename: .github/workflows/check_failed_tests.yml
patch: @@ -24,10 +24,6 @@ on:
       pr_number:
         required: false
         type: string
-    outputs:
-      report:
-        description: "Content of the report of new failures"
-        value: ${{ jobs.process_new_failures_with_commit_info.outputs.report }}
 
 env:
   HF_HOME: /mnt/cache
@@ -205,8 +201,6 @@ jobs:
     if: needs.check_new_failures.outputs.process == 'true'
     runs-on:
       group: aws-g5-4xlarge-cache
-    outputs:
-      report: ${{ steps.set_output.outputs.report }}
     container:
       image: ${{ inputs.docker }}
       options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
@@ -262,28 +256,19 @@ jobs:
             echo EOF
           } >> "$GITHUB_ENV"
 
-      # The output is useful if a caller needs more processing, for example, we have a chain
-      # self-comment-ci.yml -> self-scheduled.yml -> this one (check_failed_tests.yml),
-      # and `self-comment-ci.yml` needs further processing before sending a GitHub comment to the pull request page.
-      - name: Show results & Set outputs
-        id: set_output
+      - name: Show results
         working-directory: /transformers
         run: |
           ls -l new_failures_with_bad_commit.json
           cat new_failures_with_bad_commit.json
 
-          {
-            echo 'report<<EOF'
-            cat new_failures_with_bad_commit.json
-            echo ''  # Force a newline
-            echo EOF
-          } >> "$GITHUB_OUTPUT"
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: new_failures_with_bad_commit_${{ inputs.job }}
-          path: /transformers/new_failures_with_bad_commit.json
+          path: |
+            /transformers/new_failures_with_bad_commit.json
+            /transformers/new_failures_with_bad_commit_url.txt
 
       - name: Prepare Slack report title
         working-directory: /transformers

filename: .github/workflows/self-comment-ci.yml
patch: @@ -210,23 +210,59 @@ jobs:
     if: ${{ always() && needs.create_run.result == 'success' }}
     runs-on: ubuntu-22.04
     steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: new_failures_with_bad_commit_{run_models_gpu,run_quantization_torch_gpu}
+          path: ./new_failures
+          merge-multiple: false
+
+      - name: List downloaded artifacts
+        run: |
+          echo "Downloaded artifact files:"
+          if [ -d "./new_failures/" ]; then
+            find ./new_failures/ -type f
+          else
+            echo "No artifacts downloaded (directory doesn't exist)"
+          fi
+
       - name: Show reports from jobs
-        env:
-          MODEL_REPORT: ${{ needs.model-ci.outputs.report }}
-          QUANT_REPORT: ${{ needs.quantization-ci.outputs.report }}
         run: |
-          echo "$MODEL_REPORT"
-          echo "$QUANT_REPORT"
+          echo "=== Model CI Report ==="
+          if [ -f "./new_failures/new_failures_with_bad_commit_run_models_gpu/new_failures_with_bad_commit.json" ]; then
+            cat ./new_failures/new_failures_with_bad_commit_run_models_gpu/new_failures_with_bad_commit.json
+          else
+            echo "No model CI report found"
+          fi
+          
+          echo ""
+          echo "=== Quantization CI Report ==="
+          if [ -f "./new_failures/new_failures_with_bad_commit_run_quantization_torch_gpu/new_failures_with_bad_commit.json" ]; then
+            cat ./new_failures/new_failures_with_bad_commit_run_quantization_torch_gpu/new_failures_with_bad_commit.json
+          else
+            echo "No quantization CI report found"
+          fi
 
       - name: Process and filter reports
-        env:
-          MODEL_REPORT: ${{ needs.model-ci.outputs.report }}
-          QUANT_REPORT: ${{ needs.quantization-ci.outputs.report }}
         run: |
           # Preprocess with Python
           python3 << 'PYTHON_SCRIPT'
           import json
           import os
+          from pathlib import Path
+          
+          def count_failures(data):
+            """
+            Count total number of failures (excluding None commits)
+            """
+            total = 0
+            for model, model_result in data.items():
+                for device, failures in model_result.items():
+                    # Count failures where commit is not None
+                    total += sum(
+                        1 for failure in failures 
+                        if isinstance(failure, dict) and failure.get('commit') is not None
+                    )
+            return total
           
           def filter_and_format_report(data):
             """
@@ -260,25 +296,58 @@ jobs:
             
             return "\n".join(lines).strip()
           
-          # Load and filter reports
-          model_report_str = os.environ.get('MODEL_REPORT', '{}')
-          quant_report_str = os.environ.get('QUANT_REPORT', '{}')
+          # Read reports from downloaded artifact files
+          model_report_path = Path('./new_failures/new_failures_with_bad_commit_run_models_gpu/new_failures_with_bad_commit.json')
+          quant_report_path = Path('./new_failures/new_failures_with_bad_commit_run_quantization_torch_gpu/new_failures_with_bad_commit.json')
+
+          # Read URL files if they exist
+          model_url_path = Path('./new_failures/new_failures_with_bad_commit_run_models_gpu/new_failures_with_bad_commit_url.txt')
+          quant_url_path = Path('./new_failures/new_failures_with_bad_commit_run_quantization_torch_gpu/new_failures_with_bad_commit_url.txt')
+
+          model_url = None
+          if model_url_path.exists():
+              with open(model_url_path, 'r') as f:
+                  model_url = f.read().strip()
+          
+          quant_url = None
+          if quant_url_path.exists():
+              with open(quant_url_path, 'r') as f:
+                  quant_url = f.read().strip()
+
+          model_report = {}
+          if model_report_path.exists():
+              with open(model_report_path, 'r') as f:
+                  model_report = json.load(f)
+          
+          quant_report = {}
+          if quant_report_path.exists():
+              with open(quant_report_path, 'r') as f:
+                  quant_report = json.load(f)
           
-          model_report = json.loads(model_report_str) if model_report_str else {}
-          quant_report = json.loads(quant_report_str) if quant_report_str else {}
+          # Count failures
+          model_count = count_failures(model_report)
+          quant_count = count_failures(quant_report)
           
           formatted_model = filter_and_format_report(model_report)
           formatted_quant = filter_and_format_report(quant_report)
           
           # Write to files
           with open('model_ci.txt', 'w') as f:
-              f.write(formatted_model)
               if formatted_model:
+                  if model_url:
+                      f.write(f"❌ **[{model_count} new failed tests from this PR]({model_url})** 😭\n\n")
+                  else:
+                      f.write(f"❌ **{model_count} new failed tests from this PR** 😭\n\n")
+                  f.write(formatted_model)
                   f.write('\n')
           
           with open('quantization_ci.txt', 'w') as f:
-              f.write(formatted_quant)
               if formatted_quant:
+                  if quant_url:
+                      f.write(f"❌ **[{quant_count} new failed tests from this PR]({quant_url})** 😭\n\n")
+                  else:
+                      f.write(f"❌ **{quant_count} new failed tests from this PR** 😭\n\n")
+                  f.write(formatted_quant)
                   f.write('\n')
           PYTHON_SCRIPT
 
@@ -310,8 +379,6 @@ jobs:
               if [ -s model_ci.txt ]; then
                 echo '### Model CI Report'
                 echo ''
-                echo '#### ❌ Failed tests'
-                echo ''
                 cat model_ci.txt
                 echo ''
               fi
@@ -320,14 +387,12 @@ jobs:
               if [ -s quantization_ci.txt ]; then
                 echo '### Quantization CI Report'
                 echo ''
-                echo '#### ❌ Failed tests'
-                echo ''
                 cat quantization_ci.txt
                 echo ''
               fi
             else
               echo "STATUS=success" >> $GITHUB_ENV
-              echo '✅ No failing test specific to this PR 🎉 !'
+              echo '✅ No failing test specific to this PR 🎉 👏 !'
             fi
           } > comment_body.txt
 

filename: .github/workflows/self-scheduled.yml
patch: @@ -44,10 +44,6 @@ on:
       pr_number:
         required: false
         type: string
-    outputs:
-      report:
-        description: "Content of the report of new failures"
-        value: ${{ jobs.check_new_failures.outputs.report }}
 
 env:
   HF_HOME: /mnt/cache

filename: .github/workflows/slack-report.yml
patch: @@ -100,7 +100,7 @@ jobs:
             python utils/notification_service.py "$folder_slices"
           fi
 
-      # Upload complete failure tables, as they might be big and only truncated versions could be sent to Slack.
+      # Upload the directory containing CI reports prepared in `notification_service.py`
       - name: Failure table artifacts
         uses: actions/upload-artifact@v4
         with:

filename: utils/process_bad_commit_report.py
patch: @@ -71,6 +71,10 @@
         repo_type="dataset",
         token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
     )
+    url = f"https://huggingface.co/datasets/{report_repo_id}/raw/{commit_info.oid}/{report_repo_folder}/ci_results_{job_name}/new_failures_with_bad_commit.json"
+
+    with open("new_failures_with_bad_commit_url.txt", "w") as fp:
+        fp.write(url)
 
     # TODO: extend
     team_members = [