Repository: huggingface/transformers
Pull request: Operation Green CI

This is the PR where I just keep patching things until it's green and then I merge

Changed files:
filename: tests/models/plbart/test_tokenization_plbart.py
patch: @@ -18,6 +18,7 @@
 from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
 from transformers.testing_utils import (
     get_tests_dir,
+    is_flaky,
     nested_simplify,
     require_sentencepiece,
     require_tokenizers,
@@ -288,6 +289,7 @@ def check_language_codes(self):
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"], 50002)
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"], 50003)
 
+    @is_flaky
     def test_python_en_tokenizer_batch_encode_plus(self):
         ids = self.tokenizer(self.src_text).input_ids[0]
         self.assertListEqual(self.expected_src_tokens, ids)
@@ -300,6 +302,7 @@ def test_python_en_tokenizer_decode_ignores_language_codes(self):
         self.assertEqual(result, expected_english)
         self.assertNotIn(self.tokenizer.eos_token, result)
 
+    @is_flaky
     def test_python_en_tokenizer_truncation(self):
         src_text = ["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 20]
         self.assertIsInstance(src_text[0], str)
@@ -320,6 +323,7 @@ def test_special_tokens_unaffacted_by_save_load(self):
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids, original_special_tokens)
 
     @require_torch
+    @is_flaky
     def test_batch_fairseq_parity(self):
         batch = self.tokenizer(self.src_text, text_target=self.tgt_text, padding=True, return_tensors="pt")
         batch["decoder_input_ids"] = shift_tokens_right(batch["labels"], self.tokenizer.pad_token_id)

filename: tests/trainer/test_trainer.py
patch: @@ -928,7 +928,7 @@ def test_gradient_accumulation_loss_alignment_with_loss_func(self):
         set_seed(42)
 
         model_name = "roneneldan/TinyStories-33M"
-        dataset_name = "wikitext"
+        dataset_name = "Salesforce/wikitext"
         dataset_config = "wikitext-2-raw-v1"
         dataset = datasets.load_dataset(dataset_name, dataset_config, split="train[:40]")
         tokenizer = AutoTokenizer.from_pretrained(model_name)

filename: utils/fetch_hub_objects_for_ci.py
patch: @@ -1,4 +1,5 @@
 import os
+import re
 
 import httpx
 from huggingface_hub import hf_hub_download, snapshot_download
@@ -48,6 +49,88 @@ def url_to_local_path(url, return_url_if_not_found=True):
     return filename
 
 
+def parse_hf_url(url):
+    """
+    Parse a HuggingFace Hub URL into components for hf_hub_download.
+
+    Returns dict with (repo_id, filename, repo_type, revision) or None if not a HF URL.
+    """
+    pattern = r"https://huggingface\.co/(datasets/)?([^/]+/[^/]+)/resolve/([^/]+)/(.+)"
+    match = re.match(pattern, url)
+    if not match:
+        return None
+
+    is_dataset = match.group(1) is not None
+    revision = match.group(3)
+    return {
+        "repo_id": match.group(2),
+        "filename": match.group(4),
+        "repo_type": "dataset" if is_dataset else "model",
+        "revision": revision if revision != "main" else None,
+    }
+
+
+def validate_downloaded_content(filepath):
+    with open(filepath, "r") as f:
+        header = f.read(32)
+
+    for bad_sig in ["<!doctype", "<html", '{"error', '{"message']:
+        if header.lower().startswith(bad_sig):
+            raise ValueError(
+                f"Downloaded file appears to be an HTML error page, not a valid media file. "
+                f"This may indicate rate limiting. File starts with: {header[:50]!r}"
+            )
+
+    file_size = os.path.getsize(filepath)
+    if file_size < 100:
+        raise ValueError(f"Downloaded file is suspiciously small ({file_size} bytes).")
+
+    return True
+
+
+def download_test_file(url):
+    """
+    Download a URL to a local file, using hf_hub_download for HF URLs.
+
+    For HuggingFace URLs, uses hf_hub_download which handles authentication
+    automatically via the HF_TOKEN environment variable.
+
+    Returns the local filename.
+    """
+    filename = url.split("/")[-1]
+
+    # Skip if file already exists
+    if os.path.exists(filename):
+        print(f"File already exists: {filename}")
+        return filename
+
+    # Check if this is a HuggingFace URL
+    hf_parts = parse_hf_url(url)
+
+    if hf_parts:
+        # Use hf_hub_download for HF URLs - handles auth automatically via HF_TOKEN env var
+        print(f"Downloading {filename} from HuggingFace Hub...")
+        try:
+            hf_hub_download(**hf_parts, local_dir=".")
+            print(f"Successfully downloaded: {filename}")
+        except Exception as e:
+            print(f"Error downloading {filename} from HuggingFace Hub: {e}")
+            raise
+    else:
+        # Use httpx for non-HF URLs (COCO, Britannica, etc.)
+        print(f"Downloading {filename} from external URL...")
+        with open(filename, "wb") as f:
+            with httpx.stream("GET", url, follow_redirects=True) as resp:
+                resp.raise_for_status()
+                f.writelines(resp.iter_bytes(chunk_size=8192))
+
+        # Validate the downloaded content
+        validate_downloaded_content(filename)
+        print(f"Successfully downloaded: {filename}")
+
+    return filename
+
+
 if __name__ == "__main__":
     if _run_pipeline_tests:
         import datasets
@@ -197,19 +280,4 @@ def url_to_local_path(url, return_url_if_not_found=True):
 
     # Download files from URLs to local directory
     for url in URLS_FOR_TESTING_DATA:
-        filename = url_to_local_path(url, return_url_if_not_found=False)
-
-        # Skip if file already exists
-        if os.path.exists(filename):
-            print(f"File already exists: {filename}")
-            continue
-
-        print(f"Downloading {filename}...")
-        try:
-            with open(filename, "wb") as f:
-                with httpx.stream("GET", url) as resp:
-                    resp.raise_for_status()
-                    f.writelines(resp.iter_bytes(chunk_size=8192))
-            print(f"Successfully downloaded: {filename}")
-        except httpx.HTTPError as e:
-            print(f"Error downloading {filename}: {e}")
+        download_test_file(url)