Repository: huggingface/transformers
File: utils/fetch_hub_objects_for_ci.py (Python)

Code before:
    For HuggingFace URLs, uses hf_hub_download which handles authentication
    automatically via the HF_TOKEN environment variable.

    Returns the local filename.
    """
    filename = url.split("/")[-1]

    # Skip if file already exists
    if os.path.exists(filename):
        print(f"File already exists: {filename}")
        return filename

    # Check if this is a HuggingFace URL
    hf_parts = parse_hf_url(url)

    if hf_parts:
        # Use hf_hub_download for HF URLs - handles auth automatically via HF_TOKEN env var
        print(f"Downloading {filename} from HuggingFace Hub...")
        try:
            cached_path = hf_hub_download(
                repo_id=hf_parts["repo_id"],
                filename=hf_parts["filename"],
                repo_type=hf_parts["repo_type"],
                revision=hf_parts["revision"],
            )
            os.symlink(cached_path, filename)
            print(f"Successfully downloaded: {filename}")
        except Exception as e:
            print(f"Error downloading {filename} from HuggingFace Hub: {e}")
            raise
    else:
        # Use httpx for non-HF URLs (COCO, Britannica, etc.)
        print(f"Downloading {filename} from external URL...")
        try:
            with open(filename, "wb") as f:
                with httpx.stream("GET", url, follow_redirects=True) as resp:
                    resp.raise_for_status()
                    f.writelines(resp.iter_bytes(chunk_size=8192))

            # Validate the downloaded content
            validate_downloaded_content(filename)
            print(f"Successfully downloaded: {filename}")
        except httpx.HTTPError as e:
            print(f"Error downloading {filename}: {e}")
            raise
        except ValueError as e:
            # Validation failed - remove corrupted file
            print(f"Validation failed for {filename}: {e}")
            raise

    return filename

Review comment (suggestion):
Let's use `local_dir` arg as well in `hf_hub_download` instead of symlink!

Code after:
    For HuggingFace URLs, uses hf_hub_download which handles authentication
    automatically via the HF_TOKEN environment variable.

    Returns the local filename.
    """
    filename = url.split("/")[-1]

    # Skip if file already exists
    if os.path.exists(filename):
        print(f"File already exists: {filename}")
        return filename

    # Check if this is a HuggingFace URL
    hf_parts = parse_hf_url(url)

    if hf_parts:
        # Use hf_hub_download for HF URLs - handles auth automatically via HF_TOKEN env var
        print(f"Downloading {filename} from HuggingFace Hub...")
        try:
            hf_hub_download(**hf_parts, local_dir=".")
            print(f"Successfully downloaded: {filename}")
        except Exception as e:
            print(f"Error downloading {filename} from HuggingFace Hub: {e}")
            raise
    else:
        # Use httpx for non-HF URLs (COCO, Britannica, etc.)
        print(f"Downloading {filename} from external URL...")
        with open(filename, "wb") as f:
            with httpx.stream("GET", url, follow_redirects=True) as resp:
                resp.raise_for_status()
                f.writelines(resp.iter_bytes(chunk_size=8192))

        # Validate the downloaded content
        validate_downloaded_content(filename)
        print(f"Successfully downloaded: {filename}")

    return filename


if __name__ == "__main__":
    if _run_pipeline_tests:
        import datasets

        _ = datasets.load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        _ = datasets.load_dataset("hf-internal-testing/fixtures_image_utils", split="test", revision="refs/pr/1")
        _ = hf_hub_download(repo_id="nateraw/video-demo", filename="archery.mp4", repo_type="dataset")

        hf_hub_download("Narsil/asr_dummy", filename="hindi.ogg", repo_type="dataset")
        hf_hub_download(repo_id="hf-internal-testing/bool-masked-pos", filename="bool_masked_pos.pt")
        hf_hub_download(
            repo_id="hf-internal-testing/fixtures_docvqa",