Repository: huggingface/transformers
Pull request: Transformers serve -> list all generative models from the cache 

Instead of relying on an inadequate list, look for generative models within the cache. Generative models are identified by taking checking in their config whether the architecture defined is in the `MODEL_FOR_CAUSAL_LM_MAPPING_NAMES` mapping.

Changed files:
filename: src/transformers/cli/serve.py
patch: @@ -11,13 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import asyncio
 import base64
 import copy
-import datetime
 import enum
-import functools
 import gc
 import io
 import json
@@ -28,14 +25,15 @@
 import uuid
 from collections.abc import Generator, Iterable
 from contextlib import asynccontextmanager
+from functools import lru_cache
 from io import BytesIO
 from threading import Thread
 from typing import TYPE_CHECKING, Annotated, Optional, TypedDict, Union
 
 import typer
-from huggingface_hub import model_info
-from huggingface_hub.constants import HF_HUB_OFFLINE
+from huggingface_hub import scan_cache_dir
 from tokenizers.decoders import DecodeStream
+from tqdm import tqdm
 
 import transformers
 from transformers import BitsAndBytesConfig, GenerationConfig
@@ -731,51 +729,54 @@ def chunk_to_sse_element(chunk: ChatCompletionChunk | BaseModel) -> str:
         """
         return f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
 
-    @functools.cache
-    def get_gen_models(self) -> list[dict[str, any]]:
+    @staticmethod
+    @lru_cache
+    def get_gen_models(cache_dir: str | None = None) -> list[dict[str, any]]:
         """
-        This is by no means a limit to which models may be instantiated with `transformers serve`: any chat-based
-        model working with generate can work.
-
-        This is a limited list of models to ensure we have a discoverable /v1/models endpoint for third-party
-        integrations.
+        List LLMs and VLMs in the cache.
         """
-        models = [
-            "Menlo/Jan-nano",
-            "Menlo/Jan-nano-128k",
-            "Qwen/Qwen2.5-0.5B-Instruct",
-            "Qwen/Qwen2.5-3B-Instruct",
-            "Qwen/Qwen2.5-7B-Instruct",
-            "Qwen/Qwen2.5-14B-Instruct",
-            "meta-llama/Llama-3.1-8B-Instruct",
-            "meta-llama/Llama-3.2-1B-Instruct",
-            "meta-llama/Llama-3.3-70B-Instruct",
-            "HuggingFaceTB/SmolVLM-Instruct",
-            "ibm-granite/granite-vision-3.2-2b",
-            "Qwen/Qwen2.5-VL-7B-Instruct",
-        ]
-
-        if HF_HUB_OFFLINE:
-            return [
-                {
-                    "id": model,
-                    "object": "model",
-                    "created": datetime.datetime.now().timestamp(),
-                    "owned_by": model.split("/")[0],
-                }
-                for model in models
-            ]
-        else:
-            model_infos = [model_info(model) for model in models]
-            return [
-                {
-                    "id": model.id,
-                    "object": "model",
-                    "created": model.created_at.timestamp(),
-                    "owned_by": model.author,
-                }
-                for model in model_infos
-            ]
+        from transformers.models.auto.modeling_auto import (
+            MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
+            MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES,
+        )
+
+        generative_models = []
+
+        logger.warning("Scanning the cache directory for LLMs and VLMs.")
+        for repo in tqdm(scan_cache_dir(cache_dir).repos):
+            if repo.repo_type != "model":
+                continue
+
+            refs = repo.refs
+            for ref, revision_info in refs.items():
+                files = revision_info.files
+                config_path = next((f.file_path for f in files if f.file_name == "config.json"), None)
+
+                if not config_path:
+                    continue
+
+                config = json.loads(config_path.open().read())
+
+                if not (isinstance(config, dict) and "architectures" in config):
+                    continue
+
+                architectures = config["architectures"]
+                llms = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES.values()
+                vlms = MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES.values()
+
+                if any(arch for arch in architectures if arch in [*llms, *vlms]):
+                    author = repo.repo_id.split("/") if "/" in repo.repo_id else ""
+                    repo_handle = repo.repo_id + (f"@{ref}" if ref != "main" else "")
+                    generative_models.append(
+                        {
+                            "owned_by": author,
+                            "id": repo_handle,
+                            "object": "model",
+                            "created": repo.last_modified,
+                        }
+                    )
+
+        return generative_models
 
     def continuous_batching_chat_completion(self, req: dict, request_id: str) -> StreamingResponse | JSONResponse:
         """

filename: tests/cli/test_serve.py
patch: @@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import tempfile
 import time
 import unittest
 from threading import Thread
 from unittest.mock import Mock, patch
 
 import httpx
-from huggingface_hub import ChatCompletionStreamOutput, InferenceClient
+from huggingface_hub import ChatCompletionStreamOutput, InferenceClient, hf_hub_download
 from parameterized import parameterized
 
 from transformers import GenerationConfig
@@ -153,6 +154,33 @@ def test_build_chat_completion_chunk():
     assert expected_choices_content in chunk
 
 
+def test_generative_model_list():
+    with tempfile.TemporaryDirectory() as cache_dir:
+        # "download" a few models, including some non-generative models
+        hf_hub_download("Menlo/Jan-nano", "config.json", cache_dir=cache_dir)
+        hf_hub_download("Menlo/Jan-nano-128k", "config.json", cache_dir=cache_dir)
+        hf_hub_download("Qwen/Qwen2.5-0.5B-Instruct", "config.json", cache_dir=cache_dir)
+        hf_hub_download("HuggingFaceTB/SmolVLM-Instruct", "config.json", cache_dir=cache_dir)
+        hf_hub_download("google-bert/bert-base-cased", "config.json", cache_dir=cache_dir)
+
+        expected_results = {
+            "HuggingFaceTB/SmolVLM-Instruct": ["HuggingFaceTB", "SmolVLM-Instruct"],
+            "Qwen/Qwen2.5-0.5B-Instruct": ["Qwen", "Qwen2.5-0.5B-Instruct"],
+            "Menlo/Jan-nano": ["Menlo", "Jan-nano"],
+            "Menlo/Jan-nano-128k": ["Menlo", "Jan-nano-128k"],
+        }
+
+        # list models
+        result = Serve.get_gen_models(cache_dir)
+        assert len(expected_results) == len(result)
+
+        local_repos = {repo["id"]: repo["owned_by"] for repo in result}
+
+        for key, value in expected_results.items():
+            assert key in local_repos
+            assert local_repos[key] == value
+
+
 @require_openai
 def test_build_response_event():
     """