Repository: xorbitsai/inference
Pull request: ENH: cache status
Resolves #592 

Introduce `cache_status` into the output of `list_model_registrations`.

For LLMs, `cache_status` will be incorporated into each model's specification. For PyTorch models, this will be a boolean value. For GGML models, it will manifest as a list of booleans to reflect the status across various quantizations.

For embedding and image models, `cache_status` will be attributed to the entire model family and will consistently be a boolean.

The frontend has been updated to display whether a model is cached, using the cache_status provided by the backend:

![image](https://github.com/xorbitsai/inference/assets/109661872/56ec799f-e5c9-4464-8472-8be272c016aa)


Problem statement:
ENH: let users know if the model is cached or not
### Is your feature request related to a problem? Please describe
The describe model API should indicate to the caller whether the model has been cached.

Additionally, updates are required for the web UI:
- A "cached" tag should be added to the model card if the model has been cached.
- For GGML/GGUF models, alongside the tag, a hint should be provided when a quantization model is selected.

Hints from issue discussion:

Commit message: UT

Patch to review:
diff --git a/xinference/api/restful_api.py b/xinference/api/restful_api.py
index 9075d48479..930b1b37ee 100644
--- a/xinference/api/restful_api.py
+++ b/xinference/api/restful_api.py
@@ -23,7 +23,16 @@
 
 import gradio as gr
 import xoscar as xo
-from fastapi import APIRouter, FastAPI, File, Form, HTTPException, Request, UploadFile
+from fastapi import (
+    APIRouter,
+    FastAPI,
+    File,
+    Form,
+    HTTPException,
+    Query,
+    Request,
+    UploadFile,
+)
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
@@ -860,10 +869,12 @@ async def unregister_model(self, model_type: str, model_name: str):
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def list_model_registrations(self, model_type: str) -> List[Dict[str, Any]]:
+    async def list_model_registrations(
+        self, model_type: str, detailed: bool = Query(False)
+    ) -> List[Dict[str, Any]]:
         try:
             return await (await self._get_supervisor_ref()).list_model_registrations(
-                model_type
+                model_type, detailed=detailed
             )
         except ValueError as re:
             logger.error(re, exc_info=True)
diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py
index 63ba0c44ef..ed08c474a9 100644
--- a/xinference/core/supervisor.py
+++ b/xinference/core/supervisor.py
@@ -33,8 +33,12 @@
 )
 
 if TYPE_CHECKING:
+    from ..model.embedding import EmbeddingModelSpec
+    from ..model.image import ImageModelFamilyV1
+    from ..model.llm import LLMFamilyV1
     from .worker import WorkerActor
 
+
 logger = getLogger(__name__)
 
 
@@ -102,8 +106,47 @@ def get_status(self) -> Dict:
             "workers": self._worker_status,
         }
 
+    @staticmethod
+    def _to_llm_reg(llm_family: "LLMFamilyV1", is_builtin: bool) -> Dict[str, Any]:
+        from ..model.llm import get_cache_status
+
+        specs = []
+        for spec in llm_family.model_specs:
+            cache_status = get_cache_status(llm_family, spec)
+            specs.append({**spec.dict(), "cache_status": cache_status})
+
+        return {**llm_family.dict(), "is_builtin": is_builtin, "model_specs": specs}
+
+    @staticmethod
+    def _to_embedding_model_reg(
+        model_spec: "EmbeddingModelSpec", is_builtin: bool
+    ) -> Dict[str, Any]:
+        from ..model.embedding import get_cache_status
+
+        cache_status = get_cache_status(model_spec)
+        return {
+            **model_spec.dict(),
+            "cache_status": cache_status,
+            "is_builtin": is_builtin,
+        }
+
+    @staticmethod
+    def _to_image_model_reg(
+        model_family: "ImageModelFamilyV1", is_builtin: bool
+    ) -> Dict[str, Any]:
+        from ..model.image import get_cache_status
+
+        cache_status = get_cache_status(model_family)
+        return {
+            **model_family.dict(),
+            "cache_status": cache_status,
+            "is_builtin": is_builtin,
+        }
+
     @log_sync(logger=logger)
-    def list_model_registrations(self, model_type: str) -> List[Dict[str, Any]]:
+    def list_model_registrations(
+        self, model_type: str, detailed: bool = False
+    ) -> List[Dict[str, Any]]:
         def sort_helper(item):
             assert isinstance(item["model_name"], str)
             return item.get("model_name").lower()
@@ -111,37 +154,43 @@ def sort_helper(item):
         if model_type == "LLM":
             from ..model.llm import BUILTIN_LLM_FAMILIES, get_user_defined_llm_families
 
-            ret = [
-                {"model_name": f.model_name, "is_builtin": True}
-                for f in BUILTIN_LLM_FAMILIES
-            ]
-            user_defined_llm_families = get_user_defined_llm_families()
-            ret.extend(
-                [
-                    {"model_name": f.model_name, "is_builtin": False}
-                    for f in user_defined_llm_families
-                ]
-            )
+            ret = []
+            for family in BUILTIN_LLM_FAMILIES:
+                if detailed:
+                    ret.append(self._to_llm_reg(family, True))
+                else:
+                    ret.append({"model_name": family.model_name, "is_builtin": True})
 
-            ret.sort(key=sort_helper)
+            for family in get_user_defined_llm_families():
+                if detailed:
+                    ret.append(self._to_llm_reg(family, False))
+                else:
+                    ret.append({"model_name": family.model_name, "is_builtin": False})
 
+            ret.sort(key=sort_helper)
             return ret
         elif model_type == "embedding":
             from ..model.embedding import BUILTIN_EMBEDDING_MODELS
 
-            ret = [
-                {"model_name": model_name, "is_builtin": True}
-                for model_name in BUILTIN_EMBEDDING_MODELS
-            ]
+            ret = []
+            for model_name, family in BUILTIN_EMBEDDING_MODELS.items():
+                if detailed:
+                    ret.append(self._to_embedding_model_reg(family, is_builtin=True))
+                else:
+                    ret.append({"model_name": model_name, "is_builtin": True})
+
             ret.sort(key=sort_helper)
             return ret
         elif model_type == "image":
             from ..model.image import BUILTIN_IMAGE_MODELS
 
-            ret = [
-                {"model_name": model_name, "is_builtin": True}
-                for model_name in BUILTIN_IMAGE_MODELS
-            ]
+            ret = []
+            for model_name, family in BUILTIN_IMAGE_MODELS.items():
+                if detailed:
+                    ret.append(self._to_image_model_reg(family, is_builtin=True))
+                else:
+                    ret.append({"model_name": model_name, "is_builtin": True})
+
             ret.sort(key=sort_helper)
             return ret
         else:
diff --git a/xinference/model/embedding/__init__.py b/xinference/model/embedding/__init__.py
index 3c018c9da4..59a2592b5b 100644
--- a/xinference/model/embedding/__init__.py
+++ b/xinference/model/embedding/__init__.py
@@ -16,7 +16,7 @@
 import json
 import os
 
-from .core import EmbeddingModelSpec
+from .core import EmbeddingModelSpec, get_cache_status
 
 _model_spec_json = os.path.join(os.path.dirname(__file__), "model_spec.json")
 _model_spec_modelscope_json = os.path.join(
diff --git a/xinference/model/embedding/core.py b/xinference/model/embedding/core.py
index 94c193eb26..9a6364dafb 100644
--- a/xinference/model/embedding/core.py
+++ b/xinference/model/embedding/core.py
@@ -98,6 +98,18 @@ def cache(model_spec: EmbeddingModelSpec):
     return cache_dir
 
 
+def get_cache_status(
+    model_spec: EmbeddingModelSpec,
+) -> bool:
+    cache_dir = os.path.realpath(
+        os.path.join(XINFERENCE_CACHE_DIR, model_spec.model_name)
+    )
+    if not os.path.exists(cache_dir):
+        os.makedirs(cache_dir, exist_ok=True)
+    meta_path = os.path.join(cache_dir, "__valid_download")
+    return valid_model_revision(meta_path, model_spec.model_revision)
+
+
 class EmbeddingModel:
     def __init__(self, model_uid: str, model_path: str, device: Optional[str] = None):
         self._model_uid = model_uid
diff --git a/xinference/model/embedding/tests/test_embedding_models.py b/xinference/model/embedding/tests/test_embedding_models.py
index 9bbe132fa7..7dc6a9171e 100644
--- a/xinference/model/embedding/tests/test_embedding_models.py
+++ b/xinference/model/embedding/tests/test_embedding_models.py
@@ -11,7 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import os
+import shutil
 
 from ...utils import valid_model_revision
 from ..core import EmbeddingModel, EmbeddingModelSpec, cache
@@ -45,28 +47,33 @@
 
 
 def test_model():
-    model_path = cache(TEST_MODEL_SPEC)
-    model = EmbeddingModel("mock", model_path)
-    # input is a string
-    input_text = "what is the capital of China?"
-    model.load()
-    r = model.create_embedding(input_text)
-    assert len(r["data"]) == 1
-    for d in r["data"]:
-        assert len(d["embedding"]) == 384
-
-    # input is a lit
-    input_texts = [
-        "what is the capital of China?",
-        "how to implement quick sort in python?",
-        "Beijing",
-        "sorting algorithms",
-    ]
-    model.load()
-    r = model.create_embedding(input_texts)
-    assert len(r["data"]) == 4
-    for d in r["data"]:
-        assert len(d["embedding"]) == 384
+    model_path = None
+    try:
+        model_path = cache(TEST_MODEL_SPEC)
+        model = EmbeddingModel("mock", model_path)
+        # input is a string
+        input_text = "what is the capital of China?"
+        model.load()
+        r = model.create_embedding(input_text)
+        assert len(r["data"]) == 1
+        for d in r["data"]:
+            assert len(d["embedding"]) == 384
+
+        # input is a lit
+        input_texts = [
+            "what is the capital of China?",
+            "how to implement quick sort in python?",
+            "Beijing",
+            "sorting algorithms",
+        ]
+        model.load()
+        r = model.create_embedding(input_texts)
+        assert len(r["data"]) == 4
+        for d in r["data"]:
+            assert len(d["embedding"]) == 384
+    finally:
+        if model_path is not None:
+            shutil.rmtree(model_path, ignore_errors=True)
 
 
 def test_model_from_modelscope():
@@ -82,21 +89,38 @@ def test_model_from_modelscope():
 
 
 def test_meta_file():
-    cache_dir = cache(TEST_MODEL_SPEC)
-    meta_path = os.path.join(cache_dir, "__valid_download")
-    assert valid_model_revision(meta_path, TEST_MODEL_SPEC.model_revision)
-
-    # test another version of the same model
-    assert not valid_model_revision(meta_path, TEST_MODEL_SPEC2.model_revision)
-    cache_dir = cache(TEST_MODEL_SPEC2)
-    meta_path = os.path.join(cache_dir, "__valid_download")
-    assert valid_model_revision(meta_path, TEST_MODEL_SPEC2.model_revision)
-
-    # test functionality of the new version model
-    model = EmbeddingModel("mock", cache_dir)
-    input_text = "I can do this all day."
-    model.load()
-    r = model.create_embedding(input_text)
-    assert len(r["data"]) == 1
-    for d in r["data"]:
-        assert len(d["embedding"]) == 384
+    cache_dir = None
+    try:
+        cache_dir = cache(TEST_MODEL_SPEC)
+        meta_path = os.path.join(cache_dir, "__valid_download")
+        assert valid_model_revision(meta_path, TEST_MODEL_SPEC.model_revision)
+
+        # test another version of the same model
+        assert not valid_model_revision(meta_path, TEST_MODEL_SPEC2.model_revision)
+        cache_dir = cache(TEST_MODEL_SPEC2)
+        meta_path = os.path.join(cache_dir, "__valid_download")
+        assert valid_model_revision(meta_path, TEST_MODEL_SPEC2.model_revision)
+
+        # test functionality of the new version model
+        model = EmbeddingModel("mock", cache_dir)
+        input_text = "I can do this all day."
+        model.load()
+        r = model.create_embedding(input_text)
+        assert len(r["data"]) == 1
+        for d in r["data"]:
+            assert len(d["embedding"]) == 384
+    finally:
+        shutil.rmtree(cache_dir, ignore_errors=True)
+
+
+def test_get_cache_status():
+    from ..core import get_cache_status
+
+    model_path = None
+    try:
+        assert get_cache_status(TEST_MODEL_SPEC) is False
+        model_path = cache(TEST_MODEL_SPEC)
+        assert get_cache_status(TEST_MODEL_SPEC) is True
+    finally:
+        if model_path is not None:
+            shutil.rmtree(model_path, ignore_errors=True)
diff --git a/xinference/model/image/__init__.py b/xinference/model/image/__init__.py
index 2f912d3928..432bbdd3a9 100644
--- a/xinference/model/image/__init__.py
+++ b/xinference/model/image/__init__.py
@@ -16,7 +16,7 @@
 import json
 import os
 
-from .core import ImageModelFamilyV1
+from .core import ImageModelFamilyV1, get_cache_status
 
 _model_spec_json = os.path.join(os.path.dirname(__file__), "model_spec.json")
 BUILTIN_IMAGE_MODELS = dict(
diff --git a/xinference/model/image/core.py b/xinference/model/image/core.py
index 0170f7c8cb..65c699d024 100644
--- a/xinference/model/image/core.py
+++ b/xinference/model/image/core.py
@@ -20,6 +20,7 @@
 
 from ...constants import XINFERENCE_CACHE_DIR
 from ..core import ModelDescription
+from ..utils import valid_model_revision
 from .stable_diffusion.core import DiffusionModel
 
 MAX_ATTEMPTS = 3
@@ -70,6 +71,11 @@ def cache(model_spec: ImageModelFamilyV1):
     )
     if not os.path.exists(cache_dir):
         os.makedirs(cache_dir, exist_ok=True)
+
+    meta_path = os.path.join(cache_dir, "__valid_download")
+    if valid_model_revision(meta_path, model_spec.model_revision):
+        return cache_dir
+
     for current_attempt in range(1, MAX_ATTEMPTS + 1):
         try:
             huggingface_hub.snapshot_download(
@@ -89,9 +95,28 @@ def cache(model_spec: ImageModelFamilyV1):
         raise RuntimeError(
             f"Failed to download model '{model_spec.model_name}' after {MAX_ATTEMPTS} attempts"
         )
+
+    with open(meta_path, "w") as f:
+        import json
+
+        desc = ImageModelDescription(model_spec)
+        json.dump(desc.to_dict(), f)
+
     return cache_dir
 
 
+def get_cache_status(
+    model_spec: ImageModelFamilyV1,
+) -> bool:
+    cache_dir = os.path.realpath(
+        os.path.join(XINFERENCE_CACHE_DIR, model_spec.model_name)
+    )
+    if not os.path.exists(cache_dir):
+        os.makedirs(cache_dir, exist_ok=True)
+    meta_path = os.path.join(cache_dir, "__valid_download")
+    return valid_model_revision(meta_path, model_spec.model_revision)
+
+
 def create_image_model_instance(
     model_uid: str, model_name: str, **kwargs
 ) -> Tuple[DiffusionModel, ImageModelDescription]:
diff --git a/xinference/model/image/tests/test_stable_diffusion.py b/xinference/model/image/tests/test_stable_diffusion.py
index 809cd82d3d..397474bfe8 100644
--- a/xinference/model/image/tests/test_stable_diffusion.py
+++ b/xinference/model/image/tests/test_stable_diffusion.py
@@ -15,6 +15,7 @@
 import io
 import logging
 import os.path
+import shutil
 from io import BytesIO
 
 import pytest
@@ -34,20 +35,25 @@
 
 
 def test_model():
-    model_path = cache(TEST_MODEL_SPEC)
-    model = DiffusionModel("mock", model_path)
-    # input is a string
-    input_text = "an apple"
-    model.load()
-    r = model.text_to_image(input_text, size="256*256")
-    assert len(r["data"]) == 1
-    assert os.path.exists(r["data"][0]["url"])
-    r = model.text_to_image(input_text, size="256*256", response_format="b64_json")
-    assert len(r["data"]) == 1
-    b64_json = r["data"][0]["b64_json"]
-    image_bytes = base64.decodebytes(b64_json)
-    img = Image.open(BytesIO(image_bytes))
-    assert img.size == (256, 256)
+    model_path = None
+    try:
+        model_path = cache(TEST_MODEL_SPEC)
+        model = DiffusionModel("mock", model_path)
+        # input is a string
+        input_text = "an apple"
+        model.load()
+        r = model.text_to_image(input_text, size="256*256")
+        assert len(r["data"]) == 1
+        assert os.path.exists(r["data"][0]["url"])
+        r = model.text_to_image(input_text, size="256*256", response_format="b64_json")
+        assert len(r["data"]) == 1
+        b64_json = r["data"][0]["b64_json"]
+        image_bytes = base64.decodebytes(b64_json)
+        img = Image.open(BytesIO(image_bytes))
+        assert img.size == (256, 256)
+    finally:
+        if model_path is not None:
+            shutil.rmtree(model_path)
 
 
 @pytest.mark.skip(reason="Stable diffusion controlnet requires too many GRAM.")
@@ -136,3 +142,16 @@ def test_restful_api_for_image_with_mlsd_controlnet(setup):
         num_inference_steps=20,
     )
     logger.info("test result %s", r)
+
+
+def test_get_cache_status():
+    from ..core import get_cache_status
+
+    model_path = None
+    try:
+        assert get_cache_status(TEST_MODEL_SPEC) is False
+        model_path = cache(TEST_MODEL_SPEC)
+        assert get_cache_status(TEST_MODEL_SPEC) is True
+    finally:
+        if model_path is not None:
+            shutil.rmtree(model_path)
diff --git a/xinference/model/llm/__init__.py b/xinference/model/llm/__init__.py
index abb44caaa8..6271c9bec0 100644
--- a/xinference/model/llm/__init__.py
+++ b/xinference/model/llm/__init__.py
@@ -26,6 +26,7 @@
     LLMSpecV1,
     PromptStyleV1,
     PytorchLLMSpecV1,
+    get_cache_status,
     get_user_defined_llm_families,
     match_llm,
     match_llm_cls,
diff --git a/xinference/model/llm/llm_family.py b/xinference/model/llm/llm_family.py
index d555755918..d4ec01f973 100644
--- a/xinference/model/llm/llm_family.py
+++ b/xinference/model/llm/llm_family.py
@@ -581,6 +581,36 @@ def cache_from_huggingface(
     return cache_dir
 
 
+def get_cache_status(
+    llm_family: LLMFamilyV1,
+    llm_spec: "LLMSpecV1",
+) -> Union[bool, List[bool]]:
+    cache_dir = _get_cache_dir(llm_family, llm_spec)
+    if llm_spec.model_format == "pytorch":
+        return _skip_download(
+            cache_dir,
+            llm_spec.model_format,
+            llm_spec.model_hub,
+            llm_spec.model_revision,
+            "none",
+        )
+    elif llm_spec.model_format in ["ggmlv3", "ggufv2"]:
+        ret = []
+        for q in llm_spec.quantizations:
+            ret.append(
+                _skip_download(
+                    cache_dir,
+                    llm_spec.model_format,
+                    llm_spec.model_hub,
+                    llm_spec.model_revision,
+                    q,
+                )
+            )
+        return ret
+    else:
+        raise ValueError(f"Unsupported model format: {llm_spec.model_format}")
+
+
 def _is_linux():
     return platform.system() == "Linux"
 
diff --git a/xinference/model/llm/tests/test_llm_family.py b/xinference/model/llm/tests/test_llm_family.py
index 107afe75ae..46f190b703 100644
--- a/xinference/model/llm/tests/test_llm_family.py
+++ b/xinference/model/llm/tests/test_llm_family.py
@@ -866,3 +866,75 @@ def test_skip_download_ggml():
     finally:
         os.remove(ms_meta_path)
         assert not os.path.exists(ms_meta_path)
+
+
+def test_get_cache_status_pytorch():
+    from ..llm_family import cache_from_huggingface, get_cache_status
+
+    spec = PytorchLLMSpecV1(
+        model_format="pytorch",
+        model_size_in_billions=1,
+        quantizations=["4-bit", "8-bit", "none"],
+        model_id="facebook/opt-125m",
+    )
+    family = LLMFamilyV1(
+        version=1,
+        context_length=2048,
+        model_type="LLM",
+        model_name="opt",
+        model_lang=["en"],
+        model_ability=["embed", "generate"],
+        model_specs=[spec],
+        prompt_style=None,
+    )
+
+    cache_status = get_cache_status(llm_family=family, llm_spec=spec)
+    assert not isinstance(cache_status, list)
+    assert not cache_status
+
+    cache_dir = cache_from_huggingface(family, spec, quantization=None)
+    cache_status = get_cache_status(llm_family=family, llm_spec=spec)
+    assert not isinstance(cache_status, list)
+    assert cache_status
+
+    assert os.path.exists(cache_dir)
+    assert os.path.exists(os.path.join(cache_dir, "README.md"))
+    assert os.path.islink(os.path.join(cache_dir, "README.md"))
+    shutil.rmtree(cache_dir)
+
+
+def test_get_cache_status_ggml():
+    from ..llm_family import cache_from_huggingface, get_cache_status
+
+    spec = GgmlLLMSpecV1(
+        model_format="ggmlv3",
+        model_size_in_billions=3,
+        model_id="TheBloke/orca_mini_3B-GGML",
+        quantizations=["q4_0", "q5_0"],
+        model_file_name_template="README.md",
+    )
+    family = LLMFamilyV1(
+        version=1,
+        context_length=2048,
+        model_type="LLM",
+        model_name="orca",
+        model_lang=["en"],
+        model_ability=["embed", "chat"],
+        model_specs=[spec],
+        prompt_style=None,
+    )
+
+    cache_status = get_cache_status(llm_family=family, llm_spec=spec)
+    assert isinstance(cache_status, list)
+    assert not any(cache_status)
+
+    cache_dir = cache_from_huggingface(family, spec, quantization="q4_0")
+    cache_status = get_cache_status(llm_family=family, llm_spec=spec)
+    assert isinstance(cache_status, list)
+    assert len(cache_status) == 2
+    assert cache_status[0] and not cache_status[1]
+
+    assert os.path.exists(cache_dir)
+    assert os.path.exists(os.path.join(cache_dir, "README.md"))
+    assert os.path.islink(os.path.join(cache_dir, "README.md"))
+    shutil.rmtree(cache_dir)
diff --git a/xinference/model/utils.py b/xinference/model/utils.py
index 5ab0425fd6..7f84136c89 100644
--- a/xinference/model/utils.py
+++ b/xinference/model/utils.py
@@ -118,7 +118,7 @@ def valid_model_revision(
             logger.debug("Legacy meta file detected.")
             return True
 
-        if "model_revision" in meta_data:  # embedding
+        if "model_revision" in meta_data:  # embedding, image
             real_revision = meta_data["model_revision"]
         elif "revision" in meta_data:  # llm
             real_revision = meta_data["revision"]
diff --git a/xinference/web/ui/src/scenes/launch_model/embeddingCard.js b/xinference/web/ui/src/scenes/launch_model/embeddingCard.js
index c42e00d6d2..20dfe33caa 100644
--- a/xinference/web/ui/src/scenes/launch_model/embeddingCard.js
+++ b/xinference/web/ui/src/scenes/launch_model/embeddingCard.js
@@ -3,10 +3,7 @@ import { v1 as uuidv1 } from "uuid";
 import { ApiContext } from "../../components/apiContext";
 import { Box, Chip } from "@mui/material";
 import { CircularProgress } from "@mui/material";
-import {
-  UndoOutlined,
-  RocketLaunchOutlined,
-} from "@mui/icons-material";
+import { UndoOutlined, RocketLaunchOutlined } from "@mui/icons-material";
 
 const CARD_HEIGHT = 270;
 const CARD_WIDTH = 270;
@@ -33,7 +30,7 @@ const EmbeddingCard = ({ url, modelData }) => {
     const modelDataWithID = {
       model_uid: uuid,
       model_name: modelData.model_name,
-      model_type: "embedding"
+      model_type: "embedding",
     };
 
     // First fetch request to initiate the model
@@ -175,8 +172,8 @@ const EmbeddingCard = ({ url, modelData }) => {
       fontSize: "0.8em",
     },
     langRow: {
-      margin: "2px 5px 40px 5px"
-    }
+      margin: "2px 5px 40px 5px",
+    },
   };
 
   // Set two different states based on mouse hover
@@ -199,30 +196,29 @@ const EmbeddingCard = ({ url, modelData }) => {
             {(() => {
               if (modelData.language.includes("en")) {
                 return (
-                  <Chip label="EN" variant="outlined" size="small" sx={{ marginRight: "10px" }} />
+                  <Chip
+                    label="EN"
+                    variant="outlined"
+                    size="small"
+                    sx={{ marginRight: "10px" }}
+                  />
                 );
               }
             })()}
             {(() => {
               if (modelData.language.includes("zh")) {
-                return (
-                  <Chip label="ZH" variant="outlined" size="small" />
-                );
+                return <Chip label="ZH" variant="outlined" size="small" />;
               }
             })()}
           </div>
         </div>
         <div style={styles.iconRow}>
           <div style={styles.iconItem}>
-            <span style={styles.boldIconText}>
-              {modelData.dimensions}
-            </span>
+            <span style={styles.boldIconText}>{modelData.dimensions}</span>
             <small style={styles.smallText}>dimensions</small>
           </div>
           <div style={styles.iconItem}>
-            <span style={styles.boldIconText}>
-              {modelData.max_tokens}
-            </span>
+            <span style={styles.boldIconText}>{modelData.max_tokens}</span>
             <small style={styles.smallText}>max tokens</small>
           </div>
         </div>
@@ -248,13 +244,7 @@ const EmbeddingCard = ({ url, modelData }) => {
             title="Launch Embedding"
             style={styles.buttonContainer}
             onClick={() => launchModel(url, modelData)}
-            disabled={
-              isCallingApi ||
-              isUpdatingModel ||
-              !(
-                modelData
-              )
-            }
+            disabled={isCallingApi || isUpdatingModel || !modelData}
           >
             {(() => {
               if (isCallingApi || isUpdatingModel) {
@@ -270,11 +260,7 @@ const EmbeddingCard = ({ url, modelData }) => {
                     />
                   </Box>
                 );
-              } else if (
-                !(
-                  modelData
-                )
-              ) {
+              } else if (!modelData) {
                 return (
                   <Box
                     style={{ ...styles.buttonItem, backgroundColor: "#f2f2f2" }}
diff --git a/xinference/web/ui/src/scenes/launch_model/index.js b/xinference/web/ui/src/scenes/launch_model/index.js
index 111576d481..69d5979f01 100644
--- a/xinference/web/ui/src/scenes/launch_model/index.js
+++ b/xinference/web/ui/src/scenes/launch_model/index.js
@@ -1,13 +1,9 @@
-import React, { } from "react";
+import React from "react";
 import Title from "../../components/Title";
 import LaunchLLM from "./launchLLM";
 import LaunchEmbedding from "./launchEmbedding";
-import {
-  Box,
-  Tab
-} from "@mui/material";
-import { TabContext, TabList, TabPanel } from "@mui/lab"
-
+import { Box, Tab } from "@mui/material";
+import { TabContext, TabList, TabPanel } from "@mui/lab";
 
 const LaunchModel = () => {
   const [value, setValue] = React.useState("1");
@@ -26,14 +22,14 @@ const LaunchModel = () => {
             <Tab label="Embedding Models" value="2" />
           </TabList>
         </Box>
-        <TabPanel value="1" sx={{padding: 0}}>
+        <TabPanel value="1" sx={{ padding: 0 }}>
           <LaunchLLM />
         </TabPanel>
-        <TabPanel value="2" sx={{padding: 0}}>
+        <TabPanel value="2" sx={{ padding: 0 }}>
           <LaunchEmbedding />
         </TabPanel>
       </TabContext>
-    </Box >
+    </Box>
   );
 };
 
diff --git a/xinference/web/ui/src/scenes/launch_model/launchEmbedding.js b/xinference/web/ui/src/scenes/launch_model/launchEmbedding.js
index 69756c7d6a..55aa23e2c0 100644
--- a/xinference/web/ui/src/scenes/launch_model/launchEmbedding.js
+++ b/xinference/web/ui/src/scenes/launch_model/launchEmbedding.js
@@ -21,9 +21,7 @@ const LaunchEmbedding = () => {
     const modelName = registration.model_name
       ? registration.model_name.toLowerCase()
       : "";
-    if (
-      !modelName.includes(searchTerm.toLowerCase())
-    ) {
+    if (!modelName.includes(searchTerm.toLowerCase())) {
       return false;
     }
     return true;
@@ -35,9 +33,12 @@ const LaunchEmbedding = () => {
     try {
       setIsCallingApi(true);
 
-      const response = await fetch(`${endPoint}/v1/model_registrations/embedding`, {
-        method: "GET",
-      });
+      const response = await fetch(
+        `${endPoint}/v1/model_registrations/embedding`,
+        {
+          method: "GET",
+        },
+      );
 
       const registrations = await response.json();
       const newRegistrationData = await Promise.all(
@@ -78,11 +79,14 @@ const LaunchEmbedding = () => {
 
   return (
     <Box m="20px">
-      <div style={{ display: "grid", gridTemplateColumns: "1fr", margin: "30px 2rem" }}>
-        <FormControl
-          variant="outlined"
-          margin="normal"
-        >
+      <div
+        style={{
+          display: "grid",
+          gridTemplateColumns: "1fr",
+          margin: "30px 2rem",
+        }}
+      >
+        <FormControl variant="outlined" margin="normal">
           <TextField
             id="search"
             type="search"
diff --git a/xinference/web/ui/src/scenes/launch_model/launchLLM.js b/xinference/web/ui/src/scenes/launch_model/launchLLM.js
index 07c967d5d1..281cf57098 100644
--- a/xinference/web/ui/src/scenes/launch_model/launchLLM.js
+++ b/xinference/web/ui/src/scenes/launch_model/launchLLM.js
@@ -8,7 +8,8 @@ import {
   Select,
   MenuItem,
   InputLabel,
-  Tabs, Tab
+  Tabs,
+  Tab,
 } from "@mui/material";
 import { ApiContext } from "../../components/apiContext";
 
@@ -65,30 +66,16 @@ const LaunchLLM = () => {
     try {
       setIsCallingApi(true);
 
-      const response = await fetch(`${endPoint}/v1/model_registrations/LLM`, {
-        method: "GET",
-      });
+      const response = await fetch(
+        `${endPoint}/v1/model_registrations/LLM?detailed=true`,
+        {
+          method: "GET",
+        },
+      );
 
       const registrations = await response.json();
 
-      const newRegistrationData = await Promise.all(
-        registrations.map(async (registration) => {
-          const desc = await fetch(
-            `${endPoint}/v1/model_registrations/LLM/${registration.model_name}`,
-            {
-              method: "GET",
-            },
-          );
-
-          return {
-            ...(await desc.json()),
-            is_builtin: registration.is_builtin,
-          };
-        }),
-      );
-
-      setRegistrationData(newRegistrationData);
-      console.log(newRegistrationData);
+      setRegistrationData(registrations);
     } catch (error) {
       console.error("Error:", error);
     } finally {
@@ -111,17 +98,21 @@ const LaunchLLM = () => {
   function a11yProps(index) {
     return {
       id: `simple-tab-${index}`,
-      'aria-controls': `simple-tabpanel-${index}`,
+      "aria-controls": `simple-tabpanel-${index}`,
     };
   }
 
   return (
     <Box m="20px">
-      <div style={{ display: "grid", gridTemplateColumns: "150px 1fr", columnGap: "20px", margin: "30px 2rem" }}>
-        <FormControl
-          variant="outlined"
-          margin="normal"
-        >
+      <div
+        style={{
+          display: "grid",
+          gridTemplateColumns: "150px 1fr",
+          columnGap: "20px",
+          margin: "30px 2rem",
+        }}
+      >
+        <FormControl variant="outlined" margin="normal">
           <InputLabel id="ability-select-label">Model Ability</InputLabel>
           <Select
             id="ability"
@@ -137,10 +128,7 @@ const LaunchLLM = () => {
             <MenuItem value="chat">chat</MenuItem>
           </Select>
         </FormControl>
-        <FormControl
-          variant="outlined"
-          margin="normal"
-        >
+        <FormControl variant="outlined" margin="normal">
           <TextField
             id="search"
             type="search"
@@ -158,7 +146,7 @@ const LaunchLLM = () => {
             <ModelCard url={endPoint} modelData={filteredRegistration} />
           ))}
       </div>
-    </Box >
+    </Box>
   );
 };
 
diff --git a/xinference/web/ui/src/scenes/launch_model/modelCard.js b/xinference/web/ui/src/scenes/launch_model/modelCard.js
index b440006a92..560ea266de 100644
--- a/xinference/web/ui/src/scenes/launch_model/modelCard.js
+++ b/xinference/web/ui/src/scenes/launch_model/modelCard.js
@@ -1,7 +1,14 @@
 import React, { useState, useContext, useEffect } from "react";
 import { v1 as uuidv1 } from "uuid";
 import { ApiContext } from "../../components/apiContext";
-import { FormControl, InputLabel, Select, MenuItem, Box, Chip } from "@mui/material";
+import {
+  FormControl,
+  InputLabel,
+  Select,
+  MenuItem,
+  Box,
+  Chip,
+} from "@mui/material";
 import { CircularProgress } from "@mui/material";
 import {
   ChatOutlined,
@@ -242,9 +249,9 @@ const ModelCard = ({ url, modelData }) => {
     smallText: {
       fontSize: "0.8em",
     },
-    langRow: {
-      margin: "2px 5px"
-    }
+    tagRow: {
+      margin: "2px 5px",
+    },
   };
 
   // Set two different states based on mouse hover
@@ -262,19 +269,40 @@ const ModelCard = ({ url, modelData }) => {
       {/* First state: show description page */}
       <Box style={styles.descriptionCard}>
         <h2 style={styles.h2}>{modelData.model_name}</h2>
-        <div style={styles.langRow}>
-          {(()=> {
-            if ( modelData.model_lang.includes("en")) {
+        <div style={styles.tagRow}>
+          {(() => {
+            if (modelData.model_lang.includes("en")) {
+              return <Chip label="EN" variant="outlined" size="small" />;
+            }
+          })()}
+          {(() => {
+            if (modelData.model_lang.includes("zh")) {
               return (
-                <Chip label="EN" variant="outlined" size="small" sx={{marginRight: "10px"}}/>
+                <Chip
+                  label="ZH"
+                  variant="outlined"
+                  size="small"
+                  sx={{ marginLeft: "10px" }}
+                />
               );
             }
           })()}
-          {(()=> {
-            if ( modelData.model_lang.includes("zh")) {
-              return (
-                <Chip label="ZH" variant="outlined" size="small"/>
+          {(() => {
+            if (
+              modelData.model_specs.some((spec) =>
+                spec.model_format === "pytorch"
+                  ? spec.cache_status
+                  : spec.cache_status.some((cs) => cs),
               )
+            ) {
+              return (
+                <Chip
+                  label="Cached"
+                  variant="outlined"
+                  size="small"
+                  sx={{ marginLeft: "10px" }}
+                />
+              );
             }
           })()}
         </div>
@@ -338,11 +366,22 @@ const ModelCard = ({ url, modelData }) => {
               onChange={(e) => setModelFormat(e.target.value)}
               label="Model Format"
             >
-              {formatOptions.map((format) => (
-                <MenuItem key={format} value={format}>
-                  {format}
-                </MenuItem>
-              ))}
+              {formatOptions.map((format) => {
+                const specs = modelData.model_specs.filter(
+                  (spec) => spec.model_format === format,
+                );
+                const cached =
+                  format === "pytorch"
+                    ? specs.some((spec) => spec.cache_status)
+                    : specs.some((spec) => spec.cache_status.some((cs) => cs));
+                const displayedFormat = cached ? format + " (cached)" : format;
+
+                return (
+                  <MenuItem key={format} value={format}>
+                    {displayedFormat}
+                  </MenuItem>
+                );
+              })}
             </Select>
           </FormControl>
           <FormControl
@@ -358,11 +397,22 @@ const ModelCard = ({ url, modelData }) => {
               onChange={(e) => setModelSize(e.target.value)}
               label="Model Size"
             >
-              {sizeOptions.map((size) => (
-                <MenuItem key={size} value={size}>
-                  {size}
-                </MenuItem>
-              ))}
+              {sizeOptions.map((size) => {
+                const specs = modelData.model_specs
+                  .filter((spec) => spec.model_format === modelFormat)
+                  .filter((spec) => spec.model_size_in_billions === size);
+                const cached =
+                  modelFormat === "pytorch"
+                    ? specs.some((spec) => spec.cache_status)
+                    : specs.some((spec) => spec.cache_status.some((cs) => cs));
+                const displayedSize = cached ? size + " (cached)" : size;
+
+                return (
+                  <MenuItem key={size} value={size}>
+                    {displayedSize}
+                  </MenuItem>
+                );
+              })}
             </Select>
           </FormControl>
           {(modelData.is_builtin || modelFormat === "pytorch") && (
@@ -379,11 +429,25 @@ const ModelCard = ({ url, modelData }) => {
                 onChange={(e) => setQuantization(e.target.value)}
                 label="Quantization"
               >
-                {quantizationOptions.map((quant) => (
-                  <MenuItem key={quant} value={quant}>
-                    {quant}
-                  </MenuItem>
-                ))}
+                {quantizationOptions.map((quant, index) => {
+                  const specs = modelData.model_specs
+                    .filter((spec) => spec.model_format === modelFormat)
+                    .filter(
+                      (spec) => spec.model_size_in_billions === modelSize,
+                    );
+
+                  const cached =
+                    modelFormat === "pytorch"
+                      ? specs[0].cache_status
+                      : specs[0].cache_status[index];
+                  const displayedQuant = cached ? quant + " (cached)" : quant;
+
+                  return (
+                    <MenuItem key={quant} value={quant}>
+                      {displayedQuant}
+                    </MenuItem>
+                  );
+                })}
               </Select>
             </FormControl>
           )}


Code review comments:
diff_hunk: @@ -102,46 +106,91 @@ def get_status(self) -> Dict:
             "workers": self._worker_status,
         }
 
+    @staticmethod
+    def _to_llm_reg(llm_family: "LLMFamilyV1", is_builtin: bool) -> Dict[str, Any]:
path: xinference/core/supervisor.py
text: @user1:
Use lru_cache to wrap this function.

@author:
Using an LRU cache might not be suitable for this scenario because the return value is subject to change.

diff_hunk: @@ -102,46 +106,91 @@ def get_status(self) -> Dict:
             "workers": self._worker_status,
         }
 
+    @staticmethod
+    def _to_llm_reg(llm_family: "LLMFamilyV1", is_builtin: bool) -> Dict[str, Any]:
+        from ..model.llm import get_cache_status
+
+        specs = []
+        for spec in llm_family.model_specs:
+            cache_status = get_cache_status(llm_family, spec)
+            specs.append({**spec.dict(), "cache_status": cache_status})
+
+        return {**llm_family.dict(), "is_builtin": is_builtin, "model_specs": specs}
+
+    @staticmethod
+    def _to_embedding_model_reg(
path: xinference/core/supervisor.py
text: @user1:
Ditto.

@author:
Using an LRU cache might not be suitable for this scenario because the return value is subject to change.

diff_hunk: @@ -102,46 +106,91 @@ def get_status(self) -> Dict:
             "workers": self._worker_status,
         }
 
+    @staticmethod
+    def _to_llm_reg(llm_family: "LLMFamilyV1", is_builtin: bool) -> Dict[str, Any]:
+        from ..model.llm import get_cache_status
+
+        specs = []
+        for spec in llm_family.model_specs:
+            cache_status = get_cache_status(llm_family, spec)
+            specs.append({**spec.dict(), "cache_status": cache_status})
+
+        return {**llm_family.dict(), "is_builtin": is_builtin, "model_specs": specs}
+
+    @staticmethod
+    def _to_embedding_model_reg(
+        model_spec: "EmbeddingModelSpec", is_builtin: bool
+    ) -> Dict[str, Any]:
+        from ..model.embedding import get_cache_status
+
+        cache_status = get_cache_status(model_spec)
+        return {
+            **model_spec.dict(),
+            "cache_status": cache_status,
+            "is_builtin": is_builtin,
+        }
+
+    @staticmethod
+    def _to_image_model_reg(
path: xinference/core/supervisor.py
text: @user1:
Ditto.

@author:
Using an LRU cache might not be suitable for this scenario because the return value is subject to change.