Repository: instructlab/instructlab

Issue:
Give a more friendly error if you try to use a gguf model with vllm
I had an existing config and switched my serve backend setting to `vllm`. This was using a quantized version of Mixtral in gguf format.

The error I got didn't make it obvious at all that a gguf wasn't supported,.

```
Traceback (most recent call last):
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/transformers/configuration_utils.py", line 722, in _get_config_dict
    config_dict = cls._dict_from_json_file(resolved_config_file)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/transformers/configuration_utils.py", line 825, in _dict_from_json_file
    text = reader.read()
           ^^^^^^^^^^^^^
  File "<frozen codecs>", line 322, in decode
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe3 in position 8: invalid continuation byte

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 196, in <module>
    engine = AsyncLLMEngine.from_engine_args(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/vllm/engine/async_llm_engine.py", line 371, in from_engine_args
    engine_config = engine_args.create_engine_config()
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/vllm/engine/arg_utils.py", line 630, in create_engine_config
    model_config = ModelConfig(
                   ^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/vllm/config.py", line 137, in __init__
    self.hf_config = get_config(self.model, trust_remote_code, revision,
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/vllm/transformers_utils/config.py", line 33, in get_config
    config = AutoConfig.from_pretrained(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/transformers/models/auto/configuration_auto.py", line 965, in from_pretrained
    config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
                                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/transformers/configuration_utils.py", line 632, in get_config_dict
    config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/ec2-user/venv/lib64/python3.11/site-packages/transformers/configuration_utils.py", line 726, in _get_config_dict
    raise EnvironmentError(
OSError: It looks like the config file at 'models/mixtral-8x7b-v0.1.Q4_K_M.gguf' is not a valid JSON file.
```


Discussion:
Related to https://github.com/instructlab/instructlab/issues/1555
This error happens inside the subprocess and sdtout is never treated. This is a bug in the validation anyways. Will fix that.
Just need a few PRs to merge before I can do this one. Like https://github.com/instructlab/instructlab/pull/1576 and https://github.com/instructlab/instructlab/pull/1561
https://github.com/instructlab/instructlab/pull/1576 will fix this.



Patch:
diff --git a/src/instructlab/model/backends/backends.py b/src/instructlab/model/backends/backends.py
index 82454ea52a..2e583d5e9a 100644
--- a/src/instructlab/model/backends/backends.py
+++ b/src/instructlab/model/backends/backends.py
@@ -113,20 +113,35 @@ def is_model_gguf(model_path: pathlib.Path) -> bool:
         return first_four_bytes_int == GGUF_MAGIC
 
 
-def determine_backend(model_path: pathlib.Path) -> str:
+def determine_backend(model_path: pathlib.Path) -> Tuple[str, str]:
     """
     Determine the backend to use based on the model file properties.
     Args:
         model_path (Path): The path to the model file.
     Returns:
-        str: The backend to use.
+        Tuple[str, str]: A tuple containing two strings:
+                        - The backend to use.
+                        - The reason why the backend was selected.
     """
-    # Check if the model is a GGUF file
 
-    # If the model is a directory, it's a VLLM model - it's kinda weak, but it's a start
-    if model_path.is_dir() and sys.platform == "linux":
-        return VLLM
+    if model_path.is_dir():
+        if sys.platform != "linux":
+            raise ValueError(
+                "Model is a directory containing huggingface safetensors files but the system is not Linux. "
+                "Using a directory with safetensors file will activate the vLLM serving backend, vLLM is only supported on Linux. "
+                "If you want to run the model on a different system (e.g. macOS), please use a GGUF file."
+            )
+        # If the model is a directory, it's a VLLM model - it's kinda weak, but it's a start
+        if sys.platform == "linux":
+            logger.debug(
+                f"Model is a directory and system is Linux, using {VLLM} backend."
+            )
+            return (
+                VLLM,
+                "model path is a directory containing huggingface safetensors files and running on Linux.",
+            )
 
+    # Check if the model is a GGUF file
     try:
         is_gguf = is_model_gguf(model_path)
     except Exception as e:
@@ -135,9 +150,14 @@ def determine_backend(model_path: pathlib.Path) -> str:
         ) from e
 
     if is_gguf:
-        return LLAMA_CPP
+        logger.debug(f"Model is a GGUF file, using {LLAMA_CPP} backend.")
+        return LLAMA_CPP, "model is a GGUF file."
 
-    raise ValueError(f"The model file {model_path} is not a GGUF format. Unsupported.")
+    raise ValueError(
+        f"The model file {model_path} is not a GGUF format nor a directory containing huggingface safetensors files. Cannot determine which backend to use. \n"
+        f"Please use a GGUF file for {LLAMA_CPP} or a directory containing huggingface safetensors files for {VLLM}. \n"
+        "Note that vLLM is only supported on Linux."
+    )
 
 
 def get(model_path: pathlib.Path, backend: str | None) -> str:
@@ -151,7 +171,12 @@ def get(model_path: pathlib.Path, backend: str | None) -> str:
     """
     # Check if the model is a GGUF file
     logger.debug(f"Auto-detecting backend for model {model_path}")
-    auto_detected_backend = determine_backend(model_path)
+    try:
+        auto_detected_backend, auto_detected_backend_reason = determine_backend(
+            model_path
+        )
+    except ValueError as e:
+        raise ValueError(f"Cannot determine which backend to use: {e}") from e
 
     logger.debug(f"Auto-detected backend: {auto_detected_backend}")
     # When the backend is not set using the --backend flag, determine the backend automatically
@@ -165,12 +190,13 @@ def get(model_path: pathlib.Path, backend: str | None) -> str:
     # If the backend was set using the --backend flag, validate it.
     else:
         logger.debug(f"Validating '{backend}' backend")
-        # TODO: keep this code logic and implement a `--force` flag to override the auto-detected backend
         # If the backend was set explicitly, but we detected the model should use a different backend, raise an error
-        # if backend != auto_detected_backend:
-        #     raise ValueError(
-        #         f"The backend '{backend}' was set explicitly, but the model was detected as '{auto_detected_backend}'."
-        #     )
+        if backend != auto_detected_backend:
+            logger.warning(
+                f"The serving backend '{backend}' was configured explicitly, but the provided model is not compatible with it. "
+                f"The model was detected as '{auto_detected_backend}, reason: {auto_detected_backend_reason}'.\n"
+                "The backend startup sequence will continue with the configured backend but might fail."
+            )
 
     return backend
 


Test patch:
diff --git a/tests/test_backends.py b/tests/test_backends.py
index cd91cdff2b..2c78f5f341 100644
--- a/tests/test_backends.py
+++ b/tests/test_backends.py
@@ -1,4 +1,6 @@
 # Standard
+from unittest.mock import patch
+import pathlib
 import socket
 
 # Third Party
@@ -7,15 +9,6 @@
 # First Party
 from instructlab.model.backends import backends
 
-supported_backends = ["llama-cpp", "vllm"]  # Example supported backends
-
-
-@pytest.fixture
-def mock_supported_backends(monkeypatch):
-    monkeypatch.setattr(
-        "instructlab.model.backends.backends.SUPPORTED_BACKENDS", supported_backends
-    )
-
 
 def test_free_port():
     host = "localhost"
@@ -23,3 +16,70 @@ def test_free_port():
     # check that port is bindable
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         s.bind((host, port))
+
+
+# the test fails because the model_path is not a valid GGUF file
+def test_get_backend_auto_detection_fail_not_gguf(tmp_path: pathlib.Path):
+    tmp_gguf = tmp_path / "test.gguf"
+    # Write a known invalid header
+    invalid_header = (
+        b"\x00\x00\x00\x00"  # Use a header that will definitely not match "GGUF_MAGIC"
+    )
+    tmp_gguf.write_bytes(
+        invalid_header + bytes([0] * 4093)
+    )  # Fill the rest of the file with zeros
+    with pytest.raises(ValueError) as exc_info:
+        backends.get(tmp_gguf, None)
+    assert "is not a GGUF format" in str(exc_info.value)
+
+
+# this test succeeds because the model_path is a valid GGUF file (is_model_gguf mocked to returns True)
+@patch("instructlab.model.backends.backends.is_model_gguf", return_value=True)
+def test_get_backend_auto_detection_success_gguf(
+    m_is_model_gguf, tmp_path: pathlib.Path
+):
+    tmp_gguf = tmp_path / "test.gguf"
+    backend = backends.get(tmp_gguf, None)
+    assert backend == "llama-cpp"
+    m_is_model_gguf.assert_called_once_with(tmp_gguf)
+
+
+# this test fails because the model_path is a directory but the platform is not linux
+@patch("sys.platform", "darwin")
+def test_get_backend_auto_detection_failure_vllm_dir_but_not_linux(
+    tmp_path: pathlib.Path,
+):
+    with pytest.raises(ValueError) as exc_info:
+        backends.get(tmp_path, None)
+    assert "Cannot determine which backend to use" in str(exc_info.value)
+
+
+# this test succeeds because the model_path is a directory and the platform is linux so vllm is
+# picked
+@patch("sys.platform", "linux")
+def test_get_backend_auto_detection_success_vllm_dir(tmp_path: pathlib.Path):
+    backend = backends.get(tmp_path, None)
+    assert backend == "vllm"
+
+
+# this test fails because the OS is darwin and a directory is passed, only works on Linux
+@patch("sys.platform", "darwin")
+def test_get_backend_auto_detection_failed_vllm_dir_darwin(tmp_path: pathlib.Path):
+    with pytest.raises(ValueError) as exc_info:
+        backends.get(tmp_path, None)
+    assert (
+        "Model is a directory containing huggingface safetensors files but the system is not Linux."
+        in str(exc_info.value)
+    )
+
+
+# this test succeeds even if the auto-detection picked a different backend, we continue with what
+# the user requested
+@patch(
+    "instructlab.model.backends.backends.determine_backend",
+    return_value=("vllm", "reason for selection"),
+)
+def test_get_forced_backend_fails_autodetection(m_determine_backend):
+    backend = backends.get("", "llama-cpp")
+    assert backend == "llama-cpp"
+    m_determine_backend.assert_called_once()