feat(one-model): one model at a time

vllm does not support multi model, we can define one by one, but since I use T4 which is limited, better use 1 model only

main.py

@@ -66,20 +66,20 @@ engine_llama_3_2: LLM = LLM(
     dtype='half',                  # Use 'half' for T4
 )
 
-# ValueError: max_num_batched_tokens (512) is smaller than max_model_len (32768).
-# This effectively limits the maximum sequence length to max_num_batched_tokens and makes vLLM reject longer sequences.
-# Please increase max_num_batched_tokens or decrease max_model_len.
-engine_sailor_chat: LLM = LLM(
-    model='sail/Sailor-4B-Chat',
-    revision="89a866a7041e6ec023dd462adeca8e28dd53c83e",
-    max_num_batched_tokens=32768,    # Reduced for T4
-    max_num_seqs=16,                 # Reduced for T4
-    gpu_memory_utilization=0.85,     # Slightly increased, adjust if needed
-    tensor_parallel_size=1,
-    max_model_len=32768,
-    enforce_eager=True,              # Disable CUDA graph
-    dtype='half',                    # Use 'half' for T4
-)
+# # ValueError: max_num_batched_tokens (512) is smaller than max_model_len (32768).
+# # This effectively limits the maximum sequence length to max_num_batched_tokens and makes vLLM reject longer sequences.
+# # Please increase max_num_batched_tokens or decrease max_model_len.
+# engine_sailor_chat: LLM = LLM(
+#     model='sail/Sailor-4B-Chat',
+#     revision="89a866a7041e6ec023dd462adeca8e28dd53c83e",
+#     max_num_batched_tokens=32768,    # Reduced for T4
+#     max_num_seqs=16,                 # Reduced for T4
+#     gpu_memory_utilization=0.85,     # Slightly increased, adjust if needed
+#     tensor_parallel_size=1,
+#     max_model_len=32768,
+#     enforce_eager=True,              # Disable CUDA graph
+#     dtype='half',                    # Use 'half' for T4
+# )
 
 
 @app.get("/")
@@ -104,11 +104,6 @@ def greet_json():
                 "revision": "0cb88a4f764b7a12671c53f0838cd831a0843b95",
                 "max_model_len": engine_llama_3_2.llm_engine.model_config.max_model_len,
             },
-            {
-                "name": "sail/Sailor-4B-Chat",
-                "revision": "89a866a7041e6ec023dd462adeca8e28dd53c83e",
-                "max_model_len": engine_sailor_chat.llm_engine.model_config.max_model_len,
-            },
         ]
     }
 
@@ -146,23 +141,23 @@ def generate_text(request: GenerationRequest) -> list[RequestOutput] | dict[str,
         }
 
 
-@app.post("/generate-sailor-chat")
-def generate_text(request: GenerationRequest) -> list[RequestOutput] | dict[str, str]:
-    try:
-        sampling_params: SamplingParams = SamplingParams(
-            temperature=request.temperature,
-            max_tokens=request.max_tokens,
-            logit_bias=request.logit_bias,
-        )
-
-        # Generate text
-        return engine_sailor_chat.generate(
-            prompts=request.prompt,
-            sampling_params=sampling_params
-        )
-
-    except Exception as e:
-        return {
-            "error": str(e)
-        }
-
+# @app.post("/generate-sailor-chat")
+# def generate_text(request: GenerationRequest) -> list[RequestOutput] | dict[str, str]:
+#     try:
+#         sampling_params: SamplingParams = SamplingParams(
+#             temperature=request.temperature,
+#             max_tokens=request.max_tokens,
+#             logit_bias=request.logit_bias,
+#         )
+#
+#         # Generate text
+#         return engine_sailor_chat.generate(
+#             prompts=request.prompt,
+#             sampling_params=sampling_params
+#         )
+#
+#     except Exception as e:
+#         return {
+#             "error": str(e)
+#         }
+#