Upload 2 files

llmdolphin.py

@@ -219,7 +219,7 @@ def get_raw_prompt(msg: str):
 
 
 @torch.inference_mode()
-@spaces.GPU(duration=60)
+@spaces.GPU(duration=59)
 def dolphin_respond(
     message: str,
     history: list[tuple[str, str]],
@@ -234,13 +234,15 @@ def dolphin_respond(
     progress=gr.Progress(track_tqdm=True),
 ):
     try:
+        model_path = Path(f"{llm_models_dir}/{model}")
+        if not model_path.exists(): raise gr.Error(f"Model file not found: {str(model_path)}")
         progress(0, desc="Processing...")
         override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
         llm = Llama(
-            model_path=str(Path(f"{llm_models_dir}/{model}")),
+            model_path=str(model_path),
             flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
@@ -294,8 +296,8 @@ def dolphin_respond(
             yield [(outputs, None)]
     except Exception as e:
         print(e)
-        gr.Warning(f"Error: {e}")
-        yield [("", None)]
+        raise gr.Error(f"Error: {e}")
+        #yield [("", None)]
 
 
 def dolphin_parse(
@@ -320,7 +322,7 @@ def dolphin_parse(
 
 
 @torch.inference_mode()
-@spaces.GPU(duration=60)
+@spaces.GPU(duration=59)
 def dolphin_respond_auto(
     message: str,
     history: list[tuple[str, str]],
@@ -335,6 +337,7 @@ def dolphin_respond_auto(
     progress=gr.Progress(track_tqdm=True),
 ):
     try:
+        model_path = Path(f"{llm_models_dir}/{model}")
         #if not is_japanese(message): return [(None, None)]
         progress(0, desc="Processing...")
 
@@ -343,7 +346,7 @@ def dolphin_respond_auto(
         else: chat_template = llm_models[model][1]
 
         llm = Llama(
-            model_path=str(Path(f"{llm_models_dir}/{model}")),
+            model_path=str(model_path),
             flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
@@ -429,26 +432,29 @@ cv2.setNumThreads(1)
 
 
 @torch.inference_mode()
-@spaces.GPU()
+@spaces.GPU(duration=59)
 def respond_playground(
-    message,
+    message: str,
     history: list[tuple[str, str]],
-    model,
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-    top_k,
-    repeat_penalty,
-    state,
+    model: str = default_llm_model_filename,
+    system_message: str = get_dolphin_sysprompt(),
+    max_tokens: int = 1024,
+    temperature: float = 0.7,
+    top_p: float = 0.95,
+    top_k: int = 40,
+    repeat_penalty: float = 1.1,
+    state: dict = {},
+    progress=gr.Progress(track_tqdm=True),
 ):
     try:
+        model_path = Path(f"{llm_models_dir}/{model}")
+        if not model_path.exists(): raise gr.Error(f"Model file not found: {str(model_path)}")
         override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format: chat_template = override_llm_format
         else: chat_template = llm_models[model][1]
 
         llm = Llama(
-            model_path=str(Path(f"{llm_models_dir}/{model}")),
+            model_path=str(model_path),
             flash_attn=True,
             n_gpu_layers=81, # 81
             n_batch=1024,
@@ -496,4 +502,5 @@ def respond_playground(
             yield outputs
     except Exception as e:
         print(e)
-        yield ""
+        raise gr.Error(f"Error: {e}")
+        #yield ""

llmenv.py

@@ -67,6 +67,7 @@ llm_models = {
     "MISCHIEVOUS-12B-Mix_0.1v.Q4_K_M.gguf": ["mradermacher/MISCHIEVOUS-12B-Mix_0.1v-GGUF", MessagesFormatterType.MISTRAL],
     "Canidori-12B-v1.i1-Q4_K_M.gguf": ["mradermacher/Canidori-12B-v1-i1-GGUF", MessagesFormatterType.MISTRAL],
     "MT2-Gen4-MM-gemma-2-Rv0.4MTM-9B.Q4_K_M.gguf": ["mradermacher/MT2-Gen4-MM-gemma-2-Rv0.4MTM-9B-GGUF", MessagesFormatterType.ALPACA],
+    "Flammen-Trismegistus-7B.i1-Q5_K_M.gguf": ["mradermacher/Flammen-Trismegistus-7B-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Trinas_Nectar-8B-model_stock.i1-Q4_K_M.gguf": ["mradermacher/Trinas_Nectar-8B-model_stock-i1-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_Magnum_V0.2.Q4_K_M.gguf": ["mradermacher/ChatWaifu_Magnum_V0.2-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_12B_v2.0.Q5_K_M.gguf": ["mradermacher/ChatWaifu_12B_v2.0-GGUF", MessagesFormatterType.MISTRAL],
@@ -79,6 +80,79 @@ llm_models = {
     #"": ["", MessagesFormatterType.OPEN_CHAT],
     #"": ["", MessagesFormatterType.CHATML],
     #"": ["", MessagesFormatterType.PHI_3],
+    "WONMSeverusDevilv2-TIES.i1-Q5_K_M.gguf": ["mradermacher/WONMSeverusDevilv2-TIES-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "VirtuosoSmall-InstructModelStock.i1-Q4_K_M.gguf": ["mradermacher/VirtuosoSmall-InstructModelStock-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "VICIOUS_MESH-12B-GAMMA.Q4_K_M.gguf": ["mradermacher/VICIOUS_MESH-12B-GAMMA-GGUF", MessagesFormatterType.MISTRAL],
+    "VICIOUS_MESH-12B-BETA.Q4_K_M.gguf": ["mradermacher/VICIOUS_MESH-12B-BETA-GGUF", MessagesFormatterType.MISTRAL],
+    "TQ2.5-14B-Aletheia-v1.i1-Q4_K_M.gguf": ["mradermacher/TQ2.5-14B-Aletheia-v1-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Tiger-7B-v0.1.i1-Q5_K_M.gguf": ["mradermacher/Tiger-7B-v0.1-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "TaxoLlama3.1-8b-instruct.i1-Q5_K_M.gguf": ["mradermacher/TaxoLlama3.1-8b-instruct-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "RandomMergeNoNormWEIGHTED-7B-MODELSTOCK.i1-Q5_K_M.gguf": ["mradermacher/RandomMergeNoNormWEIGHTED-7B-MODELSTOCK-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "RandomMergeNoNormWEIGHTED-7B-DARETIES.i1-Q5_K_M.gguf": ["mradermacher/RandomMergeNoNormWEIGHTED-7B-DARETIES-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "RandomMergeNoNorm-7B-DARETIES.i1-Q5_K_M.gguf": ["mradermacher/RandomMergeNoNorm-7B-DARETIES-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "R_Marco_polo.Q5_K_M.gguf": ["mradermacher/R_Marco_polo-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwenvergence-14B-v3-Reason.i1-Q4_K_M.gguf": ["mradermacher/Qwenvergence-14B-v3-Reason-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwenvergence-14B-v3-Prose.i1-Q4_K_M.gguf": ["mradermacher/Qwenvergence-14B-v3-Prose-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwenvergence-14B-v3.i1-Q4_K_M.gguf": ["mradermacher/Qwenvergence-14B-v3-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwen2.5-14B-SLERPv7.i1-Q4_K_M.gguf": ["mradermacher/Qwen2.5-14B-SLERPv7-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Qwen2.5-14B-Instruct-abliterated-v2.i1-Q4_K_M.gguf": ["mradermacher/Qwen2.5-14B-Instruct-abliterated-v2-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Prima-LelantaclesV5-7b.i1-Q5_K_M.gguf": ["mradermacher/Prima-LelantaclesV5-7b-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "q2.5-veltha-14b-q5_k_m.gguf": ["djuna/Q2.5-Veltha-14B-Q5_K_M-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "OgnoExperiment27-7B.i1-Q5_K_M.gguf": ["mradermacher/OgnoExperiment27-7B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Neurotic-Jomainotrik-7b-slerp.i1-Q5_K_M.gguf": ["mradermacher/Neurotic-Jomainotrik-7b-slerp-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "NeuralPearlBeagle.Q5_K_M.gguf": ["mradermacher/NeuralPearlBeagle-GGUF", MessagesFormatterType.MISTRAL],
+    "Mistral-7B-Instruct-Ukrainian.i1-Q5_K_M.gguf": ["mradermacher/Mistral-7B-Instruct-Ukrainian-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Maverick-7B.i1-Q5_K_M.gguf": ["mradermacher/Maverick-7B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Hermes3-L3.1-DirtyHarry-8B.Q5_K_M.gguf": ["mradermacher/Hermes3-L3.1-DirtyHarry-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "Hermes3-L3.1-DarkPlanetSF-8B.i1-Q5_K_M.gguf": ["mradermacher/Hermes3-L3.1-DarkPlanetSF-8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Hermes3-L3.1-BigTalker-8B.i1-Q5_K_M.gguf": ["mradermacher/Hermes3-L3.1-BigTalker-8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Casuar-9B-Model_Stock.i1-Q4_K_M.gguf": ["mradermacher/Casuar-9B-Model_Stock-i1-GGUF", MessagesFormatterType.ALPACA],
+    "ZEUS-8B-V9.i1-Q5_K_M.gguf": ["mradermacher/ZEUS-8B-V9-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "vicious_mesh-12b-delta-q4_k_m.gguf": ["bamec66557/VICIOUS_MESH-12B-DELTA-Q4_K_M-GGUF", MessagesFormatterType.MISTRAL],
+    "Neural-Logical-Abstract-7B-slerp.Q5_K_M.gguf": ["mradermacher/Neural-Logical-Abstract-7B-slerp-GGUF", MessagesFormatterType.MISTRAL],
+    "NeuralDareBeagle-7B-slerp.Q5_K_M.gguf": ["mradermacher/NeuralDareBeagle-7B-slerp-GGUF", MessagesFormatterType.MISTRAL],
+    "Neural_Waifu_7b_V0.1.Q5_K_M.gguf": ["mradermacher/Neural_Waifu_7b_V0.1-GGUF", MessagesFormatterType.MISTRAL],
+    "Mistral-7B-Instruct_v0.2_UNA-TheBeagle-7b-v1.i1-Q5_K_M.gguf": ["mradermacher/Mistral-7B-Instruct_v0.2_UNA-TheBeagle-7b-v1-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "MISCHIEVOUS-12B-Mix_Neo.i1-Q4_K_M.gguf": ["mradermacher/MISCHIEVOUS-12B-Mix_Neo-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "MiaLatte-Indo-Mistral-7b.Q5_K_M.gguf": ["mradermacher/MiaLatte-Indo-Mistral-7b-GGUF", MessagesFormatterType.MISTRAL],
+    "MFANN-SFT-2.i1-Q5_K_M.gguf": ["mradermacher/MFANN-SFT-2-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "maestrale-chat-v0.3-beta-sft.i1-Q5_K_M.gguf": ["mradermacher/maestrale-chat-v0.3-beta-sft-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Llama-3-EZO-8b-Common-it.i1-Q5_K_M.gguf": ["mradermacher/Llama-3-EZO-8b-Common-it-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Legal-Saul-Multiverse-7b.Q5_K_M.gguf": ["mradermacher/Legal-Saul-Multiverse-7b-GGUF", MessagesFormatterType.MISTRAL],
+    "kuno-royale-v3-7b.Q5_K_M.gguf": ["mradermacher/kuno-royale-v3-7b-GGUF", MessagesFormatterType.MISTRAL],
+    "Kuno-lake-slerp-7b.Q5_K_M.gguf": ["mradermacher/Kuno-lake-slerp-7b-GGUF", MessagesFormatterType.MISTRAL],
+    "KRONOS-8B-V6.i1-Q5_K_M.gguf": ["mradermacher/KRONOS-8B-V6-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Homer-v1.0-Qwen2.5-7B.i1-Q5_K_M.gguf": ["mradermacher/Homer-v1.0-Qwen2.5-7B-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "FuseCyberMix-Qwen-2.5-7B-Instruct.i1-Q5_K_M.gguf": ["mradermacher/FuseCyberMix-Qwen-2.5-7B-Instruct-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "flammen7-mistral-7B.Q5_K_M.gguf": ["mradermacher/flammen7-mistral-7B-GGUF", MessagesFormatterType.MISTRAL],
+    "flammen6-mistral-7B.Q5_K_M.gguf": ["mradermacher/flammen6-mistral-7B-GGUF", MessagesFormatterType.MISTRAL],
+    "flammen3X-mistral-7B.i1-Q5_K_M.gguf": ["mradermacher/flammen3X-mistral-7B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Eris-Floramix-7b.i1-Q5_K_M.gguf": ["mradermacher/Eris-Floramix-7b-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Eris-Daturamix-7b.i1-Q5_K_M.gguf": ["mradermacher/Eris-Daturamix-7b-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Eris_Remix_DPO_7B.i1-Q5_K_M.gguf": ["mradermacher/Eris_Remix_DPO_7B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "dolphin-2.6-mistral-7b-dpo-chat.Q5_K_M.gguf": ["mradermacher/dolphin-2.6-mistral-7b-dpo-chat-GGUF", MessagesFormatterType.MISTRAL],
+    "BuRP_7B.i1-Q5_K_M.gguf": ["mradermacher/BuRP_7B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Blurred-Beagle-7b-slerp.Q5_K_M.gguf": ["mradermacher/Blurred-Beagle-7b-slerp-GGUF", MessagesFormatterType.MISTRAL],
+    "Blur-7B-slerp-v0.1.i1-Q5_K_M.gguf": ["mradermacher/Blur-7B-slerp-v0.1-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Biomistral-Clown-Slerp.Q5_K_M.gguf": ["mradermacher/Biomistral-Clown-Slerp-GGUF", MessagesFormatterType.MISTRAL],
+    "Deris-SSS-7B.Q5_K_M.gguf": ["mradermacher/Deris-SSS-7B-GGUF", MessagesFormatterType.MISTRAL],
+    "Dark-Waifu-7b.Q5_K_M.gguf": ["mradermacher/Dark-Waifu-7b-GGUF", MessagesFormatterType.MISTRAL],
+    "Brezn3.Q5_K_M.gguf": ["mradermacher/Brezn3-GGUF", MessagesFormatterType.MISTRAL],
+    "BreezeDolphin-SLERP-0.1.Q5_K_M.gguf": ["mradermacher/BreezeDolphin-SLERP-0.1-GGUF", MessagesFormatterType.MISTRAL],
+    "Blitz-v0.1.i1-Q5_K_M.gguf": ["mradermacher/Blitz-v0.1-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Asymmetric_Linearity-8B-Model_Stock.Q5_K_M.gguf": ["mradermacher/Asymmetric_Linearity-8B-Model_Stock-GGUF", MessagesFormatterType.LLAMA_3],
+    "KRONOS-8B-V5.i1-Q5_K_M.gguf": ["mradermacher/KRONOS-8B-V5-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "14b-Qwen2.5-Infermatic-Crea-v1.i1-Q4_K_M.gguf": ["mradermacher/14b-Qwen2.5-Infermatic-Crea-v1-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "TQ2.5-14B-Neon-v1-Q4_K_M.gguf": ["bartowski/TQ2.5-14B-Neon-v1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "TQ2.5-14B-Aletheia-v1-Q4_K_M.gguf": ["bartowski/TQ2.5-14B-Aletheia-v1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "SynthIQ-7b.i1-Q5_K_M.gguf": ["mradermacher/SynthIQ-7b-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "MISCHIEVOUS-12B-Mix_III_IV_V.i1-Q4_K_M.gguf": ["mradermacher/MISCHIEVOUS-12B-Mix_III_IV_V-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "KRONOS-8B-V4.i1-Q5_K_M.gguf": ["mradermacher/KRONOS-8B-V4-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "fratricide-12B-Unslop-Mell-DARKNESS.i1-Q4_K_M.gguf": ["mradermacher/fratricide-12B-Unslop-Mell-DARKNESS-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS-v2.i1-Q4_K_M.gguf": ["mradermacher/AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS-v2-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Llama-3-8B-ArliAI-Formax-v1.0.i1-Q5_K_M.gguf": ["mradermacher/Llama-3-8B-ArliAI-Formax-v1.0-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "MonarchLake-7B.i1-Q5_K_M.gguf": ["mradermacher/MonarchLake-7B-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Evangelion-7B.i1-Q5_K_M.gguf": ["mradermacher/Evangelion-7B-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Mistral-7B-Instruct-v0.2-Neural-Story.Q5_K_M.gguf": ["mradermacher/Mistral-7B-Instruct-v0.2-Neural-Story-GGUF", MessagesFormatterType.MISTRAL],
     "Layris_9B.Q4_K_M.gguf": ["mradermacher/Layris_9B-GGUF", MessagesFormatterType.MISTRAL],
     "Eris_Remix_7B.Q5_K_M.gguf": ["mradermacher/Eris_Remix_7B-GGUF", MessagesFormatterType.MISTRAL],