Spaces:
Running
Running
| from ctransformers import AutoModelForCausalLM | |
| import gradio as gr | |
| llms = { | |
| "tinyllama":{"name": "TheBloke/TinyLlama-1.1B-1T-OpenOrca-GGUF", "file":"tinyllama-1.1b-1t-openorca.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant <|im_end|><|im_start|>user"}, | |
| "orca2":{"name": "TheBloke/Orca-2-7B-GGUF", "file":"orca-2-7b.Q4_K_M.gguf", "suffix":"<|im_end|><|im_start|>assistant", "prefix":"<|im_start|>system You are a helpful assistant<|im_end|><|im_start|>user "}, | |
| "zephyr":{"name": "TheBloke/zephyr-7B-beta-GGUF", "file":"zephyr-7b-beta.Q4_K_M.gguf", "suffix":"</s><|assistant|>", "prefix":"<|system|>You are a helpful assistant</s><|user|> "}, | |
| "mixtral":{"name": "TheBloke/Mistral-7B-Instruct-v0.1-GGUF", "file":"mistral-7b-instruct-v0.1.Q4_K_M.gguf", "suffix":"[/INST]", "prefix":"<s>[INST] "}, | |
| "llama2":{"name": "TheBloke/Llama-2-7B-Chat-GGUF", "file":"llama-2-7b-chat.Q4_K_M.gguf", "suffix":"[/INST]", "prefix":"[INST] <<SYS>> You are a helpful assistant <</SYS>>"}, | |
| "solar":{"name": "TheBloke/SOLAR-10.7B-Instruct-v1.0-GGUF", "file":"solar-10.7b-instruct-v1.0.Q4_K_M.gguf", "suffix":"\n### Assistant:\n", "prefix":"### User:\n"}, | |
| #"open-llama": {"name": "TheBloke/open-llama-3b-v2-wizard-evol-instuct-v2-196k-GGUF", "file":"open-llama-3b-v2-wizard-evol-instuct-v2-196k.Q4_K_M.gguf", "suffix":"\n\n### RESPONSE", "prefix":"### HUMAN:\n"} | |
| } | |
| for k in llms.keys(): | |
| AutoModelForCausalLM.from_pretrained(llms[k]['name'], model_file=llms[k]['file']) | |
| import gradio as gr | |
| def predict(prompt, llm_name): | |
| prefix=llms[llm_name]['prefix'] | |
| suffix=llms[llm_name]['suffix'] | |
| user=""" | |
| {prompt}""" | |
| llm = AutoModelForCausalLM.from_pretrained(llms[llm_name]['name'], model_file=llms[llm_name]['file']) | |
| prompt = f"{prefix}{user.replace('{prompt}', prompt)}{suffix}" | |
| return llm(prompt) | |
| # Create the Gradio interface | |
| interface = gr.Interface( | |
| fn=predict, | |
| inputs=[gr.Textbox(label="Prompt", lines=20), gr.Dropdown(choices=list(llms), label="Select an LLM", value="tinyllama")], | |
| outputs="text" | |
| ) | |
| if __name__ == "__main__": | |
| interface.launch() |