| import gradio as gr | |
| from llama_cpp import Llama | |
| import json | |
| llm = Llama(model_path="./neuralhermes-2.5-mistral-7b.Q5_K_M.gguf", | |
| n_ctx=32768, | |
| n_threads=2, | |
| chat_format="chatml") | |
| def chat_completion(messages, history, system_prompt): | |
| messages_prompts = [{"role": "system", "content": system_prompt}] | |
| for human, assistant in history: | |
| messages_prompts.append({"role": "user", "content": human}) | |
| messages_prompts.append({"role": "assistant", "content": assistant}) | |
| messages_prompts.append({"role": "user", "content": messages}) | |
| response = llm.create_chat_completion( | |
| messages=messages_prompts, | |
| stream=False | |
| ) | |
| print(json.dumps(response, ensure_ascii=False, indent=2)) | |
| return response['choices'][0]['content'] | |
| def chat_stream_completion(messages, history, system_prompt): | |
| messages_prompts = [{"role": "system", "content": system_prompt}] | |
| for human, assistant in history: | |
| messages_prompts.append({"role": "user", "content": human}) | |
| messages_prompts.append({"role": "assistant", "content": assistant}) | |
| messages_prompts.append({"role": "user", "content": messages}) | |
| response = llm.create_chat_completion( | |
| messages=messages_prompts, | |
| stream=True | |
| ) | |
| partial_message = "" | |
| for chunk in response: | |
| if len(chunk['choices'][0]["delta"]) != 0 and "content" in chunk['choices'][0]["delta"]: | |
| partial_message = partial_message + \ | |
| chunk['choices'][0]["delta"]["content"] | |
| yield partial_message | |
| gr.ChatInterface(chat_stream_completion, | |
| additional_inputs=[gr.Textbox( | |
| "You are helpful AI.", label="System Prompt")] | |
| ).queue().launch(server_name="0.0.0.0") | |