| import gradio as gr | |
| import os | |
| from pathlib import Path | |
| import argparse | |
| from huggingface_hub import snapshot_download | |
| from llama_cpp import Llama | |
| repo_name = 'tolgadev/llama-2-7b-ruyallm-GGUF' | |
| model_file = "llama-2-7b-tk-mini.Q4_K_M.gguf" | |
| snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_file) | |
| DEFAULT_MODEL_PATH = model_file | |
| llm = Llama(model_path=model_file, model_type="llama") | |
| def predict(input, chatbot, max_length, top_p, temperature, history): | |
| chatbot.append((input, "")) | |
| response = "" | |
| history.append(input) | |
| for output in llm(input, stream=True, temperature=temperature, top_p=top_p, max_tokens=max_length, ): | |
| piece = output['choices'][0]['text'] | |
| response += piece | |
| chatbot[-1] = (chatbot[-1][0], response) | |
| yield chatbot, history | |
| history.append(response) | |
| yield chatbot, history | |
| def reset_user_input(): | |
| return gr.update(value="") | |
| def reset_state(): | |
| return [], [] | |
| with gr.Blocks() as demo: | |
| gr.HTML("""<h1 align="center">RuyaTabirleriLLM Chatbot Demo</h1> | |
| <h3 align="center">This is unofficial demo of `tolgadev/llama-2-7b-tk-mini-GGUF` model based on LLama2 architecture.</h3> | |
| <h4 align="center">Hit the like button if you liked! 🤗</h4>""") | |
| chatbot = gr.Chatbot() | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| user_input = gr.Textbox(show_label=False, placeholder="Input...", lines=8, elem_id="user_input") | |
| submitBtn = gr.Button("Submit", variant="primary", elem_id="submit_btn") | |
| with gr.Column(scale=1): | |
| max_length = gr.Slider(0, 2048, value=1024, step=2.0, label="Maximum Length", interactive=True) | |
| top_p = gr.Slider(0, 1, value=0.7, step=0.01, label="Top P", interactive=True) | |
| temperature = gr.Slider(0, 1.0, value=0.7, step=0.1, label="Temperature", interactive=True) | |
| emptyBtn = gr.Button("Clear History") | |
| history = gr.State([]) | |
| submitBtn.click( | |
| predict, [user_input, chatbot, max_length, top_p, temperature, history], [chatbot, history], show_progress=True | |
| ) | |
| submitBtn.click(reset_user_input, [], [user_input]) | |
| emptyBtn.click(reset_state, outputs=[chatbot, history], show_progress=True) | |
| demo.queue().launch(share=False, inbrowser=True) |