llama-2-gguf / app.py
mvasim's picture
first commit
81cf53b
raw history blame
No virus
1.55 kB
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.chains import LLMChain
from langchain.prompts import PromptTemplate
from langchain_community.llms import LlamaCpp
import gradio as gr
MODEL_PATH = "llama-2-7b-chat.Q5_K_M.gguf"
TEMPLATE = """
You are a helpful AI Assistant created by Mohammed Vasim. Mohammed Vasim is an AI Engineer.
Question: {question}
Answer: helpful answer"""
prompt = PromptTemplate.from_template(TEMPLATE)
# Callbacks support token-wise streaming
callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
# Make sure the model path is correct for your system!
llm = LlamaCpp(
model_path=MODEL_PATH,
temperature=0.75,
max_tokens=2000,
top_p=1,
callback_manager=callback_manager,
verbose=True, # Verbose is required to pass to the callback manager
)
llm_chain = LLMChain(prompt=prompt, llm=llm)
# question = "What NFL team won the Super Bowl in the year Justin Bieber was born?"
# llm_chain.run(question)
title = "Welcome to Open Source LLM"
description = "This is a Llama-2-GGUF"
def answer_query(message, history):
message = llm_chain.run(message)
return message
# Gradio chat interface
gr.ChatInterface(
fn=answer_query,
title=title,
description=description,
examples=[
["What is a Large Language Model?"],
["What's 9+2-1?"],
["Write Python code to print the Fibonacci sequence"]
]
).queue().launch(server_name="0.0.0.0")