Spaces:
Sleeping
Sleeping
File size: 1,704 Bytes
81cf53b e3bfcb3 c32b415 81cf53b e3bfcb3 81cf53b c32b415 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
from langchain.callbacks.manager import CallbackManager
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.chains import LLMChain
from langchain.prompts import PromptTemplate
from langchain_community.llms import LlamaCpp
import gradio as gr
import os
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
REPO = "TheBloke/Llama-2-7B-Chat-GGUF"
MODEL_NAME = "llama-2-7b-chat.Q5_K_M.gguf"
DOWNLOAD_MODEL = f"huggingface-cli download {REPO} {MODEL_NAME} --local-dir . --local-dir-use-symlinks False"
MODEL_PATH = "models/llama-2-7b-chat.Q5_K_M.gguf"
if not os.path.exists(MODEL_PATH):
os.system(DOWNLOAD_MODEL)
TEMPLATE = """
You are a helpful AI Assistant created by Mohammed Vasim. Mohammed Vasim is an AI Engineer.
Question: {question}
Answer: helpful answer"""
prompt = PromptTemplate.from_template(TEMPLATE)
# Callbacks support token-wise streaming
callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
# Make sure the model path is correct for your system!
llm = LlamaCpp(
model_path=MODEL_PATH,
temperature=0.75,
max_tokens=2000,
top_p=1,
callback_manager=callback_manager,
verbose=True, # Verbose is required to pass to the callback manager
)
llm_chain = LLMChain(prompt=prompt, llm=llm)
# question = "What NFL team won the Super Bowl in the year Justin Bieber was born?"
# llm_chain.run(question)
if __name__ == "__main__":
print("Hello, Friend")
chat = True
while chat:
print("Enter question or q to quit.")
question = input("Question: ")
if question == "q":
chat = False
response = llm_chain.invoke(question)
print(response['text']) |