|
import os |
|
import urllib.request |
|
import gradio as gr |
|
from llama_cpp import Llama |
|
|
|
|
|
def download_file(file_link, filename): |
|
|
|
if not os.path.isfile(filename): |
|
urllib.request.urlretrieve(file_link, filename) |
|
print("File downloaded successfully.") |
|
else: |
|
print("File already exists.") |
|
|
|
|
|
|
|
ggml_model_path = "https://huggingface.co/CRD716/ggml-vicuna-1.1-quantized/resolve/main/ggml-vicuna-7b-1.1-q4_1.bin" |
|
filename = "ggml-vicuna-7b-1.1-q4_1.bin" |
|
|
|
download_file(ggml_model_path, filename) |
|
|
|
|
|
llm = Llama(model_path=filename, n_ctx=512, n_batch=126) |
|
|
|
|
|
def generate_text(prompt="Who is the CEO of Apple?"): |
|
output = llm( |
|
prompt, |
|
max_tokens=256, |
|
temperature=0.1, |
|
top_p=0.5, |
|
echo=False, |
|
stop=["#"], |
|
) |
|
output_text = output["choices"][0]["text"].strip() |
|
|
|
|
|
cleaned_output_text = output_text.replace(prompt, "") |
|
return cleaned_output_text |
|
|
|
|
|
description = "Learning to use Vicuna-7B" |
|
|
|
examples = [ |
|
["What is the capital of France?", "The capital of France is Paris."], |
|
[ |
|
"Who wrote the novel 'Pride and Prejudice'?", |
|
"The novel 'Pride and Prejudice' was written by Jane Austen.", |
|
], |
|
["What is the square root of 64?", "The square root of 64 is 8."], |
|
] |
|
|
|
gradio_interface = gr.Interface( |
|
fn=generate_text, |
|
inputs="text", |
|
outputs="text", |
|
examples=examples, |
|
title="Learning to use Vicuna-7B", |
|
) |
|
gradio_interface.launch() |
