Alpaca-LoRa / app.py
Monster's picture
Update app.py
d9e1e84
from __future__ import annotations
from typing import Iterable
import gradio as gr
from gradio.themes.base import Base
from gradio.themes.utils import colors, fonts, sizes
from llama_cpp import Llama
from huggingface_hub import hf_hub_download
hf_hub_download(repo_id="Monster/a", filename="ggml-alpaca-7b-q4.bin", local_dir=".")
llm = Llama(model_path="./ggml-alpaca-7b-q4.bin")
ins = '''Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction:
{}
### Response:
'''
ins_inp = '''Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction:
{}
### Input:
{}
### Response:
'''
theme = gr.themes.Monochrome(
primary_hue="indigo",
secondary_hue="blue",
neutral_hue="slate",
radius_size=gr.themes.sizes.radius_sm,
font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
)
def generate(
instruction,
input=None,
temperature=0.1,
top_p=0.95,
top_k=40,):
result = ""
if input:
instruction = ins_inp.format(instruction, input)
else:
instruction = ins.format(instruction)
for x in llm(instruction, stop=['### Instruction:', '### End'], stream=True, temperature=temperature, top_p=top_p, top_k=top_k):
result += x['choices'][0]['text']
yield result
examples = [
"Instead of making a peanut butter and jelly sandwich, what else could I combine peanut butter with in a sandwich? Give five ideas",
"How do I make a campfire?",
"Explain to me the difference between nuclear fission and fusion.",
"Write an ad for sale Nikon D750."
]
def process_example(args):
for x in generate(args):
pass
return x
css = ".generating {visibility: hidden}"
# Based on the gradio theming guide and borrowed from https://huggingface.co/spaces/shivi/dolly-v2-demo
class SeafoamCustom(Base):
def __init__(
self,
*,
primary_hue: colors.Color | str = colors.emerald,
secondary_hue: colors.Color | str = colors.blue,
neutral_hue: colors.Color | str = colors.blue,
spacing_size: sizes.Size | str = sizes.spacing_md,
radius_size: sizes.Size | str = sizes.radius_md,
font: fonts.Font
| str
| Iterable[fonts.Font | str] = (
fonts.GoogleFont("Quicksand"),
"ui-sans-serif",
"sans-serif",
),
font_mono: fonts.Font
| str
| Iterable[fonts.Font | str] = (
fonts.GoogleFont("IBM Plex Mono"),
"ui-monospace",
"monospace",
),
):
super().__init__(
primary_hue=primary_hue,
secondary_hue=secondary_hue,
neutral_hue=neutral_hue,
spacing_size=spacing_size,
radius_size=radius_size,
font=font,
font_mono=font_mono,
)
super().set(
button_primary_background_fill="linear-gradient(90deg, *primary_300, *secondary_400)",
button_primary_background_fill_hover="linear-gradient(90deg, *primary_200, *secondary_300)",
button_primary_text_color="white",
button_primary_background_fill_dark="linear-gradient(90deg, *primary_600, *secondary_800)",
block_shadow="*shadow_drop_lg",
button_shadow="*shadow_drop_lg",
input_background_fill="zinc",
input_border_color="*secondary_300",
input_shadow="*shadow_drop",
input_shadow_focus="*shadow_drop_lg",
)
seafoam = SeafoamCustom()
with gr.Blocks(theme=seafoam, analytics_enabled=False, css=css) as demo:
with gr.Column():
gr.Markdown(
""" ## Alpaca-LoRa
7b quantized 4bit (q4_0)
Type in the box below and click the button to generate answers to your most pressing questions!
"""
)
with gr.Row():
with gr.Column(scale=3):
instruction = gr.Textbox(lines=2, placeholder="Tell me more about alpacas.", label="Instruction", elem_id="q-input")
with gr.Accordion("Advanced setting", open=False):
input = gr.components.Textbox(lines=2, label="Input", placeholder="none")
temperature = gr.components.Slider(minimum=0, maximum=1, value=0.1, label="Temperature")
top_p = gr.components.Slider(minimum=0, maximum=1, value=0.95, label="Top p")
top_k = gr.components.Slider(minimum=0, maximum=100, step=1, value=40, label="Top k")
with gr.Box():
gr.Markdown("**Output**")
output = gr.Markdown(elem_id="q-output")
submit = gr.Button("Generate", variant="primary")
gr.Examples(
examples=examples,
inputs=[instruction],
cache_examples=False,
fn=process_example,
outputs=[output],
)
submit.click(generate, inputs=[instruction, input, temperature, top_p, top_k], outputs=[output])
instruction.submit(generate, inputs=[instruction], outputs=[output])
demo.queue(concurrency_count=1).launch(debug=True)