File size: 1,367 Bytes
d010b9c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import torch
import gradio as gr
from transformers import pipeline, logging, AutoModelForCausalLM, AutoTokenizer
model_name = "microsoft/phi-2"
model = AutoModelForCausalLM.from_pretrained(
model_name,
trust_remote_code=True
)
model.config.use_cache = False
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token
peft_model_folder = './ckpts'
model.load_adapter(peft_model_folder)
def generate_text(input_text, max_length):
pipe = pipeline(task="text-generation",model=model,tokenizer=tokenizer, max_length=max_length)
result = pipe(f"<s>[INST] {input_text} [/INST]")
return_answer = result[0]['generated_text']
return return_answer
# Create a Gradio interface
title = "Phi2-QLora."
description = "A simple Gradio interface to demo Phi2 model finetuned on openassist dataset with Qlora."
examples = [["What is Large Language Model?"],
["Why Python is most popular Language?"],
["How to do rice?"]]
demo = gr.Interface(
generate_text,
inputs=[
gr.TextArea(label="Enter Question"),
gr.Slider(1, 200, value = 10, step=1, label="Max Length")
],
outputs=[
gr.Textbox(label="Response from Phi2 Model: "),
],
title=title,
description=description,
examples=examples,
cache_examples=False,
)
demo.launch() |