File size: 1,592 Bytes
9bc80c6 1717bbe f60d35d 1717bbe f60d35d 1717bbe 9bc80c6 6da9f6d 9bc80c6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline,BitsAndBytesConfig
#model = AutoModelForCausalLM.from_pretrained("checkpoint_500",trust_remote_code=True)
model_name = "microsoft/phi-2"
#bnb_config = BitsAndBytesConfig(
# load_in_4bit=True,
# bnb_4bit_quant_type="nf4",
# bnb_4bit_compute_dtype=torch.float16,
#)
model = AutoModelForCausalLM.from_pretrained(
model_name,
#quantization_config=bnb_config,
trust_remote_code=True
)
model.config.use_cache = False
model.load_adapter("checkpoint")
tokenizer = AutoTokenizer.from_pretrained("checkpoint_500", trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token
def inference(prompt, count):
pipe = pipeline(task="text-generation", model=model, tokenizer=tokenizer, max_length=200)
result = pipe(f"### Human: {prompt}",max_new_tokens=count)
out_text = result[0]['generated_text']
return out_text
title = "TSAI S21 Assignment: Adaptive QLoRA training on open assist oasst1 dataset, using microsoft/phi2 model"
description = "A simple Gradio interface that accepts a context and generates GPT like text "
examples = [["What is a large language model?","200"],
["Explain about monopsony","200"]
]
demo = gr.Interface(
inference,
inputs = [gr.Textbox(placeholder="Enter a prompt"), gr.Textbox(placeholder="Enter number of characters you want to generate")],
outputs = [gr.Textbox(label="Chat GPT like text")],
title = title,
description = description,
examples = examples
)
demo.launch() |