Spaces:
Build error
Build error
File size: 914 Bytes
25373e0 5cc76b1 25373e0 5cc76b1 ff3a12f 5cc76b1 ff3a12f 5cc76b1 ff3a12f 5cc76b1 25373e0 5cc76b1 25373e0 5cc76b1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
import gradio as gr
from transformers import (
AutoModelForCausalLM,
AutoConfig,
AutoTokenizer,
BitsAndBytesConfig,
)
import transformers
import torch
model_name = "tiiuae/falcon-40b"
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
model,
quantization_config=BitsAndBytesConfig(load_in_4bit=True),
trust_remote_code=True,
torch_dtype=torch.bfloat16,
device_map="auto",
)
tokenizer = AutoTokenizer.from_pretrained(model_name)
def falcon(input_text):
input_ids = tokenizer(input_text, return_tensors="pt").input_ids.to("cuda")
outputs = model.generate(input_ids, max_length=100, do_sample=True, top_k=10)
decoded = tokenizer.decode(outputs[0])
return decoded
iface = gr.Interface(fn=falcon, inputs="text", outputs="text")
iface.launch() # To create a public link, set `share=True`
|