File size: 1,213 Bytes
f9bf33a 597364b 8d6f2ec 597364b 8d6f2ec 597364b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
import streamlit as st
import torch
import transformers
from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
import os
import torch
import torch.nn as nn
import bitsandbytes as bnb
from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM
from datasets import Dataset
import pandas as pd
import transformers
from datasets import load_dataset
from peft import LoraConfig, get_peft_model
import time
peft_model_id = "foobar8675/bloom-7b1-lora-tagger"
config = PeftConfig.from_pretrained(peft_model_id)
model = AutoModelForCausalLM.from_pretrained("bigscience/bloom-7b1", return_dict=True, load_in_8bit=True, device_map='auto')
tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom-7b1")
# # Load the Lora model
model = PeftModel.from_pretrained(model, peft_model_id)
text = st.text_area('enter text in this format : “<<report>>” ->: ')
if text:
start_time = time.time()
batch = tokenizer(text, return_tensors='pt')
output_tokens = model.generate(**batch, max_new_tokens=25)
out = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
st.json(out)
st.json(f"Elapsed time: {time.time() - start_time}s")
|