Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1 |
import gradio as gr
|
2 |
import torch
|
3 |
-
from
|
|
|
|
|
4 |
|
5 |
# Replace with your model name
|
6 |
#MODEL_NAME = "ssirikon/Gemma7b-bnb-Unsloth"
|
@@ -8,35 +10,28 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, AutoConf
|
|
8 |
MODEL_NAME = "Lohith9459/gemma7b"
|
9 |
|
10 |
# Load the model and tokenizer
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
torch_dtype=torch.float16,
|
15 |
-
load_in_4bit=True, # Load the model in 4-bit precision
|
16 |
-
# Removed the unsupported argument
|
17 |
-
)
|
18 |
-
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
19 |
-
config = AutoConfig.from_pretrained(MODEL_NAME)
|
20 |
|
|
|
|
|
21 |
|
22 |
def generate_subject(email_body):
|
23 |
instruction = "Generate a subject line for the following email."
|
24 |
formatted_text = f"""Below is an instruction that describes a task. \
|
25 |
Write a response that appropriately completes the request.
|
26 |
-
|
27 |
### Instruction:
|
28 |
{instruction}
|
29 |
-
|
30 |
### Input:
|
31 |
{email_body}
|
32 |
-
|
33 |
### Response:
|
34 |
"""
|
35 |
inputs = tokenizer([formatted_text], return_tensors="pt").to("cuda")
|
36 |
text_streamer = TextStreamer(tokenizer)
|
37 |
generated_ids = model.generate(**inputs, streamer=text_streamer, max_new_tokens=512)
|
38 |
generated_text = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
|
39 |
-
|
40 |
def extract_subject(text):
|
41 |
start_tag = "### Response:"
|
42 |
start_idx = text.find(start_tag)
|
@@ -44,7 +39,7 @@ def generate_subject(email_body):
|
|
44 |
return None
|
45 |
subject = text[start_idx + len(start_tag):].strip()
|
46 |
return subject
|
47 |
-
|
48 |
return extract_subject(generated_text)
|
49 |
|
50 |
# Create the Gradio interface
|
|
|
1 |
import gradio as gr
|
2 |
import torch
|
3 |
+
from unsloth import FastLanguageModel
|
4 |
+
from transformers import TextStreamer
|
5 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
6 |
|
7 |
# Replace with your model name
|
8 |
#MODEL_NAME = "ssirikon/Gemma7b-bnb-Unsloth"
|
|
|
10 |
MODEL_NAME = "Lohith9459/gemma7b"
|
11 |
|
12 |
# Load the model and tokenizer
|
13 |
+
max_seq_length = 512
|
14 |
+
dtype = torch.bfloat16
|
15 |
+
load_in_4bit = True
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
|
17 |
+
model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.bfloat16, device_map="auto")
|
18 |
+
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
19 |
|
20 |
def generate_subject(email_body):
|
21 |
instruction = "Generate a subject line for the following email."
|
22 |
formatted_text = f"""Below is an instruction that describes a task. \
|
23 |
Write a response that appropriately completes the request.
|
|
|
24 |
### Instruction:
|
25 |
{instruction}
|
|
|
26 |
### Input:
|
27 |
{email_body}
|
|
|
28 |
### Response:
|
29 |
"""
|
30 |
inputs = tokenizer([formatted_text], return_tensors="pt").to("cuda")
|
31 |
text_streamer = TextStreamer(tokenizer)
|
32 |
generated_ids = model.generate(**inputs, streamer=text_streamer, max_new_tokens=512)
|
33 |
generated_text = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
|
34 |
+
|
35 |
def extract_subject(text):
|
36 |
start_tag = "### Response:"
|
37 |
start_idx = text.find(start_tag)
|
|
|
39 |
return None
|
40 |
subject = text[start_idx + len(start_tag):].strip()
|
41 |
return subject
|
42 |
+
|
43 |
return extract_subject(generated_text)
|
44 |
|
45 |
# Create the Gradio interface
|