Commit
•
78715f1
1
Parent(s):
9b91b1c
Fix
Browse files
app.py
CHANGED
@@ -6,7 +6,6 @@ from transformers import (
|
|
6 |
StoppingCriteria,
|
7 |
StoppingCriteriaList,
|
8 |
TextIteratorStreamer,
|
9 |
-
BitsAndBytesConfig,
|
10 |
)
|
11 |
import gradio as gr
|
12 |
import torch
|
@@ -19,18 +18,10 @@ max_new_tokens = 1024
|
|
19 |
|
20 |
print(f"Starting to load the model {model_name}")
|
21 |
|
22 |
-
bnb_config = BitsAndBytesConfig(
|
23 |
-
llm_int8_enable_fp32_cpu_offload=True,
|
24 |
-
load_in_8bit=True,
|
25 |
-
)
|
26 |
-
|
27 |
m = AutoModelForCausalLM.from_pretrained(
|
28 |
model_name,
|
29 |
-
device_map
|
30 |
-
|
31 |
-
"": "cpu",
|
32 |
-
},
|
33 |
-
quantization_config=bnb_config,
|
34 |
)
|
35 |
|
36 |
m.config.pad_token_id = m.config.eos_token_id
|
|
|
6 |
StoppingCriteria,
|
7 |
StoppingCriteriaList,
|
8 |
TextIteratorStreamer,
|
|
|
9 |
)
|
10 |
import gradio as gr
|
11 |
import torch
|
|
|
18 |
|
19 |
print(f"Starting to load the model {model_name}")
|
20 |
|
|
|
|
|
|
|
|
|
|
|
21 |
m = AutoModelForCausalLM.from_pretrained(
|
22 |
model_name,
|
23 |
+
device_map=0,
|
24 |
+
#load_in_8bit=True,
|
|
|
|
|
|
|
25 |
)
|
26 |
|
27 |
m.config.pad_token_id = m.config.eos_token_id
|