Spaces:
Runtime error
Runtime error
Upload app.py
Browse files
app.py
CHANGED
@@ -12,12 +12,12 @@ chip_map= {
|
|
12 |
'gpt_neox.final_layer_norm': 0,
|
13 |
'embed_out': 0
|
14 |
}
|
15 |
-
model = AutoModelForCausalLM.from_pretrained(
|
16 |
#model = AutoModelForCausalLM.from_pretrained(model_name)
|
17 |
|
18 |
|
19 |
def predict(input, history=[], MAX_NEW_TOKENS = 500):
|
20 |
-
text = "User: " + input + "\n\
|
21 |
new_user_input_ids = tokenizer(text, return_tensors="pt").input_ids
|
22 |
# bot_input_ids = torch.cat([torch.LongTensor(history), new_user_input_ids], dim=-1).to("cuda")
|
23 |
bot_input_ids = torch.cat([torch.LongTensor(history), new_user_input_ids], dim=-1)
|
|
|
12 |
'gpt_neox.final_layer_norm': 0,
|
13 |
'embed_out': 0
|
14 |
}
|
15 |
+
model = AutoModelForCausalLM.from_pretrained(model_name, device_map=chip_map, torch_dtype=torch.float16, load_in_8bit=True)
|
16 |
#model = AutoModelForCausalLM.from_pretrained(model_name)
|
17 |
|
18 |
|
19 |
def predict(input, history=[], MAX_NEW_TOKENS = 500):
|
20 |
+
text = "User: " + input + "\n\nAI: "
|
21 |
new_user_input_ids = tokenizer(text, return_tensors="pt").input_ids
|
22 |
# bot_input_ids = torch.cat([torch.LongTensor(history), new_user_input_ids], dim=-1).to("cuda")
|
23 |
bot_input_ids = torch.cat([torch.LongTensor(history), new_user_input_ids], dim=-1)
|