Commit
•
e234444
1
Parent(s):
2c50b69
Update handler.py
Browse filespushing model to cuda:0
- handler.py +1 -1
handler.py
CHANGED
@@ -12,7 +12,7 @@ class EndpointHandler:
|
|
12 |
# load the model
|
13 |
tokenizer = AutoTokenizer.from_pretrained(path)
|
14 |
model = AutoModelForCausalLM.from_pretrained(path, device_map="auto", torch_dtype=dtype, trust_remote_code=True)
|
15 |
-
model.to('cuda')
|
16 |
|
17 |
# create inference pipeline
|
18 |
self.pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device='cuda:0')
|
|
|
12 |
# load the model
|
13 |
tokenizer = AutoTokenizer.from_pretrained(path)
|
14 |
model = AutoModelForCausalLM.from_pretrained(path, device_map="auto", torch_dtype=dtype, trust_remote_code=True)
|
15 |
+
model.to('cuda:0')
|
16 |
|
17 |
# create inference pipeline
|
18 |
self.pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device='cuda:0')
|