rlanner-echocap commited on
Commit
2c50b69
1 Parent(s): f9a0290

Update handler.py

Browse files

Trying stack overflow fix

Files changed (1) hide show
  1. handler.py +1 -0
handler.py CHANGED
@@ -12,6 +12,7 @@ class EndpointHandler:
12
  # load the model
13
  tokenizer = AutoTokenizer.from_pretrained(path)
14
  model = AutoModelForCausalLM.from_pretrained(path, device_map="auto", torch_dtype=dtype, trust_remote_code=True)
 
15
 
16
  # create inference pipeline
17
  self.pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device='cuda:0')
 
12
  # load the model
13
  tokenizer = AutoTokenizer.from_pretrained(path)
14
  model = AutoModelForCausalLM.from_pretrained(path, device_map="auto", torch_dtype=dtype, trust_remote_code=True)
15
+ model.to('cuda')
16
 
17
  # create inference pipeline
18
  self.pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, device='cuda:0')