ikeno-ada
/

m2m100_418-mt-bitsandbytes-4bit

text2text-generation

text-generation-inference

Inference Endpoints

4-bit precision

Model card Files Files and versions Community

ikeno-ada commited on Apr 7

Commit

1bac627

•

1 Parent(s): e3c5e74

Update handler.py

Files changed (1) hide show

handler.py +1 -1

handler.py CHANGED Viewed

@@ -7,7 +7,7 @@ import torch
 class EndpointHandler():
     def __init__(self, path=""):
         # load the optimized model
-        self.model = M2M100ForConditionalGeneration.from_pretrained(path, device_map= "cuda",torch_dtype=torch.bfloat16)
         self.tokenizer = M2M100Tokenizer.from_pretrained(path)
     def __call__(self, data: Dict[str,str]) -> Dict[str, str]:

 class EndpointHandler():
     def __init__(self, path=""):
         # load the optimized model
+        self.model = M2M100ForConditionalGeneration.from_pretrained(path,torch_dtype=torch.bfloat16)
         self.tokenizer = M2M100Tokenizer.from_pretrained(path)
     def __call__(self, data: Dict[str,str]) -> Dict[str, str]: