apoorvkh
/

blip2-flan-t5-xxl-endpoint

Inference Endpoints

Model card Files Files and versions Community

apoorvkh commited on Jun 18, 2023

Commit

5eda3fd

•

1 Parent(s): ca66b5c

Loading in 8-bit

Files changed (1) hide show

handler.py +2 -2

handler.py CHANGED Viewed

@@ -22,8 +22,8 @@ class EndpointHandler():
         self.model = Blip2ForConditionalGeneration.from_pretrained(
             "Salesforce/blip2-flan-t5-xxl", device_map=device_map,
-            torch_dtype=torch.float16
-            # load_in_8bit=True,
         )
     def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:

         self.model = Blip2ForConditionalGeneration.from_pretrained(
             "Salesforce/blip2-flan-t5-xxl", device_map=device_map,
+            # torch_dtype=torch.float16
+            load_in_8bit=True,
         )
     def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]: