grim3000
/

chatGLM3-6B-Base

Inference Endpoints

Model card Files Files and versions Community

Marlon Wiprud commited on Nov 27, 2023

Commit

693f1b3

•

1 Parent(s): 4fbf700

chore: update

Files changed (1) hide show

handler.py +11 -11

handler.py CHANGED Viewed

@@ -25,17 +25,17 @@ class EndpointHandler:
         self.tokenizer = LlamaTokenizer.from_pretrained("lmsys/vicuna-7b-v1.5")
-        with init_empty_weights():
-            self.model = (
-                AutoModelForCausalLM.from_pretrained(
-                    "THUDM/cogvlm-chat-hf",
-                    torch_dtype=torch.bfloat16,
-                    low_cpu_mem_usage=True,
-                    trust_remote_code=True,
-                )
-                .to("cuda")
-                .eval()
-            )
         device_map = infer_auto_device_map(
             model,

         self.tokenizer = LlamaTokenizer.from_pretrained("lmsys/vicuna-7b-v1.5")
+        # with init_empty_weights():
+        #     self.model = (
+        #         AutoModelForCausalLM.from_pretrained(
+        #             "THUDM/cogvlm-chat-hf",
+        #             torch_dtype=torch.bfloat16,
+        #             low_cpu_mem_usage=True,
+        #             trust_remote_code=True,
+        #         )
+        #         .to("cuda")
+        #         .eval()
+        #     )
         device_map = infer_auto_device_map(
             model,