cloudyu
/

Mixtral_34Bx2_MoE_60B

Text Generation

Mixture of Experts

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

cloudyu commited on Jan 6

Commit

6426e89

•

1 Parent(s): f49d7cf

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -44,7 +44,7 @@ model_path = "cloudyu/Mixtral_34Bx2_MoE_60B"
 tokenizer = AutoTokenizer.from_pretrained(model_path, use_default_system_prompt=False)
 model = AutoModelForCausalLM.from_pretrained(
-    model_path, torch_dtype=torch.float32, device_map='cpu',local_files_only=False
 )
 print(model)
 prompt = input("please input prompt:")

 tokenizer = AutoTokenizer.from_pretrained(model_path, use_default_system_prompt=False)
 model = AutoModelForCausalLM.from_pretrained(
+        model_path, torch_dtype=torch.bfloat16, device_map='cpu'
 )
 print(model)
 prompt = input("please input prompt:")