neuralmagic
/

Meta-Llama-3.1-405B-Instruct-FP8-dynamic

Text Generation

text-generation-inference

Inference Endpoints

compressed-tensors

Model card Files Files and versions Community

Lin-K76 commited on Aug 22

Commit

cd22294

•

1 Parent(s): 55d57ff

Update README.md

Files changed (1) hide show

README.md +2 -2

README.md CHANGED Viewed

@@ -118,11 +118,11 @@ model_stub = "meta-llama/Meta-Llama-3.1-405B-Instruct"
 model_name = model_stub.split("/")[-1]
 device_map = calculate_offload_device_map(
-    model_stub, reserve_for_hessians=False, num_gpus=8, torch_dtype=torch.float16
 )
 model = SparseAutoModelForCausalLM.from_pretrained(
-    model_stub, torch_dtype=torch.float16, device_map=device_map
 )
 output_dir = f"./{model_name}-FP8-dynamic"

 model_name = model_stub.split("/")[-1]
 device_map = calculate_offload_device_map(
+    model_stub, reserve_for_hessians=False, num_gpus=8, torch_dtype="auto"
 )
 model = SparseAutoModelForCausalLM.from_pretrained(
+    model_stub, torch_dtype="auto", device_map=device_map
 )
 output_dir = f"./{model_name}-FP8-dynamic"