amd
/

Llama-3.1-405B-Instruct-FP8-KV

Model card Files Files and versions Community

bowenbaoamd commited on Oct 7

Commit

2ff6250

•

1 Parent(s): 2fb49ed

Update README.md

Remove trailing spaces.

Files changed (1) hide show

README.md +4 -4

README.md CHANGED Viewed

@@ -19,9 +19,9 @@ base_model: meta-llama/Meta-Llama-3.1-405B-Instruct
 ```sh
 export MODEL_DIR = [local model checkpoint folder] or meta-llama/Meta-Llama-3.1-405B-Instruct
 # single GPU
-python3 quantize_quark.py \
         --model_dir $MODEL_DIR \
-        --output_dir Meta-Llama-3.1-405B-Instruct-FP8-KV \
         --quant_scheme w_fp8_a_fp8 \
         --kv_cache_dtype fp8 \
         --num_calib_data 128 \
@@ -29,9 +29,9 @@ python3 quantize_quark.py \
         --no_weight_matrix_merge
 # If model size is too large for single GPU, please use multi GPU instead.
-python3 quantize_quark.py \
         --model_dir $MODEL_DIR \
-        --output_dir Meta-Llama-3.1-405B-Instruct-FP8-KV \
         --quant_scheme w_fp8_a_fp8 \
         --kv_cache_dtype fp8 \
         --num_calib_data 128 \

 ```sh
 export MODEL_DIR = [local model checkpoint folder] or meta-llama/Meta-Llama-3.1-405B-Instruct
 # single GPU
+python3 quantize_quark.py \
         --model_dir $MODEL_DIR \
+        --output_dir Meta-Llama-3.1-405B-Instruct-FP8-KV \
         --quant_scheme w_fp8_a_fp8 \
         --kv_cache_dtype fp8 \
         --num_calib_data 128 \
         --no_weight_matrix_merge
 # If model size is too large for single GPU, please use multi GPU instead.
+python3 quantize_quark.py \
         --model_dir $MODEL_DIR \
+        --output_dir Meta-Llama-3.1-405B-Instruct-FP8-KV \
         --quant_scheme w_fp8_a_fp8 \
         --kv_cache_dtype fp8 \
         --num_calib_data 128 \