baseten
/

btest-Qwen2.5-0.5B-Instruct-FP8-dynamic-NVIDIA-H100-80GB-HBM3-v0.13.0-TP2

Model card Files Files and versions Community

bdubayah commited on Dec 11, 2024

Commit

1573ef5

·

verified ·

1 Parent(s): 39b7ae0

Upload folder using huggingface_hub

Files changed (3) hide show

config.json +2 -2
rank0.engine +2 -2
rank1.engine +2 -2

config.json CHANGED Viewed

@@ -117,8 +117,8 @@
         "plugin_config": {
             "dtype": "float16",
             "bert_attention_plugin": "auto",
-            "gpt_attention_plugin": "float16",
-            "gemm_plugin": "float16",
             "gemm_swiglu_plugin": null,
             "fp8_rowwise_gemm_plugin": null,
             "smooth_quant_gemm_plugin": null,

         "plugin_config": {
             "dtype": "float16",
             "bert_attention_plugin": "auto",
+            "gpt_attention_plugin": "auto",
+            "gemm_plugin": null,
             "gemm_swiglu_plugin": null,
             "fp8_rowwise_gemm_plugin": null,
             "smooth_quant_gemm_plugin": null,

rank0.engine CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e7f6dbdb0c7158b15a28063b7dc9c6e705eb9c76795b01c7349c055d3cd42b2
-size 780889412

 version https://git-lfs.github.com/spec/v1
+oid sha256:6dee1d8c5f5be88acbbf6492ddae8e1b635edf5a49f66659965232d96bedc9c5
+size 789463676

rank1.engine CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebf1ea318c015ba433a317d6fe6223a19ea4b8b03428ea75783261030e9560e3
-size 780889412

 version https://git-lfs.github.com/spec/v1
+oid sha256:33c524b022857564d6b8846fbb9014a1fa3b687f7bfe39a41a266bb67b72f01d
+size 791781380