Hugging Face's logo

pankajroark
/
llama-fp16-engine

Model card Files Community
pankajroark commited on
Commit
a3806a1
•
1 Parent(s): 4b63086

tp8 checkpoint

Browse files
Files changed (10) hide show
  1. 7b-sq-int8kv-tp8/config.json +47 -0
  2. 7b-sq-int8kv-tp8/llama_float16_tp8_rank0.engine +3 -0
  3. 7b-sq-int8kv-tp8/llama_float16_tp8_rank1.engine +3 -0
  4. 7b-sq-int8kv-tp8/llama_float16_tp8_rank2.engine +3 -0
  5. 7b-sq-int8kv-tp8/llama_float16_tp8_rank3.engine +3 -0
  6. 7b-sq-int8kv-tp8/llama_float16_tp8_rank4.engine +3 -0
  7. 7b-sq-int8kv-tp8/llama_float16_tp8_rank5.engine +3 -0
  8. 7b-sq-int8kv-tp8/llama_float16_tp8_rank6.engine +3 -0
  9. 7b-sq-int8kv-tp8/llama_float16_tp8_rank7.engine +3 -0
  10. 7b-sq-int8kv-tp8/model.cache +0 -0
7b-sq-int8kv-tp8/config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "builder_config": {
3
+ "fp8": false,
4
+ "hidden_act": "silu",
5
+ "hidden_size": 4096,
6
+ "int8": true,
7
+ "max_batch_size": 8,
8
+ "max_input_len": 2048,
9
+ "max_num_tokens": null,
10
+ "max_output_len": 512,
11
+ "max_position_embeddings": 4096,
12
+ "name": "llama",
13
+ "num_heads": 32,
14
+ "num_kv_heads": 32,
15
+ "num_layers": 32,
16
+ "parallel_build": false,
17
+ "pipeline_parallel": 1,
18
+ "precision": "float16",
19
+ "quant_mode": 70,
20
+ "tensor_parallel": 8,
21
+ "use_refit": false,
22
+ "vocab_size": 32000
23
+ },
24
+ "plugin_config": {
25
+ "attention_qk_half_accumulation": false,
26
+ "bert_attention_plugin": false,
27
+ "context_fmha_type": 0,
28
+ "gemm_plugin": "float16",
29
+ "gpt_attention_plugin": "float16",
30
+ "identity_plugin": false,
31
+ "layernorm_plugin": false,
32
+ "layernorm_quantization_plugin": false,
33
+ "lookup_plugin": false,
34
+ "nccl_plugin": "float16",
35
+ "paged_kv_cache": false,
36
+ "quantize_per_token_plugin": true,
37
+ "quantize_tensor_plugin": true,
38
+ "remove_input_padding": true,
39
+ "rmsnorm_plugin": false,
40
+ "rmsnorm_quantization_plugin": "float16",
41
+ "smooth_quant_gemm_plugin": "float16",
42
+ "tokens_per_block": 0,
43
+ "use_custom_all_reduce": false,
44
+ "weight_only_groupwise_quant_matmul_plugin": false,
45
+ "weight_only_quant_matmul_plugin": false
46
+ }
47
+ }
7b-sq-int8kv-tp8/llama_float16_tp8_rank0.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1eec20d48d3fa61b676fd5dd6dd39b39931ea0956db6bb28f71b386a1a72670
3
+ size 1108282076
7b-sq-int8kv-tp8/llama_float16_tp8_rank1.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:121590a074e2d5e61e252bf9281d85c299da2f4226509612cdb44ca9b90929c1
3
+ size 1108282076
7b-sq-int8kv-tp8/llama_float16_tp8_rank2.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1db11b56bb9b713287505eddab7380a03e8e82e4e2ba34f1c5cc62090468f710
3
+ size 1108282076
7b-sq-int8kv-tp8/llama_float16_tp8_rank3.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c901a4f3e355466a291a732a389a9221ee449933f09a0ae9644c826b04bb97ab
3
+ size 1108282076
7b-sq-int8kv-tp8/llama_float16_tp8_rank4.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b24d6382176e57822dc853d3934c54f6247c8be7ed83a9e55cd8cb8b8d8c6b07
3
+ size 1108282076
7b-sq-int8kv-tp8/llama_float16_tp8_rank5.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd3c19a606e6f1f16c7528ec2d66a96047e71fa262d5ecb7c3d5714ab9507ca1
3
+ size 1108282076
7b-sq-int8kv-tp8/llama_float16_tp8_rank6.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd795efc090c7f2291ea3f9243c1814ef7f07e2c8532db9ef632de5ca5a61c93
3
+ size 1108282076
7b-sq-int8kv-tp8/llama_float16_tp8_rank7.engine ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e810c7fe2acbbde973841906db254c6fa738d3f07534a947a53d209efefda623
3
+ size 1108282076
7b-sq-int8kv-tp8/model.cache ADDED
Binary file (109 kB). View file