|
{ |
|
"version": "0.9.0.dev2024032600", |
|
"pretrained_config": { |
|
"architecture": "LlamaForCausalLM", |
|
"dtype": "float16", |
|
"logits_dtype": "float32", |
|
"vocab_size": 32000, |
|
"max_position_embeddings": 4096, |
|
"hidden_size": 4096, |
|
"num_hidden_layers": 32, |
|
"num_attention_heads": 32, |
|
"num_key_value_heads": 32, |
|
"head_size": 128, |
|
"hidden_act": "silu", |
|
"intermediate_size": 11008, |
|
"norm_epsilon": 1e-05, |
|
"position_embedding_type": "rope_gpt_neox", |
|
"use_parallel_embedding": true, |
|
"embedding_sharding_dim": 0, |
|
"share_embedding_table": false, |
|
"mapping": { |
|
"world_size": 1, |
|
"tp_size": 1, |
|
"pp_size": 1 |
|
}, |
|
"quantization": { |
|
"quant_algo": "FP8", |
|
"kv_cache_quant_algo": "FP8", |
|
"group_size": 128, |
|
"smoothquant_val": null, |
|
"has_zero_point": false, |
|
"pre_quant_scale": false, |
|
"exclude_modules": null |
|
}, |
|
"kv_dtype": "fp8", |
|
"producer": { |
|
"name": "ammo", |
|
"version": "0.7.4" |
|
}, |
|
"rotary_base": 10000.0, |
|
"mlp_bias": false, |
|
"attn_bias": false, |
|
"rotary_scaling": null, |
|
"moe_num_experts": 0, |
|
"moe_top_k": 0, |
|
"moe_tp_mode": 2, |
|
"moe_normalization_mode": 1 |
|
}, |
|
"build_config": { |
|
"max_input_len": 512, |
|
"max_output_len": 200, |
|
"max_batch_size": 128, |
|
"max_beam_width": 1, |
|
"max_num_tokens": 4096, |
|
"opt_num_tokens": null, |
|
"max_prompt_embedding_table_size": 0, |
|
"gather_context_logits": false, |
|
"gather_generation_logits": false, |
|
"strongly_typed": true, |
|
"builder_opt": null, |
|
"profiling_verbosity": "layer_names_only", |
|
"enable_debug_output": false, |
|
"max_draft_len": 0, |
|
"use_refit": false, |
|
"input_timing_cache": null, |
|
"output_timing_cache": null, |
|
"lora_config": { |
|
"lora_dir": [], |
|
"lora_ckpt_source": "hf", |
|
"max_lora_rank": 64, |
|
"lora_target_modules": [], |
|
"trtllm_modules_to_hf_modules": {} |
|
}, |
|
"auto_parallel_config": { |
|
"world_size": 1, |
|
"gpus_per_node": 8, |
|
"cluster_key": "H100-PCIe", |
|
"cluster_info": null, |
|
"sharding_cost_model": "alpha_beta", |
|
"comm_cost_model": "alpha_beta", |
|
"enable_pipeline_parallelism": false, |
|
"enable_shard_unbalanced_shape": false, |
|
"enable_shard_dynamic_shape": false, |
|
"enable_reduce_scatter": true, |
|
"builder_flags": null, |
|
"debug_mode": false, |
|
"infer_shape": true, |
|
"validation_mode": false, |
|
"same_buffer_io": { |
|
"past_key_value_(\\d+)": "present_key_value_\\1" |
|
}, |
|
"same_spec_io": {}, |
|
"sharded_io_allowlist": [ |
|
"past_key_value_\\d+", |
|
"present_key_value_\\d*" |
|
], |
|
"fast_reduce": true, |
|
"fill_weights": false, |
|
"parallel_config_cache": null, |
|
"profile_cache": null, |
|
"dump_path": null, |
|
"debug_outputs": [] |
|
}, |
|
"weight_sparsity": false, |
|
"use_fused_mlp": false, |
|
"plugin_config": { |
|
"bert_attention_plugin": "float16", |
|
"gpt_attention_plugin": "float16", |
|
"gemm_plugin": null, |
|
"smooth_quant_gemm_plugin": null, |
|
"identity_plugin": null, |
|
"layernorm_quantization_plugin": null, |
|
"rmsnorm_quantization_plugin": null, |
|
"nccl_plugin": null, |
|
"lookup_plugin": null, |
|
"lora_plugin": null, |
|
"weight_only_groupwise_quant_matmul_plugin": null, |
|
"weight_only_quant_matmul_plugin": null, |
|
"quantize_per_token_plugin": false, |
|
"quantize_tensor_plugin": false, |
|
"moe_plugin": "float16", |
|
"mamba_conv1d_plugin": "float16", |
|
"context_fmha": true, |
|
"context_fmha_fp32_acc": false, |
|
"paged_kv_cache": true, |
|
"remove_input_padding": true, |
|
"use_custom_all_reduce": false, |
|
"multi_block_mode": false, |
|
"enable_xqa": true, |
|
"attention_qk_half_accumulation": false, |
|
"tokens_per_block": 128, |
|
"use_paged_context_fmha": false, |
|
"use_fp8_context_fmha": false, |
|
"use_context_fmha_for_generation": false, |
|
"multiple_profiles": false, |
|
"paged_state": true, |
|
"streamingllm": false |
|
} |
|
} |
|
} |