bloom-deepspeed-inference-int8 / ds_inference_config.json
jeffra's picture
remove bloom prefix and update ds config
aa00a66
{
"type": "BLOOM",
"checkpoints": {
"non_tp": [
"non-tp.pt"
],
"tp": [
"tp_00_00.pt",
"tp_01_00.pt",
"tp_02_00.pt",
"tp_03_00.pt",
"tp_00_01.pt",
"tp_01_01.pt",
"tp_02_01.pt",
"tp_03_01.pt",
"tp_00_02.pt",
"tp_01_02.pt",
"tp_02_02.pt",
"tp_03_02.pt",
"tp_00_03.pt",
"tp_01_03.pt",
"tp_02_03.pt",
"tp_03_03.pt",
"tp_00_04.pt",
"tp_01_04.pt",
"tp_02_04.pt",
"tp_03_04.pt",
"tp_00_05.pt",
"tp_01_05.pt",
"tp_02_05.pt",
"tp_03_05.pt",
"tp_00_06.pt",
"tp_01_06.pt",
"tp_02_06.pt",
"tp_03_06.pt",
"tp_00_07.pt",
"tp_01_07.pt",
"tp_02_07.pt",
"tp_03_07.pt"
]
},
"version": 1,
"parallelization": "tp",
"tp_size": 4,
"dtype": "int8"
}