File size: 294 Bytes
5d8a119
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
{
  "fp16": {

    "enabled": true,

    "initial_scale_power": 12

  },
  "zero_optimization": {

    "stage": 2,

    "reduce_bucket_size": 5e7,

    "allgather_bucket_size": 1.25e9,

    "overlap_comm": true,

    "contiguous_gradients": true

  },
  "zero_allow_untested_optimizer": true
}