Upload pythia-19Mnew.yml
Browse files- pythia-19Mnew.yml +106 -0
pythia-19Mnew.yml
ADDED
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"pipe-parallel-size": 1,
|
3 |
+
"model-parallel-size": 1,
|
4 |
+
|
5 |
+
# model settings
|
6 |
+
"num-layers": 6,
|
7 |
+
"hidden-size": 512,
|
8 |
+
"num-attention-heads": 8,
|
9 |
+
"seq-length": 2048,
|
10 |
+
"max-position-embeddings": 2048,
|
11 |
+
"pos-emb": "rotary",
|
12 |
+
"rotary-pct": 0.25,
|
13 |
+
"no-weight-tying": true,
|
14 |
+
"gpt-j-residual": true,
|
15 |
+
"output-layer-parallelism": "column",
|
16 |
+
|
17 |
+
"scaled-upper-triang-masked-softmax-fusion": true,
|
18 |
+
"bias-gelu-fusion": true,
|
19 |
+
|
20 |
+
# init methods
|
21 |
+
"init_method": "small_init",
|
22 |
+
"output_layer_init_method": "wang_init",
|
23 |
+
|
24 |
+
"optimizer": {
|
25 |
+
"type": "Adam",
|
26 |
+
"params": {
|
27 |
+
"lr": 0.001,
|
28 |
+
"betas": [0.9, 0.95],
|
29 |
+
"eps": 1.0e-8,
|
30 |
+
}
|
31 |
+
},
|
32 |
+
"min_lr": 0.0001,
|
33 |
+
|
34 |
+
"zero_optimization": {
|
35 |
+
"stage": 1,
|
36 |
+
"allgather_partitions": True,
|
37 |
+
"allgather_bucket_size": 500000000,
|
38 |
+
"overlap_comm": True,
|
39 |
+
"reduce_scatter": True,
|
40 |
+
"reduce_bucket_size": 500000000,
|
41 |
+
"contiguous_gradients": True,
|
42 |
+
"cpu_offload": False
|
43 |
+
},
|
44 |
+
|
45 |
+
"train_micro_batch_size_per_gpu": 32,
|
46 |
+
"gas": 1,
|
47 |
+
"data-impl": "mmap",
|
48 |
+
"num_workers": 1,
|
49 |
+
|
50 |
+
# activation checkpointing
|
51 |
+
"checkpoint-activations": true,
|
52 |
+
"checkpoint-num-layers": 1,
|
53 |
+
"partition-activations": true,
|
54 |
+
"synchronize-each-layer": true,
|
55 |
+
|
56 |
+
# regularization
|
57 |
+
"gradient_clipping": 1.0,
|
58 |
+
"weight-decay": 0.1,
|
59 |
+
"hidden-dropout": 0,
|
60 |
+
"attention-dropout": 0,
|
61 |
+
|
62 |
+
# precision settings
|
63 |
+
"fp16": {
|
64 |
+
"fp16": true,
|
65 |
+
"enabled": true,
|
66 |
+
"loss_scale": 0,
|
67 |
+
"loss_scale_window": 1000,
|
68 |
+
"initial_scale_power": 12,
|
69 |
+
"hysteresis": 2,
|
70 |
+
"min_loss_scale": 1,
|
71 |
+
},
|
72 |
+
|
73 |
+
"train-iters": 143000,
|
74 |
+
"lr-decay-iters": 143000,
|
75 |
+
"distributed-backend": "nccl",
|
76 |
+
"lr-decay-style": "cosine",
|
77 |
+
"warmup": 0.01,
|
78 |
+
"save-interval": 1000,
|
79 |
+
"eval-interval": 100000,
|
80 |
+
"eval-iters": 10,
|
81 |
+
|
82 |
+
"log-interval": 10,
|
83 |
+
"steps_per_print": 10,
|
84 |
+
"wall_clock_breakdown": true,
|
85 |
+
|
86 |
+
"save": "/fsx/hailey/pythia/ckpts/19M",
|
87 |
+
"load": "/fsx/hailey/pythia/ckpts/19M",
|
88 |
+
|
89 |
+
"train-data-paths": ["/fsx/pile/pile_20B_tokenizer_text_document"],
|
90 |
+
"valid-data-paths": ["/fsx/pile/pile_20B_tokenizer_text_document"],
|
91 |
+
"test-data-paths": ["/fsx/pile/pile_20B_tokenizer_text_document"],
|
92 |
+
|
93 |
+
"tokenizer-type": "HFTokenizer",
|
94 |
+
"vocab-file": "/fsx/pile/20B_tokenizer.json",
|
95 |
+
|
96 |
+
"tensorboard-dir": "/fsx/code-fim/FIMlogs/1.3B-AR-Pile-9-6-22-rotary-1MtokBS",
|
97 |
+
"log-dir": "/fsx/code-fim/FIMlogs/1.3B-AR-Pile-9-6-22-rotary-1MtokBS",
|
98 |
+
|
99 |
+
"use_wandb": true,
|
100 |
+
"wandb_group": "Pythia 19M new",
|
101 |
+
"wandb_team": "eleutherai",
|
102 |
+
"wandb_project": "pythia",
|
103 |
+
|
104 |
+
"launcher": "openmpi",
|
105 |
+
"deepspeed_mpi": true,
|
106 |
+
}
|