Commit
•
6867ec4
1
Parent(s):
e9e63fb
Add
Browse files- sbatch_2b855brealtasky.sh +165 -0
sbatch_2b855brealtasky.sh
ADDED
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
|
3 |
+
#SBATCH --nodes=32
|
4 |
+
#SBATCH --ntasks-per-node=1
|
5 |
+
#SBATCH --cpus-per-task=32
|
6 |
+
#SBATCH --mem=256G
|
7 |
+
#SBATCH -p standard-g
|
8 |
+
#SBATCH -t 48:00:00
|
9 |
+
#SBATCH --gpus-per-node=mi250:8
|
10 |
+
#SBATCH --exclusive=user
|
11 |
+
#SBATCH --hint=nomultithread
|
12 |
+
#SBATCH --account=project_462000119
|
13 |
+
#SBATCH -o logs/%j.out
|
14 |
+
#SBATCH -e logs/%j.err
|
15 |
+
|
16 |
+
VARIANT=2b855brealtasky
|
17 |
+
|
18 |
+
# if run without sbatch, invoke here
|
19 |
+
if [ -z $SLURM_JOB_ID ]; then
|
20 |
+
mkdir -p logs
|
21 |
+
sbatch "$0"
|
22 |
+
exit
|
23 |
+
fi
|
24 |
+
|
25 |
+
set -euo pipefail
|
26 |
+
|
27 |
+
# symlink logs/latest.out and logs/latest.err
|
28 |
+
ln -f -s $SLURM_JOB_ID.out logs/latest.out
|
29 |
+
ln -f -s $SLURM_JOB_ID.err logs/latest.err
|
30 |
+
|
31 |
+
KILL_SWITCH_PATH=kill-switch-$VARIANT
|
32 |
+
CHECKPOINT_PATH=checkpoints_$VARIANT
|
33 |
+
TENSORBOARD_PATH=tensorboard_$VARIANT
|
34 |
+
|
35 |
+
# Data
|
36 |
+
VOCAB_FILE="gpt2/vocab.json"
|
37 |
+
MERGE_FILE="gpt2/merges.txt"
|
38 |
+
#TRAIN_DATA_PATH="/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_10B_text_document"
|
39 |
+
#VAL_DATA_PATH="/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
|
40 |
+
|
41 |
+
TRAIN_DATA_PATH=trainrealtasky.txt
|
42 |
+
# "train: 1.0 0:1 /scratch/project_462000119/data/realtasky/gpt2tok_realtasky_text_document"
|
43 |
+
VALID_DATA_PATH=valc4oscar.txt
|
44 |
+
# "validation_c4: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" "validation_oscar: 1.0 0:1 /scratch/project_462000119/data/oscar_validation/gpt2tok_oscarvalidation_text_document"
|
45 |
+
|
46 |
+
PP_SIZE=1
|
47 |
+
TP_SIZE=1
|
48 |
+
|
49 |
+
MICRO_BATCH_SIZE=2
|
50 |
+
GRADIENT_ACCUMULATION_STEPS=1
|
51 |
+
WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
|
52 |
+
GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
|
53 |
+
|
54 |
+
# Model parameters
|
55 |
+
source model_params.sh
|
56 |
+
MODEL_PARAM=("${PARAM_2980M[@]}")
|
57 |
+
NHIDDEN=${MODEL_PARAM[0]}
|
58 |
+
FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
|
59 |
+
KV_SIZE=${MODEL_PARAM[2]}
|
60 |
+
NHEADS=${MODEL_PARAM[3]}
|
61 |
+
NLAYERS=${MODEL_PARAM[4]}
|
62 |
+
SEQ_LEN=2048
|
63 |
+
|
64 |
+
echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
|
65 |
+
|
66 |
+
SAVE_INTERVAL=10000
|
67 |
+
|
68 |
+
# Tokens: 55000000000
|
69 |
+
# -> Samples: 26855469
|
70 |
+
TRAIN_SAMPLES=26_855_469
|
71 |
+
|
72 |
+
OPTIMIZER_ARGS=" \
|
73 |
+
--optimizer adam \
|
74 |
+
--adam-beta1 0.9 \
|
75 |
+
--adam-beta2 0.95 \
|
76 |
+
--adam-eps 1e-8 \
|
77 |
+
--lr 2e-4 \
|
78 |
+
--min-lr 2e-5 \
|
79 |
+
--lr-decay-style cosine \
|
80 |
+
--lr-decay-samples $TRAIN_SAMPLES \
|
81 |
+
--lr-warmup-samples 268_555 \
|
82 |
+
--clip-grad 1.0 \
|
83 |
+
--weight-decay 1e-1 \
|
84 |
+
"
|
85 |
+
|
86 |
+
GPT_ARGS=" \
|
87 |
+
--num-layers $NLAYERS \
|
88 |
+
--hidden-size $NHIDDEN \
|
89 |
+
--num-attention-heads $NHEADS \
|
90 |
+
--kv-channels $KV_SIZE \
|
91 |
+
--ffn-hidden-size $FFN_HIDDEN_SIZE \
|
92 |
+
--seq-length $SEQ_LEN \
|
93 |
+
--max-position-embeddings $SEQ_LEN \
|
94 |
+
--micro-batch-size $MICRO_BATCH_SIZE \
|
95 |
+
--global-batch-size $GLOBAL_BATCH_SIZE \
|
96 |
+
--train-samples $TRAIN_SAMPLES \
|
97 |
+
--vocab-file $VOCAB_FILE \
|
98 |
+
--merge-file $MERGE_FILE \
|
99 |
+
--clip-grad 1.0 \
|
100 |
+
--kill-switch-path $KILL_SWITCH_PATH \
|
101 |
+
--bf16 \
|
102 |
+
$OPTIMIZER_ARGS \
|
103 |
+
"
|
104 |
+
|
105 |
+
OUTPUT_ARGS=" \
|
106 |
+
--log-interval 10 \
|
107 |
+
--save-interval $SAVE_INTERVAL \
|
108 |
+
--eval-interval 100 \
|
109 |
+
--eval-iters 100 \
|
110 |
+
--tensorboard-dir $TENSORBOARD_PATH \
|
111 |
+
--tensorboard-queue-size 5 \
|
112 |
+
--log-timers-to-tensorboard \
|
113 |
+
--log-batch-size-to-tensorboard \
|
114 |
+
--log-validation-ppl-to-tensorboard \
|
115 |
+
"
|
116 |
+
|
117 |
+
ZERO_STAGE=0
|
118 |
+
|
119 |
+
mkdir -p ds_configs
|
120 |
+
DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
|
121 |
+
|
122 |
+
cat <<EOF > $DS_CONFIG_PATH
|
123 |
+
{
|
124 |
+
"train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
|
125 |
+
"train_batch_size": $GLOBAL_BATCH_SIZE,
|
126 |
+
"gradient_clipping": 1.0,
|
127 |
+
"zero_optimization": {
|
128 |
+
"stage": $ZERO_STAGE
|
129 |
+
},
|
130 |
+
"bf16": {
|
131 |
+
"enabled": true
|
132 |
+
},
|
133 |
+
"steps_per_print": 2000,
|
134 |
+
"wall_clock_breakdown": false
|
135 |
+
}
|
136 |
+
EOF
|
137 |
+
|
138 |
+
DEEPSPEED_ARGS=" \
|
139 |
+
--deepspeed \
|
140 |
+
--deepspeed_config $DS_CONFIG_PATH \
|
141 |
+
--zero-stage $ZERO_STAGE \
|
142 |
+
"
|
143 |
+
|
144 |
+
CMD=" \
|
145 |
+
Megatron-DeepSpeed/pretrain_gpt.py \
|
146 |
+
--tensor-model-parallel-size $TP_SIZE \
|
147 |
+
--pipeline-model-parallel-size $PP_SIZE \
|
148 |
+
$GPT_ARGS \
|
149 |
+
$OUTPUT_ARGS \
|
150 |
+
--save $CHECKPOINT_PATH \
|
151 |
+
--load $CHECKPOINT_PATH \
|
152 |
+
--train-weighted-split-paths-path $TRAIN_DATA_PATH \
|
153 |
+
--valid-weighted-split-paths-path $VALID_DATA_PATH \
|
154 |
+
--data-impl mmap \
|
155 |
+
$DEEPSPEED_ARGS \
|
156 |
+
"
|
157 |
+
|
158 |
+
echo $CMD
|
159 |
+
|
160 |
+
echo "START $SLURM_JOBID: $(date)"
|
161 |
+
|
162 |
+
# bash launch_srun.sh $CMD
|
163 |
+
srun --label launch.sh $CMD
|
164 |
+
|
165 |
+
echo "END $SLURM_JOBID: $(date)"
|