Commit
•
b757584
1
Parent(s):
79391b8
Add
Browse files- eval_all_gen.sh +357 -0
- launch.sh +53 -0
- rank_eval_all.sh +253 -0
- rank_eval_all_prefix_ul2.sh +258 -0
- run_eval_lumi_gen.sh +73 -0
- run_eval_lumi_xp3eval.sh +162 -0
eval_all_gen.sh
ADDED
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
#SBATCH --exclude=nid005159
|
3 |
+
#SBATCH --nodes=1
|
4 |
+
#SBATCH --ntasks-per-node=1
|
5 |
+
#SBATCH --cpus-per-task=4
|
6 |
+
#SBATCH --mem=256G
|
7 |
+
#SBATCH -p small-g
|
8 |
+
#SBATCH -t 2-0:00:00
|
9 |
+
#SBATCH --gpus-per-node=mi250:0
|
10 |
+
#SBATCH --exclusive=user
|
11 |
+
#SBATCH --hint=nomultithread
|
12 |
+
#SBATCH --account=project_462000119
|
13 |
+
#SBATCH -o logs/%j.out
|
14 |
+
#SBATCH -e logs/%j.err
|
15 |
+
|
16 |
+
# if run without sbatch, invoke here
|
17 |
+
if [ -z $SLURM_JOB_ID ]; then
|
18 |
+
mkdir -p logs
|
19 |
+
sbatch "$0"
|
20 |
+
exit
|
21 |
+
fi
|
22 |
+
|
23 |
+
set -euo pipefail
|
24 |
+
|
25 |
+
# symlink logs/latest_eval.out and logs/latest_eval.err
|
26 |
+
ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
|
27 |
+
ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
|
28 |
+
|
29 |
+
source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
|
30 |
+
|
31 |
+
echo "START TIME: $(date)"
|
32 |
+
|
33 |
+
# defining the right environment variables
|
34 |
+
export HF_DATASETS_OFFLINE=1
|
35 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
36 |
+
|
37 |
+
# Converted transformer checkpoint
|
38 |
+
# cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
|
39 |
+
|
40 |
+
# Data
|
41 |
+
CONFIGS=(
|
42 |
+
copa,"best_option",validation
|
43 |
+
copa,"choose",validation
|
44 |
+
copa,"i_am_hesitating",validation
|
45 |
+
copa,"cause_effect",validation
|
46 |
+
copa,"plausible_alternatives",validation
|
47 |
+
superglue_rte,"MNLI crowdsource",validation
|
48 |
+
superglue_rte,"GPT-3 style",validation
|
49 |
+
superglue_rte,"does it follow that",validation
|
50 |
+
superglue_rte,"should assume",validation
|
51 |
+
superglue_rte,"guaranteed true",validation
|
52 |
+
anli_r1,"guaranteed/possible/impossible",dev_r1
|
53 |
+
anli_r1,"MNLI crowdsource",dev_r1
|
54 |
+
anli_r1,"GPT-3 style",dev_r1
|
55 |
+
anli_r1,"justified in saying",dev_r1
|
56 |
+
anli_r1,"can we infer",dev_r1
|
57 |
+
anli_r2,"guaranteed/possible/impossible",dev_r2
|
58 |
+
anli_r2,"MNLI crowdsource",dev_r2
|
59 |
+
anli_r2,"GPT-3 style",dev_r2
|
60 |
+
anli_r2,"justified in saying",dev_r2
|
61 |
+
anli_r2,"can we infer",dev_r2
|
62 |
+
anli_r3,"guaranteed/possible/impossible",dev_r3
|
63 |
+
anli_r3,"MNLI crowdsource",dev_r3
|
64 |
+
anli_r3,"GPT-3 style",dev_r3
|
65 |
+
anli_r3,"justified in saying",dev_r3
|
66 |
+
anli_r3,"can we infer",dev_r3
|
67 |
+
cb,"guaranteed/possible/impossible",validation
|
68 |
+
cb,"MNLI crowdsource",validation
|
69 |
+
cb,"GPT-3 style",validation
|
70 |
+
cb,"justified in saying",validation
|
71 |
+
cb,"can we infer",validation
|
72 |
+
winogrande,"underscore refer to",validation
|
73 |
+
winogrande,"Replace",validation
|
74 |
+
winogrande,"stand for",validation
|
75 |
+
winogrande,"does underscore refer to",validation
|
76 |
+
winogrande,"True or False",validation
|
77 |
+
story_cloze_2016,"Story Continuation and Options",validation
|
78 |
+
story_cloze_2016,"Answer Given options",validation
|
79 |
+
story_cloze_2016,"Novel Correct Ending",validation
|
80 |
+
story_cloze_2016,"Generate Ending",validation
|
81 |
+
story_cloze_2016,"Choose Story Ending",validation
|
82 |
+
boolq,"after_reading",validation
|
83 |
+
boolq,"GPT-3 Style",validation
|
84 |
+
boolq,"yes_no_question",validation
|
85 |
+
boolq,"exercise",validation
|
86 |
+
boolq,"valid_binary",validation
|
87 |
+
arc_easy,"pick_the_most_correct_option",test
|
88 |
+
arc_easy,"qa_options",test
|
89 |
+
arc_easy,"i_am_hesitating",test
|
90 |
+
arc_easy,"multiple_choice",test
|
91 |
+
arc_easy,"heres_a_problem",test
|
92 |
+
arc_challenge,"pick_the_most_correct_option",test
|
93 |
+
arc_challenge,"qa_options",test
|
94 |
+
arc_challenge,"i_am_hesitating",test
|
95 |
+
arc_challenge,"multiple_choice",test
|
96 |
+
arc_challenge,"heres_a_problem",test
|
97 |
+
sciq,"Direct Question (Closed Book)",test
|
98 |
+
sciq,"Multiple Choice (Closed Book)",test
|
99 |
+
sciq,"Multiple Choice Question First",test
|
100 |
+
sciq,"Multiple Choice",test
|
101 |
+
sciq,"Direct Question",test
|
102 |
+
piqa,"what_is_the_correct_ending"
|
103 |
+
piqa,"pick_correct_choice_index"
|
104 |
+
piqa,"Correct the solution"
|
105 |
+
piqa,"choose the most appropriate solution"
|
106 |
+
piqa,"no prompt needed"
|
107 |
+
GEM/wiki_lingua_en,"tldr_en"
|
108 |
+
GEM/wiki_lingua_en,"article_summary_en"
|
109 |
+
GEM/wiki_lingua_en,"summarize_above_en"
|
110 |
+
GEM/wiki_lingua_en,"rephrase_en"
|
111 |
+
GEM/wiki_lingua_en,"write_abstract_en"
|
112 |
+
gem_xsum,"article_DOC_summary"
|
113 |
+
gem_xsum,"summarize_DOC"
|
114 |
+
gem_xsum,"summarize_this_DOC_summary"
|
115 |
+
gem_xsum,"DOC_tldr"
|
116 |
+
gem_xsum,"DOC_boils_down_to_simple_idea_that"
|
117 |
+
GEM/web_nlg_en,"PALM_prompt"
|
118 |
+
GEM/web_nlg_en,"explicit-graph-description2"
|
119 |
+
GEM/web_nlg_en,"non-explicit-description"
|
120 |
+
GEM/web_nlg_en,"very-explicit-description"
|
121 |
+
GEM/web_nlg_en,"implicit-graph-description"
|
122 |
+
e2e_nlg_cleaned,"generate_text_restaurant"
|
123 |
+
e2e_nlg_cleaned,"coherent_text"
|
124 |
+
e2e_nlg_cleaned,"create_text_for_me"
|
125 |
+
e2e_nlg_cleaned,"generate_gramatically_correct_text"
|
126 |
+
e2e_nlg_cleaned,"text"
|
127 |
+
)
|
128 |
+
|
129 |
+
CONFIGS=(
|
130 |
+
GEM/wiki_lingua_en,"tldr_en"
|
131 |
+
gem_xsum,"article_DOC_summary"
|
132 |
+
GEM/web_nlg_en,"PALM_prompt"
|
133 |
+
e2e_nlg_cleaned,"generate_text_restaurant"
|
134 |
+
)
|
135 |
+
|
136 |
+
CKPTS=(
|
137 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b12bc4/transformers
|
138 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/transformers
|
139 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
|
140 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
|
141 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
|
142 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
|
143 |
+
)
|
144 |
+
|
145 |
+
CKPTS=(
|
146 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/transformers
|
147 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
|
148 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
|
149 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
|
150 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
|
151 |
+
)
|
152 |
+
|
153 |
+
CKPTS=(
|
154 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
|
155 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
|
156 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
|
157 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
|
158 |
+
)
|
159 |
+
|
160 |
+
CKPTS=(
|
161 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b12boscar/transformers
|
162 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/transformers
|
163 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
|
164 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
|
165 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
|
166 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
|
167 |
+
)
|
168 |
+
|
169 |
+
CKPTS=(
|
170 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/transformers
|
171 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
|
172 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
|
173 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
|
174 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
|
175 |
+
)
|
176 |
+
|
177 |
+
CKPTS=(
|
178 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/transformers
|
179 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/transformers
|
180 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/transformers
|
181 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/transformers
|
182 |
+
)
|
183 |
+
|
184 |
+
CKPTS=(
|
185 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
|
186 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
|
187 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
|
188 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
|
189 |
+
)
|
190 |
+
|
191 |
+
CKPTS=(
|
192 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
|
193 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
|
194 |
+
)
|
195 |
+
|
196 |
+
CKPTS=(
|
197 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b1b25oscar/transformers
|
198 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b4boscar/transformers
|
199 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/transformers
|
200 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b11boscar/transformers
|
201 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b14boscar/transformers
|
202 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18boscar/transformers
|
203 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b28boscar/transformers
|
204 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b55boscar/transformers
|
205 |
+
)
|
206 |
+
|
207 |
+
CKPTS=(
|
208 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/transformers
|
209 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/transformers
|
210 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/transformers
|
211 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/transformers
|
212 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
|
213 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/transformers
|
214 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/transformers
|
215 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/transformers
|
216 |
+
)
|
217 |
+
|
218 |
+
CKPTSX=(
|
219 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b25b/transformers
|
220 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b35b/transformers
|
221 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b44b/transformers
|
222 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b58b/transformers
|
223 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b88b/transformers
|
224 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b44b/transformers
|
225 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b88b/transformers
|
226 |
+
)
|
227 |
+
|
228 |
+
|
229 |
+
CKPTSX=(
|
230 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
|
231 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
|
232 |
+
)
|
233 |
+
|
234 |
+
CKPTSX=(
|
235 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/transformers
|
236 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/transformers
|
237 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/transformers
|
238 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/transformers
|
239 |
+
)
|
240 |
+
|
241 |
+
CKPTSX=(
|
242 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed1/transformers
|
243 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed2/transformers
|
244 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed3/transformers
|
245 |
+
)
|
246 |
+
CKPTS=(
|
247 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
|
248 |
+
)
|
249 |
+
|
250 |
+
|
251 |
+
CKPTS=(
|
252 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/transformers
|
253 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/transformers
|
254 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/transformers
|
255 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/transformers
|
256 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
|
257 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/transformers
|
258 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/transformers
|
259 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/transformers
|
260 |
+
)
|
261 |
+
|
262 |
+
CKPTS=(
|
263 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm3-2b8-55b-c4/transformers
|
264 |
+
)
|
265 |
+
|
266 |
+
FEWSHOT_CONFIGS=(
|
267 |
+
0
|
268 |
+
1
|
269 |
+
2
|
270 |
+
3
|
271 |
+
4
|
272 |
+
5
|
273 |
+
)
|
274 |
+
|
275 |
+
TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/gpt2
|
276 |
+
|
277 |
+
|
278 |
+
# Iterate through all possible combinations of data config, model ckpt & fewshot config and run the jobs
|
279 |
+
for ((i=0; i<${#CKPTS[@]}; i++)); do
|
280 |
+
for ((j=0; j<${#FEWSHOT_CONFIGS[@]}; j++)); do
|
281 |
+
for ((k=0; k<${#CONFIGS[@]}; k++)); do
|
282 |
+
#echo "sbatch --export=CKPT=${CKPTS[$i]},FEWSHOT_CONFIG=${FEWSHOT_CONFIGS[$j]},DATASET=${DATASETS[$k]} eval.sh"
|
283 |
+
|
284 |
+
DATA_CONFIG=${CONFIGS[$k]}
|
285 |
+
IFS=',' read dataset_name template_name x <<< "${DATA_CONFIG}"
|
286 |
+
MODEL_CKPT=${CKPTS[$i]}
|
287 |
+
MODEL_CKPT_NO_TRF=${MODEL_CKPT%/*}
|
288 |
+
MODEL_NAME=${MODEL_CKPT_NO_TRF##*/}
|
289 |
+
OUTPUT_PATH=$MODEL_CKPT_NO_TRF/evaluation/generation_sss_denoiser
|
290 |
+
mkdir -p $OUTPUT_PATH
|
291 |
+
OUTPUT_NAME=$MODEL_NAME\_$dataset_name\_$template_name\_${FEWSHOT_CONFIGS[$j]}
|
292 |
+
|
293 |
+
eval_script="./eval_$i-$j-$k.slurm"
|
294 |
+
cat <<EOT > $eval_script
|
295 |
+
#!/bin/bash
|
296 |
+
#SBATCH --exclude=nid005159
|
297 |
+
#SBATCH --nodes=1
|
298 |
+
#SBATCH --ntasks-per-node=1
|
299 |
+
#SBATCH --cpus-per-task=8
|
300 |
+
#SBATCH --mem=256G
|
301 |
+
#SBATCH -p small-g
|
302 |
+
#SBATCH -t 2-0:00:00
|
303 |
+
#SBATCH --gpus-per-node=mi250:1
|
304 |
+
#SBATCH --exclusive=user
|
305 |
+
#SBATCH --hint=nomultithread
|
306 |
+
#SBATCH --account=project_462000119
|
307 |
+
#SBATCH -o logs/%j.out
|
308 |
+
#SBATCH -e logs/%j.err
|
309 |
+
set -euo pipefail
|
310 |
+
# symlink logs/latest_eval.out and logs/latest_eval.err
|
311 |
+
ln -f -s "\$SLURM_JOB_ID.out" logs/latest_eval.out
|
312 |
+
ln -f -s "\$SLURM_JOB_ID.err" logs/latest_eval.err
|
313 |
+
source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
|
314 |
+
echo "START TIME: $(date)"
|
315 |
+
# defining the right environment variables
|
316 |
+
export HF_DATASETS_OFFLINE=1
|
317 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
318 |
+
# Converted transformer checkpoint
|
319 |
+
cd /pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/bigscience/lm-evaluation-harness
|
320 |
+
python main.py \
|
321 |
+
--model_api_name 'hf-causal' \
|
322 |
+
--model_args pretrained=${CKPTS[$i]},use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
|
323 |
+
--device cuda \
|
324 |
+
--batch_size 16 \
|
325 |
+
--no_tracking \
|
326 |
+
--task_name $dataset_name \
|
327 |
+
--template_names "$template_name" \
|
328 |
+
--bootstrap_iters 10 \
|
329 |
+
--limit 3000 \
|
330 |
+
--num_fewshot ${FEWSHOT_CONFIGS[$j]} \
|
331 |
+
--output_dir $OUTPUT_PATH \
|
332 |
+
--output_path "$OUTPUT_NAME"
|
333 |
+
python main.py \
|
334 |
+
--model_api_name 'hf-causal' \
|
335 |
+
--model_args pretrained=${CKPTS[$i]},use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
|
336 |
+
--device cuda \
|
337 |
+
--batch_size 8 \
|
338 |
+
--no_tracking \
|
339 |
+
--task_name $dataset_name \
|
340 |
+
--template_names "$template_name" \
|
341 |
+
--bootstrap_iters 10 \
|
342 |
+
--limit 3000 \
|
343 |
+
--num_fewshot ${FEWSHOT_CONFIGS[$j]} \
|
344 |
+
--output_dir $OUTPUT_PATH \
|
345 |
+
--output_path "$OUTPUT_NAME"
|
346 |
+
echo "END TIME: $(date)"
|
347 |
+
EOT
|
348 |
+
# Submit the job
|
349 |
+
sbatch $eval_script
|
350 |
+
# Sleep for a bit to avoid hitting the job submission limit
|
351 |
+
sleep 0.1
|
352 |
+
done
|
353 |
+
done
|
354 |
+
done
|
355 |
+
|
356 |
+
|
357 |
+
echo "END TIME: $(date)"
|
launch.sh
ADDED
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
|
3 |
+
# Launch script using torch.distributed.run(). Used by slurm
|
4 |
+
# scripts, don't invoke directly.
|
5 |
+
|
6 |
+
# Samuel's fix for apparent error in SLURM initialization
|
7 |
+
if [ $SLURM_LOCALID -eq 0 ]; then
|
8 |
+
rm -rf /dev/shm/*
|
9 |
+
rocm-smi || true
|
10 |
+
else
|
11 |
+
sleep 2
|
12 |
+
fi
|
13 |
+
|
14 |
+
export NCCL_SOCKET_IFNAME=hsn0,hsn1,hsn2,hsn3
|
15 |
+
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
|
16 |
+
export FI_CXI_DEFAULT_CQ_SIZE=131072
|
17 |
+
|
18 |
+
# debugging (noisy)
|
19 |
+
#export NCCL_DEBUG=INFO
|
20 |
+
#export RCCL_KERNEL_COLL_TRACE_ENABLE=1
|
21 |
+
#export NCCL_DEBUG_SUBSYS=INIT,COLL
|
22 |
+
|
23 |
+
module --quiet purge
|
24 |
+
module load cray-python
|
25 |
+
|
26 |
+
module load CrayEnv
|
27 |
+
module load PrgEnv-cray/8.3.3
|
28 |
+
module load craype-accel-amd-gfx90a
|
29 |
+
module load cray-python
|
30 |
+
|
31 |
+
module use /pfs/lustrep2/projappl/project_462000125/samantao-public/mymodules
|
32 |
+
module load suse-repo-deps/sam-default
|
33 |
+
module load rocm/sam-5.2.3.lua
|
34 |
+
module load rccl/sam-develop.lua
|
35 |
+
module load aws-ofi-rccl/sam-default.lua
|
36 |
+
|
37 |
+
source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
|
38 |
+
|
39 |
+
MASTER_NODE=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
|
40 |
+
MASTER_PORT=9999
|
41 |
+
|
42 |
+
echo "Launching on $SLURMD_NODENAME ($SLURM_PROCID/$SLURM_JOB_NUM_NODES)," \
|
43 |
+
"master $MASTER_NODE port $MASTER_PORT," \
|
44 |
+
"GPUs $SLURM_GPUS_ON_NODE," \
|
45 |
+
"CUDA: $(python -c 'import torch; print(torch.cuda.is_available())')"
|
46 |
+
|
47 |
+
python -u -m torch.distributed.run \
|
48 |
+
--nnodes $SLURM_JOB_NUM_NODES \
|
49 |
+
--nproc_per_node $SLURM_GPUS_ON_NODE \
|
50 |
+
--node_rank=$SLURM_PROCID \
|
51 |
+
--master_addr $MASTER_NODE \
|
52 |
+
--master_port $MASTER_PORT \
|
53 |
+
"$@"
|
rank_eval_all.sh
ADDED
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
#SBATCH --exclude=nid005159
|
3 |
+
#SBATCH --nodes=1
|
4 |
+
#SBATCH --ntasks-per-node=1
|
5 |
+
#SBATCH --cpus-per-task=4
|
6 |
+
#SBATCH --mem=256G
|
7 |
+
#SBATCH -p small-g
|
8 |
+
#SBATCH -t 2-0:00:00
|
9 |
+
#SBATCH --gpus-per-node=mi250:0
|
10 |
+
#SBATCH --exclusive=user
|
11 |
+
#SBATCH --hint=nomultithread
|
12 |
+
#SBATCH --account=project_462000119
|
13 |
+
#SBATCH -o logs/%j.out
|
14 |
+
#SBATCH -e logs/%j.err
|
15 |
+
|
16 |
+
# if run without sbatch, invoke here
|
17 |
+
if [ -z $SLURM_JOB_ID ]; then
|
18 |
+
mkdir -p logs
|
19 |
+
sbatch "$0"
|
20 |
+
exit
|
21 |
+
fi
|
22 |
+
|
23 |
+
set -euo pipefail
|
24 |
+
|
25 |
+
# symlink logs/latest_eval.out and logs/latest_eval.err
|
26 |
+
ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
|
27 |
+
ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
|
28 |
+
|
29 |
+
source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
|
30 |
+
|
31 |
+
echo "START TIME: $(date)"
|
32 |
+
|
33 |
+
# defining the right environment variables
|
34 |
+
export HF_DATASETS_OFFLINE=1
|
35 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
36 |
+
|
37 |
+
# Converted transformer checkpoint
|
38 |
+
# cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
|
39 |
+
|
40 |
+
|
41 |
+
CKPTS=(
|
42 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b12bc4/global_step80108
|
43 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/global_step80108
|
44 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/global_step80108
|
45 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/global_step80108
|
46 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/global_step80108
|
47 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/global_step80108
|
48 |
+
)
|
49 |
+
|
50 |
+
CKPTS=(
|
51 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b12boscar/global_step80108
|
52 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/global_step80108
|
53 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/global_step80108
|
54 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/global_step80108
|
55 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/global_step80108
|
56 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/global_step80108
|
57 |
+
)
|
58 |
+
|
59 |
+
|
60 |
+
|
61 |
+
CKPTS=(
|
62 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b25b/global_step84877
|
63 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b35b/global_step84877
|
64 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b88b/global_step84877
|
65 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b88b/global_step84877
|
66 |
+
)
|
67 |
+
|
68 |
+
CKPTS=(
|
69 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b44b/global_step84877
|
70 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b58b/global_step84877
|
71 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b44b/global_step84877
|
72 |
+
)
|
73 |
+
|
74 |
+
CKPTSX=(
|
75 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/global_step80108
|
76 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/global_step80108
|
77 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/global_step80108
|
78 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/global_step80108
|
79 |
+
)
|
80 |
+
|
81 |
+
CKPTS=(
|
82 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
|
83 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
|
84 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
|
85 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
|
86 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
|
87 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
|
88 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
|
89 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
|
90 |
+
)
|
91 |
+
|
92 |
+
CKPTSX=(
|
93 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b1b25oscar/global_step52452
|
94 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b4boscar/global_step52452
|
95 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/global_step52452
|
96 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b11boscar/global_step52452
|
97 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b14boscar/global_step52452
|
98 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18boscar/global_step52452
|
99 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b28boscar/global_step52452
|
100 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b55boscar/global_step52452
|
101 |
+
)
|
102 |
+
|
103 |
+
CKPTSX=(
|
104 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed1/global_step52452
|
105 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed2/global_step52452
|
106 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed3/global_step52452
|
107 |
+
)
|
108 |
+
|
109 |
+
CKPTS=(
|
110 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
|
111 |
+
)
|
112 |
+
|
113 |
+
CKPTS=(
|
114 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
|
115 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
|
116 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
|
117 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
|
118 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
|
119 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
|
120 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
|
121 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
|
122 |
+
)
|
123 |
+
|
124 |
+
|
125 |
+
CKPTS=(
|
126 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm3-2b8-55b-c4/global_step52452
|
127 |
+
)
|
128 |
+
|
129 |
+
FEWSHOT_CONFIGS=(
|
130 |
+
0
|
131 |
+
1
|
132 |
+
2
|
133 |
+
3
|
134 |
+
4
|
135 |
+
5
|
136 |
+
)
|
137 |
+
|
138 |
+
TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2
|
139 |
+
|
140 |
+
# Iterate through all possible combinations of data config, model ckpt & fewshot config and run the jobs
|
141 |
+
for ((i=0; i<${#CKPTS[@]}; i++)); do
|
142 |
+
for ((j=0; j<${#FEWSHOT_CONFIGS[@]}; j++)); do
|
143 |
+
#echo "sbatch --export=CKPT=${CKPTS[$i]},FEWSHOT_CONFIG=${FEWSHOT_CONFIGS[$j]},DATASET=${DATASETS[$k]} eval.sh"
|
144 |
+
MODEL_CKPT=${CKPTS[$i]}
|
145 |
+
MODEL_CKPT_NO_STEP=${MODEL_CKPT%/*}
|
146 |
+
MODEL_NAME=${MODEL_CKPT_NO_STEP##*/}
|
147 |
+
mkdir -p $MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser
|
148 |
+
#mv $MODEL_CKPT_NO_STEP/evaluation/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.* $MODEL_CKPT_NO_STEP/evaluation/rankeval/
|
149 |
+
OUTPUT_PATH=$MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.json
|
150 |
+
eval_script="./eval_$i-$j.slurm"
|
151 |
+
cat <<EOT > $eval_script
|
152 |
+
#!/bin/bash
|
153 |
+
#SBATCH --exclude=nid005159
|
154 |
+
#SBATCH --nodes=1
|
155 |
+
#SBATCH --ntasks-per-node=1
|
156 |
+
#SBATCH --cpus-per-task=8
|
157 |
+
#SBATCH --mem=256G
|
158 |
+
#SBATCH -p small-g
|
159 |
+
#SBATCH -t 2-0:00:00
|
160 |
+
#SBATCH --gpus-per-node=mi250:1
|
161 |
+
#SBATCH --exclusive=user
|
162 |
+
#SBATCH --hint=nomultithread
|
163 |
+
#SBATCH --account=project_462000119
|
164 |
+
#SBATCH -o logs/%j.out
|
165 |
+
#SBATCH -e logs/%j.err
|
166 |
+
|
167 |
+
export HF_DATASETS_OFFLINE=1
|
168 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
169 |
+
|
170 |
+
VOCAB_FILE="gpt2/vocab.json"
|
171 |
+
MERGE_FILE="gpt2/merges.txt"
|
172 |
+
|
173 |
+
PP_SIZE=1
|
174 |
+
TP_SIZE=1
|
175 |
+
# different from the training MICRO_BATCH_SIZE - no optim memory, so can do bigger BS
|
176 |
+
# make as big as it can fit into gpu w/o OOM, but not too close to 100%
|
177 |
+
EVAL_MICRO_BATCH_SIZE=1
|
178 |
+
MICRO_BS_MULTIPLIER=1
|
179 |
+
|
180 |
+
# Model parameters
|
181 |
+
SEQ_LEN=2048
|
182 |
+
|
183 |
+
# Dummy arguments
|
184 |
+
MEGATRON_REQUIRED_ARGS=" \
|
185 |
+
--num-layers -1 \
|
186 |
+
--hidden-size -1 \
|
187 |
+
--num-attention-heads -1 \
|
188 |
+
--seq-length -1 \
|
189 |
+
--max-position-embeddings -1 \
|
190 |
+
"
|
191 |
+
|
192 |
+
ZERO_STAGE=0
|
193 |
+
|
194 |
+
mkdir -p ds_configs
|
195 |
+
DS_CONFIG_PATH="ds_configs/\$SLURM_JOB_ID.json"
|
196 |
+
|
197 |
+
cat <<EOF > "\$DS_CONFIG_PATH"
|
198 |
+
{
|
199 |
+
"train_micro_batch_size_per_gpu": 1,
|
200 |
+
"train_batch_size": 1,
|
201 |
+
"gradient_clipping": 1.0,
|
202 |
+
"zero_optimization": {
|
203 |
+
"stage": \$ZERO_STAGE
|
204 |
+
},
|
205 |
+
"bf16": {
|
206 |
+
"enabled": true
|
207 |
+
},
|
208 |
+
"steps_per_print": 2000,
|
209 |
+
"wall_clock_breakdown": false
|
210 |
+
}
|
211 |
+
EOF
|
212 |
+
|
213 |
+
DEEPSPEED_ARGS=" \
|
214 |
+
--deepspeed \
|
215 |
+
--deepspeed_config \$DS_CONFIG_PATH \
|
216 |
+
--zero-stage \$ZERO_STAGE \
|
217 |
+
"
|
218 |
+
|
219 |
+
CMD="Megatron-DeepSpeed/tasks/eval_harness/evaluate.py \
|
220 |
+
--load $MODEL_CKPT \
|
221 |
+
--results_path $OUTPUT_PATH \
|
222 |
+
--tensor-model-parallel-size \$TP_SIZE \
|
223 |
+
--pipeline-model-parallel-size \$PP_SIZE \
|
224 |
+
--vocab-file \$VOCAB_FILE \
|
225 |
+
--merge-file \$MERGE_FILE \
|
226 |
+
--micro-batch-size \$EVAL_MICRO_BATCH_SIZE \
|
227 |
+
--no-load-optim \
|
228 |
+
--no-load-rng \
|
229 |
+
--bf16 \
|
230 |
+
--inference \
|
231 |
+
--seq-length \$SEQ_LEN \
|
232 |
+
--task_list anli_r1,anli_r2,anli_r3,cb,copa,hellaswag,rte,winogrande,storycloze_2016,boolq,arc_easy,arc_challenge,sciq,piqa \
|
233 |
+
--intermed_results \
|
234 |
+
--adaptive_seq_len \
|
235 |
+
--micro_bs_multiplier \$MICRO_BS_MULTIPLIER \
|
236 |
+
--fewshots ${FEWSHOT_CONFIGS[$j]} \
|
237 |
+
\$MEGATRON_REQUIRED_ARGS \
|
238 |
+
\$DEEPSPEED_ARGS \
|
239 |
+
"
|
240 |
+
|
241 |
+
echo "\$CMD"
|
242 |
+
|
243 |
+
echo "START \$SLURM_JOBID: $(date)"
|
244 |
+
|
245 |
+
srun --label launch.sh \$CMD
|
246 |
+
|
247 |
+
echo "END \$SLURM_JOBID: $(date)"
|
248 |
+
EOT
|
249 |
+
sbatch $eval_script
|
250 |
+
# Sleep for a bit to avoid hitting the job submission limit
|
251 |
+
sleep 0.1
|
252 |
+
done
|
253 |
+
done
|
rank_eval_all_prefix_ul2.sh
ADDED
@@ -0,0 +1,258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
#SBATCH --exclude=nid005159
|
3 |
+
#SBATCH --nodes=1
|
4 |
+
#SBATCH --ntasks-per-node=1
|
5 |
+
#SBATCH --cpus-per-task=4
|
6 |
+
#SBATCH --mem=256G
|
7 |
+
#SBATCH -p small-g
|
8 |
+
#SBATCH -t 2-0:00:00
|
9 |
+
#SBATCH --gpus-per-node=mi250:0
|
10 |
+
#SBATCH --exclusive=user
|
11 |
+
#SBATCH --hint=nomultithread
|
12 |
+
#SBATCH --account=project_462000119
|
13 |
+
#SBATCH -o logs/%j.out
|
14 |
+
#SBATCH -e logs/%j.err
|
15 |
+
|
16 |
+
# if run without sbatch, invoke here
|
17 |
+
if [ -z $SLURM_JOB_ID ]; then
|
18 |
+
mkdir -p logs
|
19 |
+
sbatch "$0"
|
20 |
+
exit
|
21 |
+
fi
|
22 |
+
|
23 |
+
set -euo pipefail
|
24 |
+
|
25 |
+
# symlink logs/latest_eval.out and logs/latest_eval.err
|
26 |
+
ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
|
27 |
+
ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
|
28 |
+
|
29 |
+
source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
|
30 |
+
|
31 |
+
echo "START TIME: $(date)"
|
32 |
+
|
33 |
+
# defining the right environment variables
|
34 |
+
export HF_DATASETS_OFFLINE=1
|
35 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
36 |
+
|
37 |
+
# Converted transformer checkpoint
|
38 |
+
# cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
|
39 |
+
|
40 |
+
|
41 |
+
CKPTS=(
|
42 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b12bc4/global_step80108
|
43 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/global_step80108
|
44 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/global_step80108
|
45 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/global_step80108
|
46 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/global_step80108
|
47 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/global_step80108
|
48 |
+
)
|
49 |
+
|
50 |
+
CKPTS=(
|
51 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b12boscar/global_step80108
|
52 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/global_step80108
|
53 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/global_step80108
|
54 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/global_step80108
|
55 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/global_step80108
|
56 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/global_step80108
|
57 |
+
)
|
58 |
+
|
59 |
+
|
60 |
+
|
61 |
+
CKPTS=(
|
62 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b25b/global_step84877
|
63 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b35b/global_step84877
|
64 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b88b/global_step84877
|
65 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b88b/global_step84877
|
66 |
+
)
|
67 |
+
|
68 |
+
CKPTS=(
|
69 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b44b/global_step84877
|
70 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b58b/global_step84877
|
71 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b44b/global_step84877
|
72 |
+
)
|
73 |
+
|
74 |
+
CKPTSX=(
|
75 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/global_step80108
|
76 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/global_step80108
|
77 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/global_step80108
|
78 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/global_step80108
|
79 |
+
)
|
80 |
+
|
81 |
+
CKPTS=(
|
82 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
|
83 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
|
84 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
|
85 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
|
86 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
|
87 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
|
88 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
|
89 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
|
90 |
+
)
|
91 |
+
|
92 |
+
CKPTSX=(
|
93 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b1b25oscar/global_step52452
|
94 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b4boscar/global_step52452
|
95 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/global_step52452
|
96 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b11boscar/global_step52452
|
97 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b14boscar/global_step52452
|
98 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18boscar/global_step52452
|
99 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b28boscar/global_step52452
|
100 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b55boscar/global_step52452
|
101 |
+
)
|
102 |
+
|
103 |
+
CKPTSX=(
|
104 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed1/global_step52452
|
105 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed2/global_step52452
|
106 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed3/global_step52452
|
107 |
+
)
|
108 |
+
|
109 |
+
CKPTS=(
|
110 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
|
111 |
+
)
|
112 |
+
|
113 |
+
CKPTS=(
|
114 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/global_step52452
|
115 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/global_step52452
|
116 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/global_step52452
|
117 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/global_step52452
|
118 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/global_step52452
|
119 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/global_step52452
|
120 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/global_step52452
|
121 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
|
122 |
+
)
|
123 |
+
|
124 |
+
|
125 |
+
CKPTS=(
|
126 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm4-2b8-55b-c4/global_step52452
|
127 |
+
)
|
128 |
+
|
129 |
+
CKPTS=(
|
130 |
+
/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/checkpoints_2b855b55bc4ul2ndfixnew/global_step52452
|
131 |
+
)
|
132 |
+
|
133 |
+
FEWSHOT_CONFIGS=(
|
134 |
+
0
|
135 |
+
1
|
136 |
+
2
|
137 |
+
3
|
138 |
+
4
|
139 |
+
5
|
140 |
+
)
|
141 |
+
|
142 |
+
TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2
|
143 |
+
|
144 |
+
# Iterate through all possible combinations of data config, model ckpt & fewshot config and run the jobs
|
145 |
+
for ((i=0; i<${#CKPTS[@]}; i++)); do
|
146 |
+
for ((j=0; j<${#FEWSHOT_CONFIGS[@]}; j++)); do
|
147 |
+
#echo "sbatch --export=CKPT=${CKPTS[$i]},FEWSHOT_CONFIG=${FEWSHOT_CONFIGS[$j]},DATASET=${DATASETS[$k]} eval.sh"
|
148 |
+
MODEL_CKPT=${CKPTS[$i]}
|
149 |
+
MODEL_CKPT_NO_STEP=${MODEL_CKPT%/*}
|
150 |
+
MODEL_NAME=${MODEL_CKPT_NO_STEP##*/}
|
151 |
+
mkdir -p $MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser
|
152 |
+
#mv $MODEL_CKPT_NO_STEP/evaluation/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.* $MODEL_CKPT_NO_STEP/evaluation/rankeval/
|
153 |
+
OUTPUT_PATH=$MODEL_CKPT_NO_STEP/evaluation/rankeval_r_denoiser/$MODEL_NAME\_${FEWSHOT_CONFIGS[$j]}.json
|
154 |
+
eval_script="./eval_$i-$j.slurm"
|
155 |
+
cat <<EOT > $eval_script
|
156 |
+
#!/bin/bash
|
157 |
+
#SBATCH --exclude=nid005159
|
158 |
+
#SBATCH --nodes=1
|
159 |
+
#SBATCH --ntasks-per-node=1
|
160 |
+
#SBATCH --cpus-per-task=8
|
161 |
+
#SBATCH --mem=256G
|
162 |
+
#SBATCH -p small-g
|
163 |
+
#SBATCH -t 2-0:00:00
|
164 |
+
#SBATCH --gpus-per-node=mi250:1
|
165 |
+
#SBATCH --exclusive=user
|
166 |
+
#SBATCH --hint=nomultithread
|
167 |
+
#SBATCH --account=project_462000119
|
168 |
+
#SBATCH -o logs/%j.out
|
169 |
+
#SBATCH -e logs/%j.err
|
170 |
+
|
171 |
+
export HF_DATASETS_OFFLINE=1
|
172 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
173 |
+
|
174 |
+
VOCAB_FILE="gpt2/vocab.json"
|
175 |
+
MERGE_FILE="gpt2/merges.txt"
|
176 |
+
|
177 |
+
PP_SIZE=1
|
178 |
+
TP_SIZE=1
|
179 |
+
# different from the training MICRO_BATCH_SIZE - no optim memory, so can do bigger BS
|
180 |
+
# make as big as it can fit into gpu w/o OOM, but not too close to 100%
|
181 |
+
EVAL_MICRO_BATCH_SIZE=1
|
182 |
+
MICRO_BS_MULTIPLIER=1
|
183 |
+
|
184 |
+
# Model parameters
|
185 |
+
SEQ_LEN=2048
|
186 |
+
|
187 |
+
# Dummy arguments
|
188 |
+
MEGATRON_REQUIRED_ARGS=" \
|
189 |
+
--num-layers -1 \
|
190 |
+
--hidden-size -1 \
|
191 |
+
--num-attention-heads -1 \
|
192 |
+
--seq-length -1 \
|
193 |
+
--max-position-embeddings -1 \
|
194 |
+
"
|
195 |
+
|
196 |
+
ZERO_STAGE=0
|
197 |
+
|
198 |
+
mkdir -p ds_configs
|
199 |
+
DS_CONFIG_PATH="ds_configs/\$SLURM_JOB_ID.json"
|
200 |
+
|
201 |
+
cat <<EOF > "\$DS_CONFIG_PATH"
|
202 |
+
{
|
203 |
+
"train_micro_batch_size_per_gpu": 1,
|
204 |
+
"train_batch_size": 1,
|
205 |
+
"gradient_clipping": 1.0,
|
206 |
+
"zero_optimization": {
|
207 |
+
"stage": \$ZERO_STAGE
|
208 |
+
},
|
209 |
+
"bf16": {
|
210 |
+
"enabled": true
|
211 |
+
},
|
212 |
+
"steps_per_print": 2000,
|
213 |
+
"wall_clock_breakdown": false
|
214 |
+
}
|
215 |
+
EOF
|
216 |
+
|
217 |
+
DEEPSPEED_ARGS=" \
|
218 |
+
--deepspeed \
|
219 |
+
--deepspeed_config \$DS_CONFIG_PATH \
|
220 |
+
--zero-stage \$ZERO_STAGE \
|
221 |
+
"
|
222 |
+
|
223 |
+
CMD="Megatron-DeepSpeed/tasks/eval_harness/evaluate_prefix_ul2.py \
|
224 |
+
--load $MODEL_CKPT \
|
225 |
+
--results_path $OUTPUT_PATH \
|
226 |
+
--tensor-model-parallel-size \$TP_SIZE \
|
227 |
+
--pipeline-model-parallel-size \$PP_SIZE \
|
228 |
+
--vocab-file \$VOCAB_FILE \
|
229 |
+
--merge-file \$MERGE_FILE \
|
230 |
+
--micro-batch-size \$EVAL_MICRO_BATCH_SIZE \
|
231 |
+
--no-load-optim \
|
232 |
+
--no-load-rng \
|
233 |
+
--bf16 \
|
234 |
+
--inference \
|
235 |
+
--seq-length \$SEQ_LEN \
|
236 |
+
--task_list anli_r1,anli_r2,anli_r3,cb,copa,hellaswag,rte,winogrande,storycloze_2016,boolq,arc_easy,arc_challenge,sciq,piqa \
|
237 |
+
--intermed_results \
|
238 |
+
--adaptive_seq_len \
|
239 |
+
--micro_bs_multiplier \$MICRO_BS_MULTIPLIER \
|
240 |
+
--fewshots ${FEWSHOT_CONFIGS[$j]} \
|
241 |
+
--prefix \
|
242 |
+
\$MEGATRON_REQUIRED_ARGS \
|
243 |
+
\$DEEPSPEED_ARGS \
|
244 |
+
"
|
245 |
+
|
246 |
+
echo "\$CMD"
|
247 |
+
|
248 |
+
echo "START \$SLURM_JOBID: $(date)"
|
249 |
+
|
250 |
+
srun --label launch.sh \$CMD
|
251 |
+
|
252 |
+
echo "END \$SLURM_JOBID: $(date)"
|
253 |
+
EOT
|
254 |
+
sbatch $eval_script
|
255 |
+
# Sleep for a bit to avoid hitting the job submission limit
|
256 |
+
sleep 0.1
|
257 |
+
done
|
258 |
+
done
|
run_eval_lumi_gen.sh
ADDED
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
#SBATCH --exclude=nid005159
|
3 |
+
#SBATCH --nodes=1
|
4 |
+
#SBATCH --ntasks-per-node=1
|
5 |
+
#SBATCH --cpus-per-task=32
|
6 |
+
#SBATCH --mem=256G
|
7 |
+
#SBATCH -p small-g
|
8 |
+
#SBATCH -t 2-0:00:00
|
9 |
+
#SBATCH --gpus-per-node=mi250:1
|
10 |
+
#SBATCH --exclusive=user
|
11 |
+
#SBATCH --hint=nomultithread
|
12 |
+
#SBATCH --account=project_462000119
|
13 |
+
#SBATCH -o logs/%j.out
|
14 |
+
#SBATCH -e logs/%j.err
|
15 |
+
|
16 |
+
# if run without sbatch, invoke here
|
17 |
+
if [ -z $SLURM_JOB_ID ]; then
|
18 |
+
mkdir -p logs
|
19 |
+
sbatch "$0"
|
20 |
+
exit
|
21 |
+
fi
|
22 |
+
|
23 |
+
set -euo pipefail
|
24 |
+
|
25 |
+
# symlink logs/latest_eval.out and logs/latest_eval.err
|
26 |
+
ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
|
27 |
+
ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
|
28 |
+
|
29 |
+
source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
|
30 |
+
|
31 |
+
echo "START TIME: $(date)"
|
32 |
+
|
33 |
+
# defining the right environment variables
|
34 |
+
export HF_DATASETS_OFFLINE=1
|
35 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
36 |
+
|
37 |
+
# Converted transformer checkpoint
|
38 |
+
MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18bc4/transformers
|
39 |
+
MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/transformers
|
40 |
+
MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4py/2b855b90c4py/transformers
|
41 |
+
#MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-realtasky/transformers
|
42 |
+
MODEL_CKPT=/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm3-2b8-55b-c4/transformers
|
43 |
+
TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/gpt2
|
44 |
+
|
45 |
+
cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness
|
46 |
+
|
47 |
+
# WMT19 ZH-EN does not work
|
48 |
+
DATASETS_AND_CONFIGS=(
|
49 |
+
GEM/wiki_lingua_en,"tldr_en"
|
50 |
+
gem_xsum,"article_DOC_summary"
|
51 |
+
GEM/web_nlg_en,"PALM_prompt"
|
52 |
+
e2e_nlg_cleaned,"generate_text_restaurant"
|
53 |
+
)
|
54 |
+
|
55 |
+
DATASET_AND_CONFIG=${DATASETS_AND_CONFIGS[$SLURM_ARRAY_TASK_ID]}
|
56 |
+
#echo $ARGUMENT
|
57 |
+
|
58 |
+
IFS=',' read dataset_name template_name <<< "${DATASET_AND_CONFIG}"
|
59 |
+
|
60 |
+
# Use this fork of lm-eval: https://github.com/bigscience-workshop/lm-evaluation-harness/pull/109
|
61 |
+
python main.py \
|
62 |
+
--model_api_name 'hf-causal' \
|
63 |
+
--model_args pretrained=$MODEL_CKPT,use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
|
64 |
+
--device cuda \
|
65 |
+
--batch_size 16 \
|
66 |
+
--no_tracking \
|
67 |
+
--task_name $dataset_name \
|
68 |
+
--template_names $template_name \
|
69 |
+
--bootstrap_iters 10 \
|
70 |
+
--limit 3000 \
|
71 |
+
--num_fewshot 1
|
72 |
+
|
73 |
+
echo "END TIME: $(date)"
|
run_eval_lumi_xp3eval.sh
ADDED
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
#SBATCH --exclude=nid005159
|
3 |
+
#SBATCH --nodes=1
|
4 |
+
#SBATCH --ntasks-per-node=1
|
5 |
+
#SBATCH --cpus-per-task=32
|
6 |
+
#SBATCH --mem=256G
|
7 |
+
#SBATCH -p small-g
|
8 |
+
#SBATCH -t 2-0:00:00
|
9 |
+
#SBATCH --gpus-per-node=mi250:1
|
10 |
+
#SBATCH --exclusive=user
|
11 |
+
#SBATCH --hint=nomultithread
|
12 |
+
#SBATCH --account=project_462000119
|
13 |
+
#SBATCH -o logs/%j.out
|
14 |
+
#SBATCH -e logs/%j.err
|
15 |
+
|
16 |
+
# if run without sbatch, invoke here
|
17 |
+
if [ -z $SLURM_JOB_ID ]; then
|
18 |
+
mkdir -p logs
|
19 |
+
sbatch "$0"
|
20 |
+
exit
|
21 |
+
fi
|
22 |
+
|
23 |
+
set -euo pipefail
|
24 |
+
|
25 |
+
# symlink logs/latest_eval.out and logs/latest_eval.err
|
26 |
+
ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
|
27 |
+
ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err
|
28 |
+
|
29 |
+
# Data
|
30 |
+
#CHECKPOINT_PATH=/scratch/project_462000119/muennighoff/nov-2022-optimization/checkpoints/global_step10
|
31 |
+
#VARIANT=global_step10
|
32 |
+
|
33 |
+
CHECKPOINT_PATH=lm1-220m/global_step14324
|
34 |
+
VARIANT=lm1-220m
|
35 |
+
CHECKPOINT_PATH=lm1-220m-7b5-oscar/global_step14324
|
36 |
+
VARIANT=lm1-220m-7b5-oscar
|
37 |
+
|
38 |
+
#CHECKPOINT_PATH=lm1-280m/global_step11269
|
39 |
+
#VARIANT=lm1-280m-5b9
|
40 |
+
#CHECKPOINT_PATH=lm1-280m-5b9-oscar/global_step11269
|
41 |
+
#VARIANT=lm1-280m-5b9-oscar
|
42 |
+
|
43 |
+
CHECKPOINT_PATH=lm1-1b1-21b-oscar/global_step39672
|
44 |
+
VARIANT=lm1-1b1-21b-oscar
|
45 |
+
#CHECKPOINT_PATH=lm1-1b1-21b/global_step39672
|
46 |
+
#VARIANT=lm1-1b1-21b
|
47 |
+
|
48 |
+
#CHECKPOINT_PATH=lm1-2b8-55b-oscar/global_step52452
|
49 |
+
#VARIANT=lm1-2b8-55b-oscar
|
50 |
+
#CHECKPOINT_PATH=lm1-2b8-55b/global_step52452
|
51 |
+
#VARIANT=lm1-2b8-55b
|
52 |
+
|
53 |
+
#CHECKPOINT_PATH=lm1-2b8-55b-oscar/global_step52452
|
54 |
+
#VARIANT=lm1-2b8-55b-oscar
|
55 |
+
#CHECKPOINT_PATH=lm1-3b9-77b/global_step73814
|
56 |
+
#VARIANT=lm1-3b9-77b
|
57 |
+
|
58 |
+
#CHECKPOINT_PATH=lm1-1b1-21b-c4/global_step39672
|
59 |
+
#VARIANT=lm1-1b1-21b-c4
|
60 |
+
|
61 |
+
|
62 |
+
# tensorboard_2b855b11bc4 tensorboard_2b855b14bc4 tensorboard_2b855b18bc4 tensorboard_2b855b28bc4 tensorboard_2b855b9bc4
|
63 |
+
#2b855b50c4py 2b855b60c4py 2b855b70c4py 2b855b80c4py 2b855b90c4py
|
64 |
+
VARIANT=2b855b70c4py
|
65 |
+
CHECKPOINT_PATH=lm1-2b8-55b-c4py/$VARIANT/global_step52452
|
66 |
+
#2b855b11bc4 2b855b14bc4 2b855b18bc4 2b855b28bc4 2b855b9bc4
|
67 |
+
#VARIANT=2b855b9boscar
|
68 |
+
#CHECKPOINT_PATH=lm1-2b8-55b-oscar-repetitions/$VARIANT/global_step52452
|
69 |
+
|
70 |
+
#VARIANT=realtasky
|
71 |
+
#CHECKPOINT_PATH=checkpoints_2b855brealtasky/global_step52452
|
72 |
+
#CHECKPOINT_PATH=lm1-2b8-55b-c4-repetitions/2b855b55bc4/global_step52452
|
73 |
+
|
74 |
+
|
75 |
+
CHECKPOINT_PATH=checkpoints_2b855b55bc4ul2valfast/global_step52452
|
76 |
+
VARIANT=ul2valfast
|
77 |
+
|
78 |
+
CHECKPOINT_PATH=lm2-2b8-55b-c4-new/global_step52452
|
79 |
+
VARIANT=ul2new
|
80 |
+
|
81 |
+
export HF_DATASETS_OFFLINE=1
|
82 |
+
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
|
83 |
+
|
84 |
+
VOCAB_FILE="gpt2/vocab.json"
|
85 |
+
MERGE_FILE="gpt2/merges.txt"
|
86 |
+
|
87 |
+
PP_SIZE=1
|
88 |
+
TP_SIZE=1
|
89 |
+
# different from the training MICRO_BATCH_SIZE - no optim memory, so can do bigger BS
|
90 |
+
# make as big as it can fit into gpu w/o OOM, but not too close to 100%
|
91 |
+
EVAL_MICRO_BATCH_SIZE=1
|
92 |
+
MICRO_BS_MULTIPLIER=1
|
93 |
+
|
94 |
+
# Model parameters
|
95 |
+
SEQ_LEN=2048
|
96 |
+
|
97 |
+
# Dummy arguments
|
98 |
+
MEGATRON_REQUIRED_ARGS=" \
|
99 |
+
--num-layers -1 \
|
100 |
+
--hidden-size -1 \
|
101 |
+
--num-attention-heads -1 \
|
102 |
+
--seq-length -1 \
|
103 |
+
--max-position-embeddings -1 \
|
104 |
+
"
|
105 |
+
|
106 |
+
ZERO_STAGE=0
|
107 |
+
|
108 |
+
mkdir -p ds_configs
|
109 |
+
DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
|
110 |
+
|
111 |
+
cat <<EOF > $DS_CONFIG_PATH
|
112 |
+
{
|
113 |
+
"train_micro_batch_size_per_gpu": 1,
|
114 |
+
"train_batch_size": 1,
|
115 |
+
"gradient_clipping": 1.0,
|
116 |
+
"zero_optimization": {
|
117 |
+
"stage": $ZERO_STAGE
|
118 |
+
},
|
119 |
+
"bf16": {
|
120 |
+
"enabled": true
|
121 |
+
},
|
122 |
+
"steps_per_print": 2000,
|
123 |
+
"wall_clock_breakdown": false
|
124 |
+
}
|
125 |
+
EOF
|
126 |
+
|
127 |
+
DEEPSPEED_ARGS=" \
|
128 |
+
--deepspeed \
|
129 |
+
--deepspeed_config $DS_CONFIG_PATH \
|
130 |
+
--zero-stage $ZERO_STAGE \
|
131 |
+
"
|
132 |
+
|
133 |
+
CMD="Megatron-DeepSpeed/tasks/eval_harness/evaluate.py \
|
134 |
+
--load $CHECKPOINT_PATH \
|
135 |
+
--results_path $VARIANT-results.json \
|
136 |
+
--tensor-model-parallel-size $TP_SIZE \
|
137 |
+
--pipeline-model-parallel-size $PP_SIZE \
|
138 |
+
--vocab-file $VOCAB_FILE \
|
139 |
+
--merge-file $MERGE_FILE \
|
140 |
+
--micro-batch-size $EVAL_MICRO_BATCH_SIZE \
|
141 |
+
--no-load-optim \
|
142 |
+
--no-load-rng \
|
143 |
+
--bf16 \
|
144 |
+
--inference \
|
145 |
+
--seq-length $SEQ_LEN \
|
146 |
+
--task_list anli_r1,anli_r2,anli_r3,cb,copa,hellaswag,rte,winogrande,storycloze_2016,boolq,arc_easy,arc_challenge,sciq,piqa \
|
147 |
+
--intermed_results \
|
148 |
+
--adaptive_seq_len \
|
149 |
+
--add_denoiser \
|
150 |
+
--micro_bs_multiplier $MICRO_BS_MULTIPLIER \
|
151 |
+
$MEGATRON_REQUIRED_ARGS \
|
152 |
+
$DEEPSPEED_ARGS \
|
153 |
+
"
|
154 |
+
|
155 |
+
echo $CMD
|
156 |
+
|
157 |
+
echo "START $SLURM_JOBID: $(date)"
|
158 |
+
|
159 |
+
srun --label launch.sh $CMD
|
160 |
+
|
161 |
+
echo "END $SLURM_JOBID: $(date)"
|
162 |
+
|