malteos commited on
Commit
699b5a6
1 Parent(s): e662bf0

Upload folder using huggingface_hub

Browse files
eval/es_hard__step_9537_1711620130.json ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "arc_easy": {
4
+ "acc,none": 0.3265993265993266,
5
+ "acc_stderr,none": 0.009623047038267645,
6
+ "acc_norm,none": 0.3274410774410774,
7
+ "acc_norm_stderr,none": 0.009629415859100605,
8
+ "alias": "arc_easy"
9
+ },
10
+ "lambada_openai": {
11
+ "perplexity,none": 861.0431215826027,
12
+ "perplexity_stderr,none": 49.374856514133086,
13
+ "acc,none": 0.15990685037842034,
14
+ "acc_stderr,none": 0.0051063353047618885,
15
+ "alias": "lambada_openai"
16
+ },
17
+ "piqa": {
18
+ "acc,none": 0.5505984766050055,
19
+ "acc_stderr,none": 0.01160593662415608,
20
+ "acc_norm,none": 0.5391730141458106,
21
+ "acc_norm_stderr,none": 0.011629966056957108,
22
+ "alias": "piqa"
23
+ },
24
+ "sciq": {
25
+ "acc,none": 0.514,
26
+ "acc_stderr,none": 0.015813097547730984,
27
+ "acc_norm,none": 0.511,
28
+ "acc_norm_stderr,none": 0.01581547119529269,
29
+ "alias": "sciq"
30
+ },
31
+ "wikitext": {
32
+ "word_perplexity,none": 14608.989515390149,
33
+ "word_perplexity_stderr,none": "N/A",
34
+ "byte_perplexity,none": 6.00903579422426,
35
+ "byte_perplexity_stderr,none": "N/A",
36
+ "bits_per_byte,none": 2.5871335156471185,
37
+ "bits_per_byte_stderr,none": "N/A",
38
+ "alias": "wikitext"
39
+ },
40
+ "winogrande": {
41
+ "acc,none": 0.5027624309392266,
42
+ "acc_stderr,none": 0.014052271211616436,
43
+ "alias": "winogrande"
44
+ }
45
+ },
46
+ "configs": {
47
+ "arc_easy": {
48
+ "task": "arc_easy",
49
+ "group": [
50
+ "ai2_arc"
51
+ ],
52
+ "dataset_path": "ai2_arc",
53
+ "dataset_name": "ARC-Easy",
54
+ "training_split": "train",
55
+ "validation_split": "validation",
56
+ "test_split": "test",
57
+ "doc_to_text": "Question: {{question}}\nAnswer:",
58
+ "doc_to_target": "{{choices.label.index(answerKey)}}",
59
+ "doc_to_choice": "{{choices.text}}",
60
+ "description": "",
61
+ "target_delimiter": " ",
62
+ "fewshot_delimiter": "\n\n",
63
+ "num_fewshot": 0,
64
+ "metric_list": [
65
+ {
66
+ "metric": "acc",
67
+ "aggregation": "mean",
68
+ "higher_is_better": true
69
+ },
70
+ {
71
+ "metric": "acc_norm",
72
+ "aggregation": "mean",
73
+ "higher_is_better": true
74
+ }
75
+ ],
76
+ "output_type": "multiple_choice",
77
+ "repeats": 1,
78
+ "should_decontaminate": true,
79
+ "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",
80
+ "metadata": {
81
+ "version": 1.0
82
+ }
83
+ },
84
+ "lambada_openai": {
85
+ "task": "lambada_openai",
86
+ "group": [
87
+ "lambada"
88
+ ],
89
+ "dataset_path": "EleutherAI/lambada_openai",
90
+ "dataset_name": "default",
91
+ "test_split": "test",
92
+ "doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
93
+ "doc_to_target": "{{' '+text.split(' ')[-1]}}",
94
+ "description": "",
95
+ "target_delimiter": " ",
96
+ "fewshot_delimiter": "\n\n",
97
+ "num_fewshot": 0,
98
+ "metric_list": [
99
+ {
100
+ "metric": "perplexity",
101
+ "aggregation": "perplexity",
102
+ "higher_is_better": false
103
+ },
104
+ {
105
+ "metric": "acc",
106
+ "aggregation": "mean",
107
+ "higher_is_better": true
108
+ }
109
+ ],
110
+ "output_type": "loglikelihood",
111
+ "repeats": 1,
112
+ "should_decontaminate": true,
113
+ "doc_to_decontamination_query": "{{text}}",
114
+ "metadata": {
115
+ "version": 1.0
116
+ }
117
+ },
118
+ "piqa": {
119
+ "task": "piqa",
120
+ "dataset_path": "piqa",
121
+ "training_split": "train",
122
+ "validation_split": "validation",
123
+ "doc_to_text": "Question: {{goal}}\nAnswer:",
124
+ "doc_to_target": "label",
125
+ "doc_to_choice": "{{[sol1, sol2]}}",
126
+ "description": "",
127
+ "target_delimiter": " ",
128
+ "fewshot_delimiter": "\n\n",
129
+ "num_fewshot": 0,
130
+ "metric_list": [
131
+ {
132
+ "metric": "acc",
133
+ "aggregation": "mean",
134
+ "higher_is_better": true
135
+ },
136
+ {
137
+ "metric": "acc_norm",
138
+ "aggregation": "mean",
139
+ "higher_is_better": true
140
+ }
141
+ ],
142
+ "output_type": "multiple_choice",
143
+ "repeats": 1,
144
+ "should_decontaminate": true,
145
+ "doc_to_decontamination_query": "goal",
146
+ "metadata": {
147
+ "version": 1.0
148
+ }
149
+ },
150
+ "sciq": {
151
+ "task": "sciq",
152
+ "dataset_path": "sciq",
153
+ "training_split": "train",
154
+ "validation_split": "validation",
155
+ "test_split": "test",
156
+ "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
157
+ "doc_to_target": 3,
158
+ "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
159
+ "description": "",
160
+ "target_delimiter": " ",
161
+ "fewshot_delimiter": "\n\n",
162
+ "num_fewshot": 0,
163
+ "metric_list": [
164
+ {
165
+ "metric": "acc",
166
+ "aggregation": "mean",
167
+ "higher_is_better": true
168
+ },
169
+ {
170
+ "metric": "acc_norm",
171
+ "aggregation": "mean",
172
+ "higher_is_better": true
173
+ }
174
+ ],
175
+ "output_type": "multiple_choice",
176
+ "repeats": 1,
177
+ "should_decontaminate": true,
178
+ "doc_to_decontamination_query": "{{support}} {{question}}",
179
+ "metadata": {
180
+ "version": 1.0
181
+ }
182
+ },
183
+ "wikitext": {
184
+ "task": "wikitext",
185
+ "dataset_path": "EleutherAI/wikitext_document_level",
186
+ "dataset_name": "wikitext-2-raw-v1",
187
+ "training_split": "train",
188
+ "validation_split": "validation",
189
+ "test_split": "test",
190
+ "doc_to_text": "",
191
+ "doc_to_target": "<function wikitext_detokenizer at 0x7fb2a52737f0>",
192
+ "process_results": "<function process_results at 0x7fb2a4fb7910>",
193
+ "description": "",
194
+ "target_delimiter": " ",
195
+ "fewshot_delimiter": "\n\n",
196
+ "num_fewshot": 0,
197
+ "metric_list": [
198
+ {
199
+ "metric": "word_perplexity"
200
+ },
201
+ {
202
+ "metric": "byte_perplexity"
203
+ },
204
+ {
205
+ "metric": "bits_per_byte"
206
+ }
207
+ ],
208
+ "output_type": "loglikelihood_rolling",
209
+ "repeats": 1,
210
+ "should_decontaminate": true,
211
+ "doc_to_decontamination_query": "{{page}}",
212
+ "metadata": {
213
+ "version": 2.0
214
+ }
215
+ },
216
+ "winogrande": {
217
+ "task": "winogrande",
218
+ "dataset_path": "winogrande",
219
+ "dataset_name": "winogrande_xl",
220
+ "training_split": "train",
221
+ "validation_split": "validation",
222
+ "doc_to_text": "<function doc_to_text at 0x7fb2a4fd3370>",
223
+ "doc_to_target": "<function doc_to_target at 0x7fb2a4df1480>",
224
+ "doc_to_choice": "<function doc_to_choice at 0x7fb2a4df16c0>",
225
+ "description": "",
226
+ "target_delimiter": " ",
227
+ "fewshot_delimiter": "\n\n",
228
+ "num_fewshot": 0,
229
+ "metric_list": [
230
+ {
231
+ "metric": "acc",
232
+ "aggregation": "mean",
233
+ "higher_is_better": true
234
+ }
235
+ ],
236
+ "output_type": "multiple_choice",
237
+ "repeats": 1,
238
+ "should_decontaminate": true,
239
+ "doc_to_decontamination_query": "sentence",
240
+ "metadata": {
241
+ "version": 1.0
242
+ }
243
+ }
244
+ },
245
+ "versions": {
246
+ "arc_easy": 1.0,
247
+ "lambada_openai": 1.0,
248
+ "piqa": 1.0,
249
+ "sciq": 1.0,
250
+ "wikitext": 2.0,
251
+ "winogrande": 1.0
252
+ },
253
+ "n-shot": {
254
+ "arc_easy": 0,
255
+ "lambada_openai": 0,
256
+ "piqa": 0,
257
+ "sciq": 0,
258
+ "wikitext": 0,
259
+ "winogrande": 0
260
+ },
261
+ "config": {
262
+ "model": "hf",
263
+ "model_args": "pretrained=/netscratch/mostendorff/experiments/pythia-data-ablations/data/continued-pythia-410m/es_hard/hf_checkpoints/global_step9537,dtype=float16",
264
+ "batch_size": "auto",
265
+ "batch_sizes": [
266
+ 64
267
+ ],
268
+ "device": "cuda:0",
269
+ "use_cache": null,
270
+ "limit": null,
271
+ "bootstrap_iters": 100000,
272
+ "gen_kwargs": null
273
+ },
274
+ "git_hash": null
275
+ }