usvsnsp commited on
Commit
ea6b035
1 Parent(s): 978cc7b

Create results.json

Browse files
Files changed (1) hide show
  1. results.json +314 -0
results.json ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "arc_challenge": {
4
+ "acc,none": 0.22525597269624573,
5
+ "acc_stderr,none": 0.0122078399954073,
6
+ "acc_norm,none": 0.22781569965870307,
7
+ "acc_norm_stderr,none": 0.012256708602326907
8
+ },
9
+ "arc_easy": {
10
+ "acc,none": 0.255050505050505,
11
+ "acc_stderr,none": 0.008944265906130731,
12
+ "acc_norm,none": 0.25673400673400676,
13
+ "acc_norm_stderr,none": 0.008963590834042407
14
+ },
15
+ "lambada_openai": {
16
+ "perplexity,none": NaN,
17
+ "perplexity_stderr,none": NaN,
18
+ "acc,none": 0.0015524936929943722,
19
+ "acc_stderr,none": 0.0005485163954600048
20
+ },
21
+ "logiqa": {
22
+ "acc,none": 0.20276497695852536,
23
+ "acc_stderr,none": 0.015770046635584564,
24
+ "acc_norm,none": 0.20276497695852536,
25
+ "acc_norm_stderr,none": 0.015770046635584564
26
+ },
27
+ "piqa": {
28
+ "acc,none": 0.49455930359085964,
29
+ "acc_stderr,none": 0.011665133500637066,
30
+ "acc_norm,none": 0.4923830250272035,
31
+ "acc_norm_stderr,none": 0.011664470424044978
32
+ },
33
+ "sciq": {
34
+ "acc,none": 0.014,
35
+ "acc_stderr,none": 0.0037172325482565773,
36
+ "acc_norm,none": 0.014,
37
+ "acc_norm_stderr,none": 0.003717232548256575
38
+ },
39
+ "winogrande": {
40
+ "acc,none": 0.5035516969218626,
41
+ "acc_stderr,none": 0.014052131146915857
42
+ },
43
+ "wsc": {
44
+ "acc,none": 0.6346153846153846,
45
+ "acc_stderr,none": 0.0474473339327792
46
+ }
47
+ },
48
+ "configs": {
49
+ "arc_challenge": {
50
+ "task": "arc_challenge",
51
+ "group": [
52
+ "ai2_arc"
53
+ ],
54
+ "dataset_path": "ai2_arc",
55
+ "dataset_name": "ARC-Challenge",
56
+ "training_split": "train",
57
+ "validation_split": "validation",
58
+ "test_split": "test",
59
+ "doc_to_text": "Question: {{question}}\nAnswer:",
60
+ "doc_to_target": "{{choices.label.index(answerKey)}}",
61
+ "doc_to_choice": "{{choices.text}}",
62
+ "description": "",
63
+ "target_delimiter": " ",
64
+ "fewshot_delimiter": "\n\n",
65
+ "num_fewshot": 0,
66
+ "metric_list": [
67
+ {
68
+ "metric": "acc",
69
+ "aggregation": "mean",
70
+ "higher_is_better": true
71
+ },
72
+ {
73
+ "metric": "acc_norm",
74
+ "aggregation": "mean",
75
+ "higher_is_better": true
76
+ }
77
+ ],
78
+ "output_type": "multiple_choice",
79
+ "repeats": 1,
80
+ "should_decontaminate": true,
81
+ "doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
82
+ },
83
+ "arc_easy": {
84
+ "task": "arc_easy",
85
+ "group": [
86
+ "ai2_arc"
87
+ ],
88
+ "dataset_path": "ai2_arc",
89
+ "dataset_name": "ARC-Easy",
90
+ "training_split": "train",
91
+ "validation_split": "validation",
92
+ "test_split": "test",
93
+ "doc_to_text": "Question: {{question}}\nAnswer:",
94
+ "doc_to_target": "{{choices.label.index(answerKey)}}",
95
+ "doc_to_choice": "{{choices.text}}",
96
+ "description": "",
97
+ "target_delimiter": " ",
98
+ "fewshot_delimiter": "\n\n",
99
+ "num_fewshot": 0,
100
+ "metric_list": [
101
+ {
102
+ "metric": "acc",
103
+ "aggregation": "mean",
104
+ "higher_is_better": true
105
+ },
106
+ {
107
+ "metric": "acc_norm",
108
+ "aggregation": "mean",
109
+ "higher_is_better": true
110
+ }
111
+ ],
112
+ "output_type": "multiple_choice",
113
+ "repeats": 1,
114
+ "should_decontaminate": true,
115
+ "doc_to_decontamination_query": "Question: {{question}}\nAnswer:"
116
+ },
117
+ "lambada_openai": {
118
+ "task": "lambada_openai",
119
+ "group": [
120
+ "lambada"
121
+ ],
122
+ "dataset_path": "EleutherAI/lambada_openai",
123
+ "dataset_name": "default",
124
+ "test_split": "test",
125
+ "doc_to_text": "{{text.split(' ')[:-1]|join(' ')}}",
126
+ "doc_to_target": "{{' '+text.split(' ')[-1]}}",
127
+ "description": "",
128
+ "target_delimiter": " ",
129
+ "fewshot_delimiter": "\n\n",
130
+ "num_fewshot": 0,
131
+ "metric_list": [
132
+ {
133
+ "metric": "perplexity",
134
+ "aggregation": "perplexity",
135
+ "higher_is_better": false
136
+ },
137
+ {
138
+ "metric": "acc",
139
+ "aggregation": "mean",
140
+ "higher_is_better": true
141
+ }
142
+ ],
143
+ "output_type": "loglikelihood",
144
+ "repeats": 1,
145
+ "should_decontaminate": true,
146
+ "doc_to_decontamination_query": "{{text}}"
147
+ },
148
+ "logiqa": {
149
+ "task": "logiqa",
150
+ "dataset_path": "EleutherAI/logiqa",
151
+ "dataset_name": "logiqa",
152
+ "training_split": "train",
153
+ "validation_split": "validation",
154
+ "test_split": "test",
155
+ "doc_to_text": "<function doc_to_text at 0x7f015d114d30>",
156
+ "doc_to_target": "<function doc_to_target at 0x7f015d115090>",
157
+ "doc_to_choice": "{{options}}",
158
+ "description": "",
159
+ "target_delimiter": " ",
160
+ "fewshot_delimiter": "\n\n",
161
+ "num_fewshot": 0,
162
+ "metric_list": [
163
+ {
164
+ "metric": "acc",
165
+ "aggregation": "mean",
166
+ "higher_is_better": true
167
+ },
168
+ {
169
+ "metric": "acc_norm",
170
+ "aggregation": "mean",
171
+ "higher_is_better": true
172
+ }
173
+ ],
174
+ "output_type": "multiple_choice",
175
+ "repeats": 1,
176
+ "should_decontaminate": true,
177
+ "doc_to_decontamination_query": "{{context}}"
178
+ },
179
+ "piqa": {
180
+ "task": "piqa",
181
+ "dataset_path": "piqa",
182
+ "training_split": "train",
183
+ "validation_split": "validation",
184
+ "doc_to_text": "Question: {{goal}}\nAnswer:",
185
+ "doc_to_target": "label",
186
+ "doc_to_choice": "{{[sol1, sol2]}}",
187
+ "description": "",
188
+ "target_delimiter": " ",
189
+ "fewshot_delimiter": "\n\n",
190
+ "num_fewshot": 0,
191
+ "metric_list": [
192
+ {
193
+ "metric": "acc",
194
+ "aggregation": "mean",
195
+ "higher_is_better": true
196
+ },
197
+ {
198
+ "metric": "acc_norm",
199
+ "aggregation": "mean",
200
+ "higher_is_better": true
201
+ }
202
+ ],
203
+ "output_type": "multiple_choice",
204
+ "repeats": 1,
205
+ "should_decontaminate": true,
206
+ "doc_to_decontamination_query": "goal"
207
+ },
208
+ "sciq": {
209
+ "task": "sciq",
210
+ "dataset_path": "sciq",
211
+ "training_split": "train",
212
+ "validation_split": "validation",
213
+ "test_split": "test",
214
+ "doc_to_text": "{{support.lstrip()}}\nQuestion: {{question}}\nAnswer:",
215
+ "doc_to_target": 3,
216
+ "doc_to_choice": "{{[distractor1, distractor2, distractor3, correct_answer]}}",
217
+ "description": "",
218
+ "target_delimiter": " ",
219
+ "fewshot_delimiter": "\n\n",
220
+ "num_fewshot": 0,
221
+ "metric_list": [
222
+ {
223
+ "metric": "acc",
224
+ "aggregation": "mean",
225
+ "higher_is_better": true
226
+ },
227
+ {
228
+ "metric": "acc_norm",
229
+ "aggregation": "mean",
230
+ "higher_is_better": true
231
+ }
232
+ ],
233
+ "output_type": "multiple_choice",
234
+ "repeats": 1,
235
+ "should_decontaminate": true,
236
+ "doc_to_decontamination_query": "{{support}} {{question}}"
237
+ },
238
+ "winogrande": {
239
+ "task": "winogrande",
240
+ "dataset_path": "winogrande",
241
+ "dataset_name": "winogrande_xl",
242
+ "training_split": "train",
243
+ "validation_split": "validation",
244
+ "doc_to_text": "<function doc_to_text at 0x7f015cf88f70>",
245
+ "doc_to_target": "<function doc_to_target at 0x7f015cf892d0>",
246
+ "doc_to_choice": "<function doc_to_choice at 0x7f015cf89630>",
247
+ "description": "",
248
+ "target_delimiter": " ",
249
+ "fewshot_delimiter": "\n\n",
250
+ "num_fewshot": 0,
251
+ "metric_list": [
252
+ {
253
+ "metric": "acc",
254
+ "aggregation": "mean",
255
+ "higher_is_better": true
256
+ }
257
+ ],
258
+ "output_type": "multiple_choice",
259
+ "repeats": 1,
260
+ "should_decontaminate": true,
261
+ "doc_to_decontamination_query": "sentence"
262
+ },
263
+ "wsc": {
264
+ "task": "wsc",
265
+ "group": [
266
+ "super-glue-lm-eval-v1"
267
+ ],
268
+ "dataset_path": "super_glue",
269
+ "dataset_name": "wsc.fixed",
270
+ "training_split": "train",
271
+ "validation_split": "validation",
272
+ "doc_to_text": "<function default_doc_to_text at 0x7f015d0149d0>",
273
+ "doc_to_target": "label",
274
+ "doc_to_choice": [
275
+ "no",
276
+ "yes"
277
+ ],
278
+ "description": "",
279
+ "target_delimiter": " ",
280
+ "fewshot_delimiter": "\n\n",
281
+ "num_fewshot": 0,
282
+ "metric_list": [
283
+ {
284
+ "metric": "acc"
285
+ }
286
+ ],
287
+ "output_type": "multiple_choice",
288
+ "repeats": 1,
289
+ "should_decontaminate": false
290
+ }
291
+ },
292
+ "versions": {
293
+ "arc_challenge": "Yaml",
294
+ "arc_easy": "Yaml",
295
+ "lambada_openai": "Yaml",
296
+ "logiqa": "Yaml",
297
+ "piqa": "Yaml",
298
+ "sciq": "Yaml",
299
+ "winogrande": "Yaml",
300
+ "wsc": "Yaml"
301
+ },
302
+ "config": {
303
+ "model": "hf",
304
+ "model_args": "pretrained=usvsnsp/pythia-70m-ppo",
305
+ "batch_size": 1,
306
+ "batch_sizes": [],
307
+ "device": null,
308
+ "use_cache": null,
309
+ "limit": null,
310
+ "bootstrap_iters": 100000,
311
+ "gen_kwargs": ""
312
+ },
313
+ "git_hash": "b8d7d6c3"
314
+ }