Commit
•
08eb31b
1
Parent(s):
aac53cc
Add files
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .gitattributes +2 -0
- config.json +30 -0
- evaluation_l1/Muennighoff_xstory_cloze/ar/Answer_Given_options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/ar/Novel_Correct_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/ar/Story_Continuation_and_Options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/eu/Answer_Given_options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/eu/Choose_Story_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/eu/Generate_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/eu/Novel_Correct_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/hi/Answer_Given_options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/hi/Choose_Story_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/hi/Generate_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/hi/Novel_Correct_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/hi/Story_Continuation_and_Options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/id/Answer_Given_options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/id/Choose_Story_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/id/Generate_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/id/Novel_Correct_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/id/Story_Continuation_and_Options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/zh/Answer_Given_options/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/zh/Choose_Story_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/zh/Generate_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/zh/Novel_Correct_Ending/results.json +9 -0
- evaluation_l1/Muennighoff_xstory_cloze/zh/Story_Continuation_and_Options/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/en/Replace/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/en/True_or_False/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/en/does_underscore_refer_to/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/en/stand_for/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/en/underscore_refer_to/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/fr/Replace/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/fr/True_or_False/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/fr/does_underscore_refer_to/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/fr/stand_for/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/pt/Replace/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/pt/True_or_False/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/pt/does_underscore_refer_to/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/pt/stand_for/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/pt/underscore_refer_to/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/zh/Replace/results.json +9 -0
- evaluation_l1/Muennighoff_xwinograd/zh/underscore_refer_to/results.json +9 -0
- evaluation_l1/anli/dev_r1/GPT-3_style/results.json +9 -0
- evaluation_l1/anli/dev_r1/MNLI_crowdsource/results.json +9 -0
- evaluation_l1/anli/dev_r1/can_we_infer/results.json +9 -0
- evaluation_l1/anli/dev_r1/guaranteed_possible_impossible/results.json +9 -0
- evaluation_l1/anli/dev_r1/justified_in_saying/results.json +9 -0
- evaluation_l1/anli/dev_r2/GPT-3_style/results.json +9 -0
- evaluation_l1/anli/dev_r2/MNLI_crowdsource/results.json +9 -0
- evaluation_l1/anli/dev_r2/can_we_infer/results.json +9 -0
- evaluation_l1/anli/dev_r2/guaranteed_possible_impossible/results.json +9 -0
- evaluation_l1/anli/dev_r2/justified_in_saying/results.json +9 -0
.gitattributes
CHANGED
@@ -30,3 +30,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
30 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
31 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
32 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
30 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
31 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
32 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
33 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
34 |
+
logs/logs/xp3capmixnewcodelonglossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text
|
config.json
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"apply_residual_connection_post_layernorm": false,
|
3 |
+
"attention_dropout": 0.0,
|
4 |
+
"attention_softmax_in_fp32": true,
|
5 |
+
"bias_dropout_fusion": true,
|
6 |
+
"architectures": [
|
7 |
+
"BloomModel"
|
8 |
+
],
|
9 |
+
"bos_token_id": 1,
|
10 |
+
"eos_token_id": 2,
|
11 |
+
"pad_token_id": 3,
|
12 |
+
"unk_token_id": 0,
|
13 |
+
"hidden_dropout": 0.0,
|
14 |
+
"initializer_range": 0.02,
|
15 |
+
"layer_norm_epsilon": 1e-05,
|
16 |
+
"masked_softmax_fusion": true,
|
17 |
+
"model_type": "bloom",
|
18 |
+
"n_embed": 2560,
|
19 |
+
"n_inner": null,
|
20 |
+
"n_layer": 30,
|
21 |
+
"num_attention_heads": 32,
|
22 |
+
"offset_alibi": 100,
|
23 |
+
"pretraining_tp": 4,
|
24 |
+
"seq_length": 2048,
|
25 |
+
"skip_bias_add": true,
|
26 |
+
"skip_bias_add_qkv": false,
|
27 |
+
"transformers_version": "4.20.0",
|
28 |
+
"use_cache": true,
|
29 |
+
"vocab_size": 250880
|
30 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/ar/Answer_Given_options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "ar",
|
4 |
+
"template_name": "Answer Given options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7511581733951026
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/ar/Novel_Correct_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "ar",
|
4 |
+
"template_name": "Novel Correct Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7822634017207147
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/ar/Story_Continuation_and_Options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "ar",
|
4 |
+
"template_name": "Story Continuation and Options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7908669755129054
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/eu/Answer_Given_options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "eu",
|
4 |
+
"template_name": "Answer Given options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.6029119788219722
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/eu/Choose_Story_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "eu",
|
4 |
+
"template_name": "Choose Story Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7094639311714097
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/eu/Generate_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "eu",
|
4 |
+
"template_name": "Generate Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5314361350099271
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/eu/Novel_Correct_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "eu",
|
4 |
+
"template_name": "Novel Correct Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.6743878226340172
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/hi/Answer_Given_options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "hi",
|
4 |
+
"template_name": "Answer Given options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.6836532097948379
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/hi/Choose_Story_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "hi",
|
4 |
+
"template_name": "Choose Story Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7888815354070152
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/hi/Generate_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "hi",
|
4 |
+
"template_name": "Generate Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5810721376571807
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/hi/Novel_Correct_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "hi",
|
4 |
+
"template_name": "Novel Correct Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7597617471872932
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/hi/Story_Continuation_and_Options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "hi",
|
4 |
+
"template_name": "Story Continuation and Options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7723362011912641
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/id/Answer_Given_options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "id",
|
4 |
+
"template_name": "Answer Given options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7127729980145598
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/id/Choose_Story_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "id",
|
4 |
+
"template_name": "Choose Story Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.8299139642620781
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/id/Generate_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "id",
|
4 |
+
"template_name": "Generate Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.6035737921906023
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/id/Novel_Correct_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "id",
|
4 |
+
"template_name": "Novel Correct Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.7180675049636003
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/id/Story_Continuation_and_Options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "id",
|
4 |
+
"template_name": "Story Continuation and Options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.8021178027796162
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/zh/Answer_Given_options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "zh",
|
4 |
+
"template_name": "Answer Given options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.786234281932495
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/zh/Choose_Story_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "zh",
|
4 |
+
"template_name": "Choose Story Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.8431502316346791
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/zh/Generate_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "zh",
|
4 |
+
"template_name": "Generate Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.6022501654533422
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/zh/Novel_Correct_Ending/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "zh",
|
4 |
+
"template_name": "Novel Correct Ending",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.8365320979483786
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xstory_cloze/zh/Story_Continuation_and_Options/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xstory_cloze",
|
3 |
+
"dataset_config_name": "zh",
|
4 |
+
"template_name": "Story Continuation and Options",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.8259430840502978
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/en/Replace/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "en",
|
4 |
+
"template_name": "Replace",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5922580645161291
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/en/True_or_False/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "en",
|
4 |
+
"template_name": "True or False",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5333333333333333
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/en/does_underscore_refer_to/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "en",
|
4 |
+
"template_name": "does underscore refer to",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5483870967741935
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/en/stand_for/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "en",
|
4 |
+
"template_name": "stand for",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5006451612903225
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/en/underscore_refer_to/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "en",
|
4 |
+
"template_name": "underscore refer to",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5574193548387096
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/fr/Replace/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "fr",
|
4 |
+
"template_name": "Replace",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.4939759036144578
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/fr/True_or_False/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "fr",
|
4 |
+
"template_name": "True or False",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5060240963855421
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/fr/does_underscore_refer_to/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "fr",
|
4 |
+
"template_name": "does underscore refer to",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.4819277108433735
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/fr/stand_for/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "fr",
|
4 |
+
"template_name": "stand for",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.4939759036144578
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/pt/Replace/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "pt",
|
4 |
+
"template_name": "Replace",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5285171102661597
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/pt/True_or_False/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "pt",
|
4 |
+
"template_name": "True or False",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5361216730038023
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/pt/does_underscore_refer_to/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "pt",
|
4 |
+
"template_name": "does underscore refer to",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5247148288973384
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/pt/stand_for/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "pt",
|
4 |
+
"template_name": "stand for",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.4828897338403042
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/pt/underscore_refer_to/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "pt",
|
4 |
+
"template_name": "underscore refer to",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5019011406844106
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/zh/Replace/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "zh",
|
4 |
+
"template_name": "Replace",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.6091269841269841
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/Muennighoff_xwinograd/zh/underscore_refer_to/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "Muennighoff/xwinograd",
|
3 |
+
"dataset_config_name": "zh",
|
4 |
+
"template_name": "underscore refer to",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.5674603174603174
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r1/GPT-3_style/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r1",
|
4 |
+
"template_name": "GPT-3 style",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.401
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r1/MNLI_crowdsource/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r1",
|
4 |
+
"template_name": "MNLI crowdsource",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.399
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r1/can_we_infer/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r1",
|
4 |
+
"template_name": "can we infer",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.386
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r1/guaranteed_possible_impossible/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r1",
|
4 |
+
"template_name": "guaranteed/possible/impossible",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.33
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r1/justified_in_saying/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r1",
|
4 |
+
"template_name": "justified in saying",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.374
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='justified in saying', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r2/GPT-3_style/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r2",
|
4 |
+
"template_name": "GPT-3 style",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.355
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r2/MNLI_crowdsource/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r2",
|
4 |
+
"template_name": "MNLI crowdsource",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.368
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r2/can_we_infer/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r2",
|
4 |
+
"template_name": "can we infer",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.335
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r2/guaranteed_possible_impossible/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r2",
|
4 |
+
"template_name": "guaranteed/possible/impossible",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.331
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|
evaluation_l1/anli/dev_r2/justified_in_saying/results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"dataset_name": "anli",
|
3 |
+
"dataset_config_name": "dev_r2",
|
4 |
+
"template_name": "justified in saying",
|
5 |
+
"evaluation": {
|
6 |
+
"accuracy": 0.346
|
7 |
+
},
|
8 |
+
"arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='justified in saying', tokenizer_name=None, use_slow_tokenizer=False)"
|
9 |
+
}
|