Muennighoff commited on
Commit
08eb31b
1 Parent(s): aac53cc
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. config.json +30 -0
  3. evaluation_l1/Muennighoff_xstory_cloze/ar/Answer_Given_options/results.json +9 -0
  4. evaluation_l1/Muennighoff_xstory_cloze/ar/Novel_Correct_Ending/results.json +9 -0
  5. evaluation_l1/Muennighoff_xstory_cloze/ar/Story_Continuation_and_Options/results.json +9 -0
  6. evaluation_l1/Muennighoff_xstory_cloze/eu/Answer_Given_options/results.json +9 -0
  7. evaluation_l1/Muennighoff_xstory_cloze/eu/Choose_Story_Ending/results.json +9 -0
  8. evaluation_l1/Muennighoff_xstory_cloze/eu/Generate_Ending/results.json +9 -0
  9. evaluation_l1/Muennighoff_xstory_cloze/eu/Novel_Correct_Ending/results.json +9 -0
  10. evaluation_l1/Muennighoff_xstory_cloze/hi/Answer_Given_options/results.json +9 -0
  11. evaluation_l1/Muennighoff_xstory_cloze/hi/Choose_Story_Ending/results.json +9 -0
  12. evaluation_l1/Muennighoff_xstory_cloze/hi/Generate_Ending/results.json +9 -0
  13. evaluation_l1/Muennighoff_xstory_cloze/hi/Novel_Correct_Ending/results.json +9 -0
  14. evaluation_l1/Muennighoff_xstory_cloze/hi/Story_Continuation_and_Options/results.json +9 -0
  15. evaluation_l1/Muennighoff_xstory_cloze/id/Answer_Given_options/results.json +9 -0
  16. evaluation_l1/Muennighoff_xstory_cloze/id/Choose_Story_Ending/results.json +9 -0
  17. evaluation_l1/Muennighoff_xstory_cloze/id/Generate_Ending/results.json +9 -0
  18. evaluation_l1/Muennighoff_xstory_cloze/id/Novel_Correct_Ending/results.json +9 -0
  19. evaluation_l1/Muennighoff_xstory_cloze/id/Story_Continuation_and_Options/results.json +9 -0
  20. evaluation_l1/Muennighoff_xstory_cloze/zh/Answer_Given_options/results.json +9 -0
  21. evaluation_l1/Muennighoff_xstory_cloze/zh/Choose_Story_Ending/results.json +9 -0
  22. evaluation_l1/Muennighoff_xstory_cloze/zh/Generate_Ending/results.json +9 -0
  23. evaluation_l1/Muennighoff_xstory_cloze/zh/Novel_Correct_Ending/results.json +9 -0
  24. evaluation_l1/Muennighoff_xstory_cloze/zh/Story_Continuation_and_Options/results.json +9 -0
  25. evaluation_l1/Muennighoff_xwinograd/en/Replace/results.json +9 -0
  26. evaluation_l1/Muennighoff_xwinograd/en/True_or_False/results.json +9 -0
  27. evaluation_l1/Muennighoff_xwinograd/en/does_underscore_refer_to/results.json +9 -0
  28. evaluation_l1/Muennighoff_xwinograd/en/stand_for/results.json +9 -0
  29. evaluation_l1/Muennighoff_xwinograd/en/underscore_refer_to/results.json +9 -0
  30. evaluation_l1/Muennighoff_xwinograd/fr/Replace/results.json +9 -0
  31. evaluation_l1/Muennighoff_xwinograd/fr/True_or_False/results.json +9 -0
  32. evaluation_l1/Muennighoff_xwinograd/fr/does_underscore_refer_to/results.json +9 -0
  33. evaluation_l1/Muennighoff_xwinograd/fr/stand_for/results.json +9 -0
  34. evaluation_l1/Muennighoff_xwinograd/pt/Replace/results.json +9 -0
  35. evaluation_l1/Muennighoff_xwinograd/pt/True_or_False/results.json +9 -0
  36. evaluation_l1/Muennighoff_xwinograd/pt/does_underscore_refer_to/results.json +9 -0
  37. evaluation_l1/Muennighoff_xwinograd/pt/stand_for/results.json +9 -0
  38. evaluation_l1/Muennighoff_xwinograd/pt/underscore_refer_to/results.json +9 -0
  39. evaluation_l1/Muennighoff_xwinograd/zh/Replace/results.json +9 -0
  40. evaluation_l1/Muennighoff_xwinograd/zh/underscore_refer_to/results.json +9 -0
  41. evaluation_l1/anli/dev_r1/GPT-3_style/results.json +9 -0
  42. evaluation_l1/anli/dev_r1/MNLI_crowdsource/results.json +9 -0
  43. evaluation_l1/anli/dev_r1/can_we_infer/results.json +9 -0
  44. evaluation_l1/anli/dev_r1/guaranteed_possible_impossible/results.json +9 -0
  45. evaluation_l1/anli/dev_r1/justified_in_saying/results.json +9 -0
  46. evaluation_l1/anli/dev_r2/GPT-3_style/results.json +9 -0
  47. evaluation_l1/anli/dev_r2/MNLI_crowdsource/results.json +9 -0
  48. evaluation_l1/anli/dev_r2/can_we_infer/results.json +9 -0
  49. evaluation_l1/anli/dev_r2/guaranteed_possible_impossible/results.json +9 -0
  50. evaluation_l1/anli/dev_r2/justified_in_saying/results.json +9 -0
.gitattributes CHANGED
@@ -30,3 +30,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
30
  *.zip filter=lfs diff=lfs merge=lfs -text
31
  *.zst filter=lfs diff=lfs merge=lfs -text
32
  *tfevents* filter=lfs diff=lfs merge=lfs -text
33
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
34
+ logs/logs/xp3capmixnewcodelonglossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "apply_residual_connection_post_layernorm": false,
3
+ "attention_dropout": 0.0,
4
+ "attention_softmax_in_fp32": true,
5
+ "bias_dropout_fusion": true,
6
+ "architectures": [
7
+ "BloomModel"
8
+ ],
9
+ "bos_token_id": 1,
10
+ "eos_token_id": 2,
11
+ "pad_token_id": 3,
12
+ "unk_token_id": 0,
13
+ "hidden_dropout": 0.0,
14
+ "initializer_range": 0.02,
15
+ "layer_norm_epsilon": 1e-05,
16
+ "masked_softmax_fusion": true,
17
+ "model_type": "bloom",
18
+ "n_embed": 2560,
19
+ "n_inner": null,
20
+ "n_layer": 30,
21
+ "num_attention_heads": 32,
22
+ "offset_alibi": 100,
23
+ "pretraining_tp": 4,
24
+ "seq_length": 2048,
25
+ "skip_bias_add": true,
26
+ "skip_bias_add_qkv": false,
27
+ "transformers_version": "4.20.0",
28
+ "use_cache": true,
29
+ "vocab_size": 250880
30
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.7511581733951026
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.7822634017207147
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/ar/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "ar",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.7908669755129054
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.6029119788219722
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.7094639311714097
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5314361350099271
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/eu/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "eu",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.6743878226340172
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.6836532097948379
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.7888815354070152
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.5810721376571807
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.7597617471872932
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/hi/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "hi",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.7723362011912641
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/id/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "id",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.7127729980145598
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/id/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "id",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.8299139642620781
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/id/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "id",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.6035737921906023
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/id/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "id",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.7180675049636003
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/id/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "id",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.8021178027796162
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Answer_Given_options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Answer Given options",
5
+ "evaluation": {
6
+ "accuracy": 0.786234281932495
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Choose_Story_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Choose Story Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.8431502316346791
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Generate_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Generate Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.6022501654533422
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Novel_Correct_Ending/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Novel Correct Ending",
5
+ "evaluation": {
6
+ "accuracy": 0.8365320979483786
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xstory_cloze/zh/Story_Continuation_and_Options/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xstory_cloze",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Story Continuation and Options",
5
+ "evaluation": {
6
+ "accuracy": 0.8259430840502978
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/en/Replace/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "en",
4
+ "template_name": "Replace",
5
+ "evaluation": {
6
+ "accuracy": 0.5922580645161291
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/en/True_or_False/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "en",
4
+ "template_name": "True or False",
5
+ "evaluation": {
6
+ "accuracy": 0.5333333333333333
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/en/does_underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "en",
4
+ "template_name": "does underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5483870967741935
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/en/stand_for/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "en",
4
+ "template_name": "stand for",
5
+ "evaluation": {
6
+ "accuracy": 0.5006451612903225
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/en/underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "en",
4
+ "template_name": "underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5574193548387096
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/Replace/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "Replace",
5
+ "evaluation": {
6
+ "accuracy": 0.4939759036144578
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/True_or_False/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "True or False",
5
+ "evaluation": {
6
+ "accuracy": 0.5060240963855421
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/does_underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "does underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.4819277108433735
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/fr/stand_for/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "fr",
4
+ "template_name": "stand for",
5
+ "evaluation": {
6
+ "accuracy": 0.4939759036144578
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/Replace/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "Replace",
5
+ "evaluation": {
6
+ "accuracy": 0.5285171102661597
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/True_or_False/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "True or False",
5
+ "evaluation": {
6
+ "accuracy": 0.5361216730038023
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/does_underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "does underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5247148288973384
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/stand_for/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "stand for",
5
+ "evaluation": {
6
+ "accuracy": 0.4828897338403042
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/pt/underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "pt",
4
+ "template_name": "underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5019011406844106
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/zh/Replace/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "Replace",
5
+ "evaluation": {
6
+ "accuracy": 0.6091269841269841
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/Muennighoff_xwinograd/zh/underscore_refer_to/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "Muennighoff/xwinograd",
3
+ "dataset_config_name": "zh",
4
+ "template_name": "underscore refer to",
5
+ "evaluation": {
6
+ "accuracy": 0.5674603174603174
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/GPT-3_style/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "GPT-3 style",
5
+ "evaluation": {
6
+ "accuracy": 0.401
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/MNLI_crowdsource/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "MNLI crowdsource",
5
+ "evaluation": {
6
+ "accuracy": 0.399
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/can_we_infer/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "can we infer",
5
+ "evaluation": {
6
+ "accuracy": 0.386
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/guaranteed_possible_impossible/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "guaranteed/possible/impossible",
5
+ "evaluation": {
6
+ "accuracy": 0.33
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r1/justified_in_saying/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r1",
4
+ "template_name": "justified in saying",
5
+ "evaluation": {
6
+ "accuracy": 0.374
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='justified in saying', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/GPT-3_style/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "GPT-3 style",
5
+ "evaluation": {
6
+ "accuracy": 0.355
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/MNLI_crowdsource/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "MNLI crowdsource",
5
+ "evaluation": {
6
+ "accuracy": 0.368
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/can_we_infer/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "can we infer",
5
+ "evaluation": {
6
+ "accuracy": 0.335
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/guaranteed_possible_impossible/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "guaranteed/possible/impossible",
5
+ "evaluation": {
6
+ "accuracy": 0.331
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }
evaluation_l1/anli/dev_r2/justified_in_saying/results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset_name": "anli",
3
+ "dataset_config_name": "dev_r2",
4
+ "template_name": "justified in saying",
5
+ "evaluation": {
6
+ "accuracy": 0.346
7
+ },
8
+ "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='justified in saying', tokenizer_name=None, use_slow_tokenizer=False)"
9
+ }