Add files using upload-large-folder tool

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml

@@ -0,0 +1,3 @@
+dataset_name: fra
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_fra

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml

@@ -0,0 +1,3 @@
+dataset_name: hau
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_hau

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml

@@ -0,0 +1,3 @@
+dataset_name: lin
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_lin

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml

@@ -0,0 +1,3 @@
+dataset_name: lug
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_lug

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml

@@ -0,0 +1,3 @@
+dataset_name: orm
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_orm

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml

@@ -0,0 +1,3 @@
+dataset_name: sna
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_sna

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml

@@ -0,0 +1,3 @@
+dataset_name: sot
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_sot

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml

@@ -0,0 +1,3 @@
+dataset_name: swa
+include: afrimmlu_common_yaml
+task: afrimmlu_direct_swa

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/direct/utils.py

@@ -0,0 +1,32 @@
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """You are a highly knowledgeable and intelligent artificial intelligence
+                model answers multiple-choice questions about {subject}
+
+                Question: {question}
+
+                Choices:
+                        A: {choice1}
+                        B: {choice2}
+                        C: {choice3}
+                        D: {choice4}
+
+                Answer:  """
+
+    choices = eval(doc["choices"])
+    text = output.format(
+        subject=doc["subject"],
+        question=doc["question"],
+        choice1=choices[0],
+        choice2=choices[1],
+        choice3=choices[2],
+        choice4=choices[3],
+    )
+    return text

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml

@@ -0,0 +1,34 @@
+group:
+  - afrimmlu_translate
+task: null
+dataset_path: masakhane/afrimmlu-translate-test
+dataset_name: null
+output_type: multiple_choice
+test_split: test
+doc_to_text: !function utils.doc_to_text
+doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}"
+doc_to_choice: !function utils.doc_to_choice
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: f1
+    aggregation: !function utils.weighted_f1_score
+    # aggregation: mean
+    average: weighted
+    hf_evaluate: true
+    higher_is_better: True
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: true
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+metadata:
+  version: 1.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml

@@ -0,0 +1,3 @@
+dataset_name: amh
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_amh

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml

@@ -0,0 +1,3 @@
+dataset_name: eng
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_eng

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml

@@ -0,0 +1,3 @@
+dataset_name: ewe
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_ewe

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml

@@ -0,0 +1,3 @@
+dataset_name: fra
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_fra

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml

@@ -0,0 +1,3 @@
+dataset_name: hau
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_hau

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml

@@ -0,0 +1,3 @@
+dataset_name: ibo
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_ibo

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml

@@ -0,0 +1,3 @@
+dataset_name: kin
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_kin

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml

@@ -0,0 +1,3 @@
+dataset_name: lin
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_lin

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml

@@ -0,0 +1,3 @@
+dataset_name: lug
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_lug

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml

@@ -0,0 +1,3 @@
+dataset_name: orm
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_orm

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml

@@ -0,0 +1,3 @@
+dataset_name: sna
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_sna

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml

@@ -0,0 +1,3 @@
+dataset_name: sot
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_sot

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml

@@ -0,0 +1,3 @@
+dataset_name: swa
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_swa

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml

@@ -0,0 +1,3 @@
+dataset_name: twi
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_twi

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml

@@ -0,0 +1,3 @@
+dataset_name: wol
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_wol

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml

@@ -0,0 +1,3 @@
+dataset_name: xho
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_xho

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml

@@ -0,0 +1,3 @@
+dataset_name: yor
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_yor

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml

@@ -0,0 +1,3 @@
+dataset_name: zul
+include: afrimmlu_common_translate_yaml
+task: afrimmlu_translate_zul

scripts/yans/lm-evaluation-harness/lm_eval/tasks/afrimmlu/translate/utils.py

@@ -0,0 +1,32 @@
+from lm_eval.utils import weighted_f1_score
+
+
+def doc_to_choice(doc):
+    choices = eval(doc["choices"])
+    return choices
+
+
+def doc_to_text(doc):
+    output = """You are a highly knowledgeable and intelligent artificial intelligence
+                model answers multiple-choice questions about '{subject}'
+
+                Question: '''{question}'''
+
+                Choices:
+                        A: ''{choice1}'''
+                        B: ''{choice2}'''
+                        C: ''{choice3}'''
+                        D: ''{choice4}'''
+
+                Answer:  """
+
+    choices = eval(doc["choices"])
+    text = output.format(
+        subject=doc["subject"],
+        question=doc["question"],
+        choice1=choices[0],
+        choice2=choices[1],
+        choice3=choices[2],
+        choice4=choices[3],
+    )
+    return text

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/flan/_held_in_template_yaml

@@ -0,0 +1,14 @@
+output_type: generate_until
+test_split: null
+doc_to_choice: null
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+generation_kwargs:
+  until:
+    - "</s>"
+  do_sample: false
+  temperature: 0.0
+metadata:
+  version: 1.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml

@@ -0,0 +1,352 @@
+group: flan_held_in
+group_alias: Flan (Held-In)
+task:
+  # ANLI R1
+  - group: anli_r1_flan
+    group_alias: ANLI R1
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    task:
+      - task: anli_r1_prompt-0
+        task_alias: prompt-0
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-1
+        task_alias: prompt-1
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-2
+        task_alias: prompt-2
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-3
+        task_alias: prompt-3
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-4
+        task_alias: prompt-4
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-5
+        task_alias: prompt-5
+        include: _held_in_template_yaml
+        doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-6
+        task_alias: prompt-6
+        include: _held_in_template_yaml
+        doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-7
+        task_alias: prompt-7
+        include: _held_in_template_yaml
+        doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r1_prompt-8
+        task_alias: prompt-8
+        include: _held_in_template_yaml
+        doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+  # ANLI R2
+  - group: anli_r2_flan
+    group_alias: ANLI R2
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    task:
+      - task: anli_r2_prompt-0
+        task_alias: prompt-0
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-1
+        task_alias: prompt-1
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-2
+        task_alias: prompt-2
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-3
+        task_alias: prompt-3
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-4
+        task_alias: prompt-4
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-5
+        task_alias: prompt-5
+        include: _held_in_template_yaml
+        doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-6
+        task_alias: prompt-6
+        include: _held_in_template_yaml
+        doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-7
+        task_alias: prompt-7
+        include: _held_in_template_yaml
+        doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r2_prompt-8
+        task_alias: prompt-8
+        include: _held_in_template_yaml
+        doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+  # ANLI R3
+  - group: anli_r3_flan
+    group_alias: ANLI R3
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    task:
+      - task: anli_r3_prompt-0
+        task_alias: prompt-0
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-1
+        task_alias: prompt-1
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-2
+        task_alias: prompt-2
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-3
+        task_alias: prompt-3
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-4
+        task_alias: prompt-4
+        include: _held_in_template_yaml
+        doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-5
+        task_alias: prompt-5
+        include: _held_in_template_yaml
+        doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-6
+        task_alias: prompt-6
+        include: _held_in_template_yaml
+        doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-7
+        task_alias: prompt-7
+        include: _held_in_template_yaml
+        doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+      - task: anli_r3_prompt-8
+        task_alias: prompt-8
+        include: _held_in_template_yaml
+        doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No"
+        doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}"
+  # Arc Easy
+  - group: arc_easy_flan
+    group_alias: Arc Easy
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    task:
+      - task: arc_easy_prompt-0
+        task_alias: prompt-0
+        include: _held_in_template_yaml
+        doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_easy_prompt-1
+        task_alias: prompt-1
+        include: _held_in_template_yaml
+        doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_easy_prompt-2
+        task_alias: prompt-2
+        include: _held_in_template_yaml
+        doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_easy_prompt-3
+        task_alias: prompt-3
+        include: _held_in_template_yaml
+        doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_easy_prompt-4
+        task_alias: prompt-4
+        include: _held_in_template_yaml
+        doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_easy_prompt-5
+        task_alias: prompt-5
+        include: _held_in_template_yaml
+        doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_easy_prompt-6
+        task_alias: prompt-6
+        include: _held_in_template_yaml
+        doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+  # Arc Challenge
+  - group: arc_challenge_flan
+    group_alias: Arc Challenge
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    task:
+      - task: arc_challenge_prompt-0
+        task_alias: prompt-0
+        include: _held_in_template_yaml
+        doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_challenge_prompt-1
+        task_alias: prompt-1
+        include: _held_in_template_yaml
+        doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_challenge_prompt-2
+        task_alias: prompt-2
+        include: _held_in_template_yaml
+        doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_challenge_prompt-3
+        task_alias: prompt-3
+        include: _held_in_template_yaml
+        doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_challenge_prompt-4
+        task_alias: prompt-4
+        include: _held_in_template_yaml
+        doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_challenge_prompt-5
+        task_alias: prompt-5
+        include: _held_in_template_yaml
+        doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+      - task: arc_challenge_prompt-6
+        task_alias: prompt-6
+        include: _held_in_template_yaml
+        doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}"
+        doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}"
+  # BoolQ
+  - group: boolq_flan
+    group_alias: BoolQ
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    task:
+      - task: boolq_prompt-0
+        task_alias: prompt-0
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-1
+        task_alias: prompt-1
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-2
+        task_alias: prompt-2
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-3
+        task_alias: prompt-3
+        include: _held_in_template_yaml
+        doc_to_text: "Text: {{passage}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-4
+        task_alias: prompt-4
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-5
+        task_alias: prompt-5
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-6
+        task_alias: prompt-6
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-7
+        task_alias: prompt-7
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-8
+        task_alias: prompt-8
+        include: _held_in_template_yaml
+        doc_to_text: "{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+      - task: boolq_prompt-9
+        task_alias: prompt-9
+        include: _held_in_template_yaml
+        doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{passage}}\n\nOPTIONS:\n- no\n- yes"
+        doc_to_target: "{{['no', 'yes'][label]}}"
+  # RTE
+  - group: rte_flan
+    group_alias: RTE
+    aggregate_metric_list:
+      - metric: acc
+        weight_by_size: True
+    task:
+      - task: rte_prompt-0
+        task_alias: prompt-0
+        include: _held_in_template_yaml
+        doc_to_text: "{{sentence1}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{sentence2}}\"?\n\nOPTIONS:\n- yes\n- no"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-1
+        task_alias: prompt-1
+        include: _held_in_template_yaml
+        doc_to_text: "{{sentence1}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-1
+        task_alias: prompt-2
+        include: _held_in_template_yaml
+        doc_to_text: "{{sentence1}}\n\nQ with options: Can we draw the following conclusion?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-3
+        task_alias: prompt-3
+        include: _held_in_template_yaml
+        doc_to_text: "{{sentence1}}\nDoes this next sentence follow, given the preceding text?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-4
+        task_alias: prompt-4
+        include: _held_in_template_yaml
+        doc_to_text: "{{sentence1}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{sentence2}}"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-5
+        task_alias: prompt-5
+        include: _held_in_template_yaml
+        doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nThe answer is"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-6
+        task_alias: prompt-6
+        include: _held_in_template_yaml
+        doc_to_text: "Read the text and determine if the sentence is true:\n\n{{sentence1}}\n\nSentence: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-7
+        task_alias: prompt-7
+        include: _held_in_template_yaml
+        doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:"
+        doc_to_target: "{{['yes', 'no'][label]}}"
+      - task: rte_prompt-8
+        task_alias: prompt-8
+        include: _held_in_template_yaml
+        doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{sentence2}}\n\n{{sentence1}}\nOPTIONS:\n- yes\n- no"
+        doc_to_target: "{{['yes', 'no'][label]}}"

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/flan/flan_held_out.yaml

@@ -0,0 +1,13 @@
+group: flan_held_out
+task:
+  # BBH
+  - bbh_zeroshot
+  - bbh_fewshot
+  - bbh_cot_fewshot
+  - bbh_cot_zeroshot
+  # MMLU
+  - mmlu
+  - mmlu_flan_n_shot_generative
+  - mmlu_flan_n_shot_loglikelihood
+  - mmlu_flan_cot_zeroshot
+  - mmlu_flan_cot_fewshot

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/minerva_math.yaml

@@ -0,0 +1,15 @@
+group: minerva_math
+task:
+  - minerva_math_algebra
+  - minerva_math_counting_and_prob
+  - minerva_math_geometry
+  - minerva_math_intermediate_algebra
+  - minerva_math_num_theory
+  - minerva_math_prealgebra
+  - minerva_math_precalc
+aggregate_metric_list:
+  - metric: exact_match
+    aggregation: mean
+    weight_by_size: true
+metadata:
+  version: 1.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/multimedqa/README.md

@@ -0,0 +1,43 @@
+# MultiMedQA (multiple-choice subset)
+
+### Paper
+
+Title: Large Language Models Encode Clinical Knowledge
+
+Abstract: https://arxiv.org/abs/2212.13138
+
+A benchmark combining four existing multiple-choice question answering datasets spanning professional medical exams and research queries.
+
+### Citation
+
+```
+@Article{Singhal2023,
+author={Singhal, Karan and Azizi, Shekoofeh and Tu, Tao and Mahdavi, S. Sara and Wei, Jason and Chung, Hyung Won and Scales, Nathan and Tanwani, Ajay and Cole-Lewis, Heather and Pfohl, Stephen and Payne, Perry and Seneviratne, Martin and Gamble, Paul and Kelly, Chris and Babiker, Abubakr and Sch{\"a}rli, Nathanael and Chowdhery, Aakanksha and Mansfield, Philip and Demner-Fushman, Dina and Ag{\"u}era y Arcas, Blaise and Webster, Dale and Corrado, Greg S. and Matias, Yossi and Chou, Katherine and Gottweis, Juraj and Tomasev, Nenad and Liu, Yun and Rajkomar, Alvin and Barral, Joelle and Semturs, Christopher and Karthikesalingam, Alan and Natarajan, Vivek},
+title={Large language models encode clinical knowledge},
+journal={Nature},
+year={2023},
+month={Aug},
+day={01},
+volume={620},
+number={7972},
+pages={172-180},
+issn={1476-4687},
+doi={10.1038/s41586-023-06291-2},
+url={https://doi.org/10.1038/s41586-023-06291-2}
+}
+```
+
+### Tasks
+
+* [PubMedQA](https://pubmedqa.github.io/) - 1,000 expert-labeled Q&A pairs where a question and corresponding PubMed abstract as context is given and the a yes/maybe/no answer must be produced. Unlike the rest of the tasks in this suite, PubMedQA is a closed-domain Q&A task.
+* [MedQA](https://github.com/jind11/MedQA) - US Medical License Exam (USMLE) questions with 4 or 5 possible answers. Typically, only the 4-option questions are used.
+* [MedMCQA](https://medmcqa.github.io/) - 4-option multiple choice questions from Indian medical entrance examinations, >191k total questions.
+* [MMLU](https://arxiv.org/abs/2009.03300) - 4-option multiple choice exam questions from a variety of domains. The following 6 domains are utilized here:
+	* Anatomy
+	* Clinical Knowledge
+	* College Medicine
+	* Medical Genetics
+	* Professional Medicine
+	* College Biology
+
+Note that MultiMedQA also includes some short-form and long-form Q&A tasks (LiveQA, MedicationQA, HealthSearchQA). Evaluation on these tasks is usually done by experts and is not typically performed automatically, and therefore is ignored here.

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml

@@ -0,0 +1,21 @@
+group: multimedqa
+task:
+  - pubmedqa
+  - medmcqa
+  - medqa_4options
+  - task: mmlu_anatomy
+    task_alias: "anatomy (mmlu)"
+  - task: mmlu_clinical_knowledge
+    task_alias: "clinical_knowledge (mmlu)"
+  - task: mmlu_college_medicine
+    task_alias: "college_medicine (mmlu)"
+  - task: mmlu_medical_genetics
+    task_alias: "medical_genetics (mmlu)"
+  - task: mmlu_professional_medicine
+    task_alias: "professional_medicine (mmlu)"
+  - task: mmlu_college_biology
+    task_alias: "college_biology (mmlu)"
+aggregate_metric_list:
+  - metric: acc
+    aggregation: mean
+    weight_by_size: True

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/openllm.yaml

@@ -0,0 +1,18 @@
+group: openllm
+group_alias: Open LLM Leaderboard
+task:
+  - task: arc_challenge
+    fewshot_split: validation
+    num_fewshot: 25
+  - task: hellaswag
+    fewshot_split: train
+    num_fewshot: 10
+  - task: truthfulqa
+    num_fewshot: 0
+  - task: mmlu
+    num_fewshot: 5
+  - task: winogrande
+    fewshot_split: train
+    num_fewshot: 5
+  - task: gsm8k
+    num_fewshot: 5

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/pythia.yaml

@@ -0,0 +1,12 @@
+group: pythia
+task:
+  - lambada_openai
+  - logiqa
+  - piqa
+  - sciq
+  - wikitext
+  - winogrande
+  - wsc
+  - ai2_arc
+  - blimp
+  - mmlu

scripts/yans/lm-evaluation-harness/lm_eval/tasks/benchmarks/t0_eval.yaml

@@ -0,0 +1,127 @@
+group: t0_eval
+task:
+  # Coreference Resolution
+  - dataset_path: super_glue
+    dataset_name: wsc.fixed
+    use_prompt: promptsource:*
+    training_split: train
+    validation_split: validation
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  # Coreference Resolution
+  - dataset_path: winogrande
+    dataset_name: winogrande_xl
+    use_prompt: promptsource:*
+    training_split: train
+    validation_split: validation
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  # Natural Language Inference
+  - dataset_path: super_glue
+    dataset_name: cb
+    use_prompt: promptsource:*
+    training_split: train
+    validation_split: validation
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  - dataset_path: super_glue
+    dataset_name: rte
+    use_prompt: promptsource:*
+    training_split: train
+    validation_split: validation
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  - task: anli_r1
+    dataset_path: anli
+    use_prompt: promptsource:*
+    training_split: train_r1
+    validation_split: dev_r1
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  - task: anli_r2
+    dataset_path: anli
+    use_prompt: promptsource:*
+    training_split: train_r2
+    validation_split: dev_r2
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  - task: anli_r3
+    dataset_path: anli
+    use_prompt: promptsource:*
+    training_split: train_r3
+    validation_split: dev_r3
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  # Sentence Completion
+  - dataset_path: super_glue
+    dataset_name: copa
+    use_prompt: promptsource:*
+    training_split: train
+    validation_split: validation
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  # Natural Language Inference
+  - dataset_path: hellaswag
+    use_prompt: promptsource:*
+    training_split: train
+    validation_split: validation
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true
+  # Word Sense Disambiguation
+  - dataset_path: super_glue
+    dataset_name: wic
+    use_prompt: promptsource:*
+    training_split: train
+    validation_split: validation
+    output_type: generate_until
+    metric_list:
+      - metric: exact_match
+        aggregation: mean
+        higher_is_better: true
+        ignore_case: true
+        ignore_punctuation: true

scripts/yans/lm-evaluation-harness/lm_eval/tasks/fld/README.md

@@ -0,0 +1,65 @@
+# FLD
+
+### Paper
+
+Title: Learning Deductive Reasoning from Synthetic Corpus based on Formal Logic
+
+Abstract: https://arxiv.org/abs/2308.07336
+
+**FLD** (**F**ormal **L**ogic **D**eduction) is a deductive reasoning benchmark.
+Given a set of facts and a hypothesis, an LLM is required to generate (i) proof steps to (dis-)prove the hypothesis, and (ii) an answer ("proved", "disproved" or unknown").
+
+Unique features of FLD are:
+* It assesses the model's logical reasoning ability *isolated from knowledge*, as the facts are randomly constructed so that referring to existing knowledge never helps solve the task.
+* It assesses diverse reasoning patterns (i.e., deduction rules), as it is based on formal logic theory.
+* As a result, it is highly challenging. Indeed, even GPT-4 can solve only about half of the problems.
+
+Homepage: https://github.com/hitachi-nlp/FLD
+
+
+### Citation
+
+```
+@InProceedings{pmlr-v202-morishita23a,
+  title = 	 {Learning Deductive Reasoning from Synthetic Corpus based on Formal Logic},
+  author =       {Morishita, Terufumi and Morio, Gaku and Yamaguchi, Atsuki and Sogawa, Yasuhiro},
+  booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
+  pages = 	 {25254--25274},
+  year = 	 {2023},
+  editor = 	 {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
+  volume = 	 {202},
+  series = 	 {Proceedings of Machine Learning Research},
+  month = 	 {23--29 Jul},
+  publisher =    {PMLR},
+  pdf = 	 {https://proceedings.mlr.press/v202/morishita23a/morishita23a.pdf},
+  url = 	 {https://proceedings.mlr.press/v202/morishita23a.html},
+}
+```
+
+### Groups and Tasks
+
+This release is the simplified version of FLD where a model is required to predict only an answer.
+This setting is described by "answer accuracy" in the original paper.
+
+#### Tasks in Group `fld`
+* `fld_default` is a basic task based on [FLD.v2](https://huggingface.co/datasets/hitachi-nlp/FLD.v2/viewer/star)
+* `fld_star`: is a more challenging version based on [FLD.v2-star](https://huggingface.co/datasets/hitachi-nlp/FLD.v2/viewer/star)
+
+#### Tasks in Group `fld_logical_formula`
+Further, we have "logical formula" versions of the benchmarks, which evaluate LLMs' pure logical reasoning capabilities within the domain of logical formulas, rather than natural language:
+* `fld_logical_formula_default`
+* `fld_logical_formula_fld_star`
+
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [x] Is the task an existing benchmark in the literature?
+  * [x] Have you referenced the original paper that introduced the task?
+  * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

scripts/yans/lm-evaluation-harness/lm_eval/tasks/fld/fld_default.yaml

@@ -0,0 +1,19 @@
+task: fld_default
+dataset_path: hitachi-nlp/FLD.v2
+dataset_name: default
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Based on the provided facts ($context$), either prove or disprove the hypothesis or state that it is unknown. {{prompt_serial}}"
+doc_to_target: world_assump_label
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+metadata:
+  version: 2.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/fld/fld_logical_formula_default.yaml

@@ -0,0 +1,21 @@
+group:
+  - fld_logical_formula
+task: fld_logical_formula_default
+dataset_path: hitachi-nlp/FLD.v2
+dataset_name: default
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Based on the provided facts ($context$), either prove or disprove the hypothesis or state that it is unknown. The facts and the hypothesis are written in logical formulas as follows: capital letters such as \"{A}\", \"{B}\", \"{AB}\" are predicates, small letters such as \"{a}\", \"{b}\", \"{ab}\" are constants, \"&\" is logical conjunction, \"v\" is logical disjunction, \"¬\" is negation, \"->\" is implication, \"(x)\" is \"for all x\", and \"(Ex)\" is \"for some x\". $hypothesis$ = {{hypothesis_formula}} ; $context$ = {{context_formula}} ; $proof$ = "
+doc_to_target: world_assump_label
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+filter_list:
+  - name: remove_whitespace
+    filter:
+      - function: remove_whitespace
+      - function: take_first
+metadata:
+  version: 2.0

scripts/yans/lm-evaluation-harness/lm_eval/tasks/fld/fld_logical_formula_star.yaml

@@ -0,0 +1,3 @@
+include: fld_logical_formula_default.yaml
+task: fld_logical_formula_star
+dataset_name: star

scripts/yans/lm-evaluation-harness/lm_eval/tasks/fld/fld_star.yaml

@@ -0,0 +1,3 @@
+include: fld_default.yaml
+task: fld_star
+dataset_name: star

scripts/yans/lm-evaluation-harness/lm_eval/tasks/leaderboard/README.md

@@ -0,0 +1,328 @@
+# Leaderboard evaluations
+Our goal with this group is to create an unchanging through time version of
+evaluations that will power the Open LLM Leaderboard on HuggingFace.
+
+As we want to evaluate models across capabilities, the list currently contains:
+- BBH (3-shots, multichoice)
+- GPQA (0-shot, multichoice)
+- mmlu-pro (5-shots, multichoice)
+- Musr (0-shot, multichoice)
+- ifeval (0-shot, generative)
+- Math-lvl-5 (4-shots, generative, minerva version)
+
+
+Details on the choice of those evals can be found [here](https://huggingface.co/spaces/open-llm-leaderboard/blog) !
+
+## BigBenchHard (BBH)
+
+A suite of 23 challenging BIG-Bench tasks which we call BIG-Bench Hard (BBH).
+These are the task for which prior language model evaluations did not
+outperform the average human-rater.
+
+### Paper
+
+Title: Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them
+
+BIG-Bench (Srivastava et al., 2022) is a diverse evaluation suite that focuses on tasks believed to be beyond the capabilities of current language models. Language models have already made good progress on this benchmark, with the best model in the BIG-Bench paper outperforming average reported human-rater results on 65% of the BIG-Bench tasks via few-shot prompting. But on what tasks do language models fall short of average human-rater performance, and are those tasks actually unsolvable by current language models?
+In this work, we focus on a suite of 23 challenging BIG-Bench tasks which we call BIG-Bench Hard (BBH). These are the task for which prior language model evaluations did not outperform the average human-rater. We find that applying chain-of-thought (CoT) prompting to BBH tasks enables PaLM to surpass the average human-rater performance on 10 of the 23 tasks, and Codex (code-davinci-002) to surpass the average human-rater performance on 17 of the 23 tasks. Since many tasks in BBH require multi-step reasoning, few-shot prompting without CoT, as done in the BIG-Bench evaluations (Srivastava et al., 2022), substantially underestimates the best performance and capabilities of language models, which is better captured via CoT prompting. As further analysis, we explore the interaction between CoT and model scale on BBH, finding that CoT enables emergent task performance on several BBH tasks with otherwise flat scaling curves.
+
+
+- paper: https://huggingface.co/papers/2210.09261
+- Homepage: https://github.com/suzgunmirac/BIG-Bench-Hard
+
+### Citation
+
+```
+@article{suzgun2022challenging,
+  title={Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them},
+  author={Suzgun, Mirac and Scales, Nathan and Sch{\"a}rli, Nathanael and Gehrmann, Sebastian and Tay, Yi and Chung, Hyung Won and Chowdhery, Aakanksha and Le, Quoc V and Chi, Ed H and Zhou, Denny and and Wei, Jason},
+  journal={arXiv preprint arXiv:2210.09261},
+  year={2022}
+}
+```
+
+### Groups
+
+- `leaderboard_bbh`
+
+### Tasks
+
+- `leaderboard_bbh_boolean_expressions`
+- `leaderboard_bbh_causal_judgement`
+- `leaderboard_bbh_date_understanding`
+- `leaderboard_bbh_disambiguation_qa`
+- `leaderboard_bbh_dyck_languages`
+- `leaderboard_bbh_formal_fallacies`
+- `leaderboard_bbh_geometric_shapes`
+- `leaderboard_bbh_hyperbaton`
+- `leaderboard_bbh_logical_deduction_five_objects`
+- `leaderboard_bbh_logical_deduction_seven_objects`
+- `leaderboard_bbh_logical_deduction_three_objects`
+- `leaderboard_bbh_movie_recommendation`
+- `leaderboard_bbh_multistep_arithmetic_two`
+- `leaderboard_bbh_navigate`
+- `leaderboard_bbh_object_counting`
+- `leaderboard_bbh_penguins_in_a_table`
+- `leaderboard_bbh_reasoning_about_colored_objects`
+- `leaderboard_bbh_ruin_names`
+- `leaderboard_bbh_salient_translation_error_detection`
+- `leaderboard_bbh_snarks`
+- `leaderboard_bbh_sports_understanding`
+- `leaderboard_bbh_temporal_sequences`
+- `leaderboard_bbh_tracking_shuffled_objects_five_objects`
+- `leaderboard_bbh_tracking_shuffled_objects_seven_objects`
+- `leaderboard_bbh_tracking_shuffled_objects_three_objects`
+- `leaderboard_bbh_web_of_lies`
+- `leaderboard_bbh_word_sorting`
+
+## GPQA
+
+### Paper
+
+Title: GPQA: A Graduate-Level Google-Proof Q&A Benchmark
+
+We present GPQA, a challenging dataset of 448 multiple-choice questions written
+by domain experts in biology, physics, and chemistry. We ensure that the
+questions are high-quality and extremely difficult: experts who have or are
+pursuing PhDs in the corresponding domains reach 65% accuracy (74% when
+discounting clear mistakes the experts identified in retrospect), while highly
+skilled non-expert validators only reach 34% accuracy, despite spending on
+average over 30 minutes with unrestricted access to the web (i.e., the
+questions are “Google-proof”). The questions are also difficult for
+state-of-the-art AI systems, with our strongest GPT-4–based baseline achieving
+39% accuracy. If we are to use future AI systems to help us answer very hard
+questions—for example, when developing new scientific knowledge—we need to
+develop scalable oversight methods that enable humans to supervise their
+outputs, which may be difficult even if the supervisors are themselves skilled
+and knowledgeable. The difficulty of GPQA both for skilled non-experts and
+frontier AI systems should enable realistic scalable oversight experiments,
+which we hope can help devise ways for human experts to reliably get truthful
+information from AI systems that surpass human capabilities.
+
+- Paper: https://huggingface.co/papers/2311.12022
+- Homepage: https://github.com/idavidrein/gpqa/tree/main
+
+### Citation
+
+```
+@misc{rein2023gpqa,
+      title={GPQA: A Graduate-Level Google-Proof Q&A Benchmark},
+      author={David Rein and Betty Li Hou and Asa Cooper Stickland and Jackson Petty and Richard Yuanzhe Pang and Julien Dirani and Julian Michael and Samuel R. Bowman},
+      year={2023},
+      eprint={2311.12022},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI}
+}
+```
+
+### Groups
+
+- `leaderboard_gpqa`
+
+### Tasks
+
+- `leaderboard_gpqa_extended`
+- `leaderboard_gpqa_diamond`
+- `leaderboard_gpqa_main`
+
+## IFEval
+
+### Paper
+
+Title: Instruction-Following Evaluation for Large Language Models
+
+One core capability of Large Language Models (LLMs) is to follow natural
+language instructions. However, the evaluation of such abilities is not
+standardized: Human evaluations are expensive, slow, and not objectively
+reproducible, while LLM-based auto-evaluation is potentially biased or limited
+by the ability of the evaluator LLM. To overcome these issues, we introduce
+Instruction-Following Eval (IFEval) for large language models. IFEval is a
+straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set
+of "verifiable instructions" such as "write in more than 400 words" and
+"mention the keyword of AI at least 3 times". We identified 25 types of those
+verifiable instructions and constructed around 500 prompts, with each prompt
+containing one or more verifiable instructions. We show evaluation results of
+two widely available LLMs on the market.
+
+- Paper: https://huggingface.co/papers/2210.09261
+- Homepage: https://github.com/google-research/google-research/tree/master/instruction_following_eval
+
+### Citation
+
+```
+@article{zhou2023instructionfollowing,
+  title={Instruction-Following Evaluation for Large Language Models},
+  author={Jeffrey Zhou and Tianjian Lu and Swaroop Mishra and Siddhartha Brahma and Sujoy Basu and Yi Luan and Denny Zhou and Le Hou},
+  journal={arXiv preprint arXiv:2311.07911},
+  year={2023},
+}
+```
+
+### Tasks
+
+- `leaderboard_ifeval`
+
+## MATH-hard
+
+This is the 4 shots variant of minerva math but only keeping the level 5 questions.
+
+### Paper
+
+Title: Measuring Mathematical Problem Solving With the MATH Dataset
+
+Many intellectual endeavors require mathematical problem solving, but this
+skill remains beyond the capabilities of computers. To measure this ability in
+machine learning models, we introduce MATH, a new dataset of 12,500 challenging
+competition mathematics problems. Each problem in MATH has a full step-by-step
+solution which can be used to teach models to generate answer derivations and
+explanations.
+
+NOTE: The few-shot and the generated answer extraction is based on the
+[Minerva](https://arxiv.org/abs/2206.14858) and exact match equivalence is
+calculated using the `sympy` library. This requires additional dependencies,
+which can be installed via the `lm-eval[math]` extra.
+
+- Paper: https://huggingface.co/papers/2103.03874
+- Homepage: https://github.com/hendrycks/math
+
+
+### Citation
+
+```
+@article{hendrycksmath2021,
+  title={Measuring Mathematical Problem Solving With the MATH Dataset},
+  author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},
+  journal={NeurIPS},
+  year={2021}
+}
+@misc{2206.14858,
+Author = {Aitor Lewkowycz and Anders Andreassen and David Dohan and Ethan Dye and Henryk Michalewski and Vinay Ramasesh and Ambrose Slone and Cem Anil and Imanol Schlag and Theo Gutman-Solo and Yuhuai Wu and Behnam Neyshabur and Guy Gur-Ari and Vedant Misra},
+Title = {Solving Quantitative Reasoning Problems with Language Models},
+Year = {2022},
+Eprint = {arXiv:2206.14858},
+}
+```
+
+### Groups
+
+- `leaderboard_math_hard`
+
+### Tasks
+
+- `leaderboard_math_algebra_hard`
+- `leaderboard_math_counting_and_prob_hard`
+- `leaderboard_math_geometry_hard`
+- `leaderboard_math_intermediate_algebra_hard`
+- `leaderboard_math_num_theory_hard`
+- `leaderboard_math_prealgebra_hard`
+- `leaderboard_math_precalc_hard`
+
+
+## MMLU-Pro
+
+### Paper
+
+Title: MMLU-Pro: A More Robust and Challenging Multi-Task Language
+Understanding Benchmark
+
+In the age of large-scale language models, benchmarks like the Massive
+Multitask Language Understanding (MMLU) have been pivotal in pushing the
+boundaries of what AI can achieve in language comprehension and reasoning
+across diverse domains. However, as models continue to improve, their
+performance on these benchmarks has begun to plateau, making it increasingly
+difficult to discern differences in model capabilities. This paper introduces
+MMLU-Pro, an enhanced dataset designed to extend the mostly knowledge-driven
+MMLU benchmark by integrating more challenging, reasoning-focused questions and
+expanding the choice set from four to ten options. Additionally, MMLU-Pro
+eliminates the trivial and noisy questions in MMLU. Our experimental results
+show that MMLU-Pro not only raises the challenge, causing a significant drop in
+accuracy by 16% to 33% compared to MMLU but also demonstrates greater stability
+under varying prompts. With 24 different prompt styles tested, the sensitivity
+of model scores to prompt variations decreased from 4-5% in MMLU to just 2% in
+MMLU-Pro. Additionally, we found that models utilizing Chain of Thought (CoT)
+reasoning achieved better performance on MMLU-Pro compared to direct answering,
+which is in stark contrast to the findings on the original MMLU, indicating
+that MMLU-Pro includes more complex reasoning questions. Our assessments
+confirm that MMLU-Pro is a more discriminative benchmark to better track
+progress in the field.
+
+- Paper: https://huggingface.co/papers/2406.01574
+- Homepage: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro
+
+### Citation
+
+```
+@misc{wang2024mmluprorobustchallengingmultitask,
+      title={MMLU-Pro: A More Robust and Challenging Multi-Task Language
+      Understanding Benchmark},
+      author={Yubo Wang and Xueguang Ma and Ge Zhang and Yuansheng Ni and Abhranil Chandra and Shiguang Guo and Weiming Ren and Aaran Arulraj and Xuan He and Ziyan Jiang and Tianle Li and Max Ku and Kai Wang and Alex Zhuang and Rongqi Fan and Xiang Yue and Wenhu Chen},
+      year={2024},
+      eprint={2406.01574},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2406.01574},
+}
+```
+
+### Groups
+
+- `leaderboard_mmlu_pro`
+
+### Tasks
+
+- `leaderboard_mmlu_pro`
+
+
+## Musr
+
+### Paper
+
+Title: MuSR: Testing the Limits of Chain-of-thought with Multistep Soft
+Reasoning  
+
+While large language models (LLMs) equipped with techniques like
+chain-of-thought prompting have demonstrated impressive capabilities, they
+still fall short in their ability to reason robustly in complex settings.
+However, evaluating LLM reasoning is challenging because system capabilities
+continue to grow while benchmark datasets for tasks like logical deduction have
+remained static. We introduce MuSR, a dataset for evaluating language models on
+multistep soft reasoning tasks specified in a natural language narrative. This
+dataset has two crucial features. First, it is created through a novel
+neurosymbolic synthetic-to-natural generation algorithm, enabling the
+construction of complex reasoning instances that challenge GPT-4 (e.g., murder
+mysteries roughly 1000 words in length) and which can be scaled further as more
+capable LLMs are released. Second, our dataset instances are free text
+narratives corresponding to real-world domains of reasoning; this makes it
+simultaneously much more challenging than other synthetically-crafted
+benchmarks while remaining realistic and tractable for human annotators to
+solve with high accuracy. We evaluate a range of LLMs and prompting techniques
+on this dataset and characterize the gaps that remain for techniques like
+chain-of-thought to perform robust reasoning.
+
+- Paper: https://huggingface.co/papers/2310.16049
+- Homepage: https://zayne-sprague.github.io/MuSR/
+
+### Citation
+
+```
+@misc{sprague2024musrtestinglimitschainofthought,
+      title={MuSR: Testing the Limits of Chain-of-thought with Multistep Soft
+      Reasoning},
+      author={Zayne Sprague and Xi Ye and Kaj Bostrom and Swarat Chaudhuri and Greg Durrett},
+      year={2024},
+      eprint={2310.16049},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2310.16049},
+}
+```
+
+### Groups
+
+- `leaderboard_musr`
+
+### Tasks
+
+- `leaderboard_musr_murder_mysteries`
+- `leaderboard_musr_object_placements`
+- `leaderboard_musr_team_allocation`

scripts/yans/lm-evaluation-harness/lm_eval/tasks/leaderboard/bbh_mc/_leaderboard_bbh.yaml

@@ -0,0 +1,26 @@
+group: leaderboard_bbh
+task:
+  - leaderboard_bbh_boolean_expressions
+  - leaderboard_bbh_causal_judgement
+  - leaderboard_bbh_date_understanding
+  - leaderboard_bbh_disambiguation_qa
+  - leaderboard_bbh_formal_fallacies
+  - leaderboard_bbh_geometric_shapes
+  - leaderboard_bbh_hyperbaton
+  - leaderboard_bbh_logical_deduction_five_objects
+  - leaderboard_bbh_logical_deduction_seven_objects
+  - leaderboard_bbh_logical_deduction_three_objects
+  - leaderboard_bbh_movie_recommendation
+  - leaderboard_bbh_navigate
+  - leaderboard_bbh_object_counting
+  - leaderboard_bbh_penguins_in_a_table
+  - leaderboard_bbh_reasoning_about_colored_objects
+  - leaderboard_bbh_ruin_names
+  - leaderboard_bbh_salient_translation_error_detection
+  - leaderboard_bbh_snarks
+  - leaderboard_bbh_sports_understanding
+  - leaderboard_bbh_temporal_sequences
+  - leaderboard_bbh_tracking_shuffled_objects_five_objects
+  - leaderboard_bbh_tracking_shuffled_objects_seven_objects
+  - leaderboard_bbh_tracking_shuffled_objects_three_objects
+  - leaderboard_bbh_web_of_lies

scripts/yans/lm-evaluation-harness/lm_eval/tasks/leaderboard/bbh_mc/boolean_expressions.yaml

@@ -0,0 +1,14 @@
+dataset_name: boolean_expressions
+description: 'Evaluate the result of a random Boolean expression.'
+doc_to_choice: ["False", "True"]
+fewshot_config:
+  sampler: first_n
+  samples:
+  - input: not ( ( not not True ) ) is
+    target: 'False'
+  - input: True and False and not True and True is
+    target: 'False'
+  - input: not not ( not ( False ) ) is
+    target: 'True'
+include: _fewshot_template_yaml
+task: leaderboard_bbh_boolean_expressions

scripts/yans/lm-evaluation-harness/lm_eval/tasks/leaderboard/bbh_mc/formal_fallacies.yaml

@@ -0,0 +1,55 @@
+dataset_name: formal_fallacies
+description: 'Distinguish deductively valid arguments from formal fallacies.'
+doc_to_choice: ["valid", "invalid"]
+fewshot_config:
+  sampler: first_n
+  samples:
+  - input: '"It is not always easy to see who is related to whom -- and in which ways.
+      The following argument pertains to this question: To begin with, Lesley is a
+      close friend of Fernando. Moreover, being a close friend of Fernando or a schoolmate
+      of Lowell is sufficient for being a great-grandfather of Leroy. It follows that
+      Lesley is a great-grandfather of Leroy."
+
+      Is the argument, given the explicitly stated premises, deductively valid or
+      invalid?
+
+      Options:
+
+      - valid
+
+      - invalid'
+    target: valid
+  - input: '"It is not always easy to see who is related to whom -- and in which ways.
+      The following argument pertains to this question: Whoever is not a great-grandfather
+      of Clyde is a stepbrother of Brian. Being an ancestor of Dana is sufficient
+      for not being a great-grandfather of Clyde. We may conclude: Everyone who is
+      an ancestor of Dana is a stepbrother of Brian, too."
+
+      Is the argument, given the explicitly stated premises, deductively valid or
+      invalid?
+
+      Options:
+
+      - valid
+
+      - invalid'
+    target: valid
+  - input: '"It is not always easy to grasp who is consuming which products. The following
+      argument pertains to this question: Every infrequent user of Paul Mitchell shampoo
+      is either a rare consumer of Nioxin shampoo or a loyal buyer of Caress soap,
+      or both. No regular consumer of Lush soap is a rare consumer of Nioxin shampoo
+      and, in the same time, a loyal buyer of Caress soap. It follows that whoever
+      is an infrequent user of Paul Mitchell shampoo is not a regular consumer of
+      Lush soap."
+
+      Is the argument, given the explicitly stated premises, deductively valid or
+      invalid?
+
+      Options:
+
+      - valid
+
+      - invalid'
+    target: invalid
+include: _fewshot_template_yaml
+task: leaderboard_bbh_formal_fallacies

scripts/yans/lm-evaluation-harness/lm_eval/tasks/leaderboard/bbh_mc/geometric_shapes.yaml

@@ -0,0 +1,84 @@
+dataset_name: geometric_shapes
+description: 'Name geometric shapes from their SVG paths.'
+doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)","(G)","(H)","(I)","(J)","(K)"]
+fewshot_config:
+  sampler: first_n
+  samples:
+  - input: 'This SVG path element <path d="M 31.00,73.00 L 32.00,59.00 L 44.00,50.00
+      L 49.00,41.00 L 64.00,37.00 L 71.00,55.00 L 64.00,76.00 L 52.00,61.00 L 31.00,73.00"/>
+      draws a
+
+      Options:
+
+      (A) circle
+
+      (B) heptagon
+
+      (C) hexagon
+
+      (D) kite
+
+      (E) line
+
+      (F) octagon
+
+      (G) pentagon
+
+      (H) rectangle
+
+      (I) sector
+
+      (J) triangle'
+    target: (F)
+  - input: 'This SVG path element <path d="M 14.19,26.04 L 51.43,39.21 L 58.44,36.69
+      L 56.63,30.17 L 48.53,26.66 L 14.19,26.04"/> draws a
+
+      Options:
+
+      (A) circle
+
+      (B) heptagon
+
+      (C) hexagon
+
+      (D) kite
+
+      (E) line
+
+      (F) octagon
+
+      (G) pentagon
+
+      (H) rectangle
+
+      (I) sector
+
+      (J) triangle'
+    target: (G)
+  - input: 'This SVG path element <path d="M 41.00,43.00 L 37.00,34.00 L 41.00,33.00
+      L 45.00,34.00 L 41.00,43.00"/> draws a
+
+      Options:
+
+      (A) circle
+
+      (B) heptagon
+
+      (C) hexagon
+
+      (D) kite
+
+      (E) line
+
+      (F) octagon
+
+      (G) pentagon
+
+      (H) rectangle
+
+      (I) sector
+
+      (J) triangle'
+    target: (D)
+include: _fewshot_template_yaml
+task: leaderboard_bbh_geometric_shapes

scripts/yans/lm-evaluation-harness/lm_eval/tasks/leaderboard/bbh_mc/hyperbaton.yaml

@@ -0,0 +1,32 @@
+dataset_name: hyperbaton
+description: 'Order adjectives correctly in English sentences.'
+doc_to_choice: ["(A)", "(B)"]
+fewshot_config:
+  sampler: first_n
+  samples:
+  - input: 'Which sentence has the correct adjective order:
+
+      Options:
+
+      (A) rubber terrible ship
+
+      (B) terrible rubber ship'
+    target: (B)
+  - input: 'Which sentence has the correct adjective order:
+
+      Options:
+
+      (A) repulsive small Brazilian exercise ship
+
+      (B) Brazilian repulsive exercise small ship'
+    target: (A)
+  - input: 'Which sentence has the correct adjective order:
+
+      Options:
+
+      (A) blue gold wonderful square shoe
+
+      (B) wonderful square blue gold shoe'
+    target: (B)
+include: _fewshot_template_yaml
+task: leaderboard_bbh_hyperbaton

scripts/yans/lm-evaluation-harness/lm_eval/tasks/leaderboard/bbh_mc/reasoning_about_colored_objects.yaml

@@ -0,0 +1,114 @@
+dataset_name: reasoning_about_colored_objects
+description: 'Answer extremely simple questions about the colors of objects on a surface.'
+doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)","(G)","(H)","(I)","(J)","(K)","(L)","(M)","(N)","(O)","(P)","(Q)","(R)"]
+fewshot_config:
+  sampler: first_n
+  samples:
+  - input: 'On the nightstand, there is a red pencil, a purple mug, a burgundy keychain,
+      a fuchsia teddy bear, a black plate, and a blue stress ball. What color is the
+      stress ball?
+
+      Options:
+
+      (A) red
+
+      (B) orange
+
+      (C) yellow
+
+      (D) green
+
+      (E) blue
+
+      (F) brown
+
+      (G) magenta
+
+      (H) fuchsia
+
+      (I) mauve
+
+      (J) teal
+
+      (K) turquoise
+
+      (L) burgundy
+
+      (M) silver
+
+      (N) gold
+
+      (O) black
+
+      (P) grey
+
+      (Q) purple
+
+      (R) pink'
+    target: (E)
+  - input: 'On the table, you see a bunch of objects arranged in a row: a purple paperclip,
+      a pink stress ball, a brown keychain, a green scrunchiephone charger, a mauve
+      fidget spinner, and a burgundy pen. What is the color of the object directly
+      to the right of the stress ball?
+
+      Options:
+
+      (A) red
+
+      (B) orange
+
+      (C) yellow
+
+      (D) green
+
+      (E) blue
+
+      (F) brown
+
+      (G) magenta
+
+      (H) fuchsia
+
+      (I) mauve
+
+      (J) teal
+
+      (K) turquoise
+
+      (L) burgundy
+
+      (M) silver
+
+      (N) gold
+
+      (O) black
+
+      (P) grey
+
+      (Q) purple
+
+      (R) pink'
+    target: (F)
+  - input: 'On the nightstand, you see the following items arranged in a row: a teal
+      plate, a burgundy keychain, a yellow scrunchiephone charger, an orange mug,
+      a pink notebook, and a grey cup. How many non-orange items do you see to the
+      left of the teal item?
+
+      Options:
+
+      (A) zero
+
+      (B) one
+
+      (C) two
+
+      (D) three
+
+      (E) four
+
+      (F) five
+
+      (G) six'
+    target: (A)
+include: _fewshot_template_yaml
+task: leaderboard_bbh_reasoning_about_colored_objects