dataset: anli
templates:
  0cc3ae39-3997-4686-8c93-5d51457efa1f: !Template
    answer_choices: Correct ||| Inconclusive ||| Incorrect
    id: 0cc3ae39-3997-4686-8c93-5d51457efa1f
    jinja: '{{premise}} Using only the above description and what you know about the
      world, "{{hypothesis}}" is definitely correct, incorrect, or inconclusive? |||
      {{ answer_choices[label] }}'
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: MNLI crowdsource
    reference: Adapted from Williams et al. 2018's instructions to crowdsourcing workers.
  179eb863-3ece-4e6f-af0f-fcb46d997306: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: 179eb863-3ece-4e6f-af0f-fcb46d997306
    jinja: 'Given {{premise}} Should we assume that "{{hypothesis}}" is true? Yes,
      no, or maybe? ||| {{ answer_choices[label] }} '
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: should assume
    reference: Webson & Pavlick 2021
  5459237b-97de-4340-bf7b-2939c3f7ca19: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: 5459237b-97de-4340-bf7b-2939c3f7ca19
    jinja: Given that {{premise}} Does it follow that {{hypothesis}} Yes, no, or maybe?
      ||| {{ answer_choices[label] }}
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: does it follow that
    reference: v0.1
  620aa3fc-d5eb-46f5-a1ee-4c754527aa97: !Template
    answer_choices: True ||| Neither ||| False
    id: 620aa3fc-d5eb-46f5-a1ee-4c754527aa97
    jinja: '{{premise}}

      Question: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label]
      }}'
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: GPT-3 style
    reference: 'Same as reported in Figure G7 of the GPT-3 paper, except that there
      is no task identifying tokens like "anli R1: ".'
  9b613182-c6ab-4427-9221-3d68f6d62765: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: 9b613182-c6ab-4427-9221-3d68f6d62765
    jinja: '{{premise}} Based on the previous passage, is it true that "{{hypothesis}}"?
      Yes, no, or maybe? ||| {{ answer_choices[label] }}'
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: based on the previous passage
    reference: "Adapted from the BoolQ prompts in Schick & Sch\xFCtze 2021."
  a850110d-f1a3-49b4-949a-d3bfe9f81344: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: a850110d-f1a3-49b4-949a-d3bfe9f81344
    jinja: '{{premise}} Are we justified in saying that "{{hypothesis}}"? Yes, no,
      or maybe? ||| {{ answer_choices[label] }} '
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: justified in saying
    reference: Webson & Pavlick 2021
  bab86d5a-4f9c-40db-b619-a7b7d5cae681: !Template
    answer_choices: True ||| Inconclusive ||| False
    id: bab86d5a-4f9c-40db-b619-a7b7d5cae681
    jinja: 'Take the following as truth: {{premise}}

      Then the following statement: "{{hypothesis}}" is {{"true"}}, {{"false"}}, or
      {{"inconclusive"}}? ||| {{ answer_choices[label] }}'
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: take the following as truth
    reference: Bers et al.
  bcd90047-3a2b-426b-b065-8a418f1317b8: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: bcd90047-3a2b-426b-b065-8a418f1317b8
    jinja: 'Given that {{premise}} Therefore, it must be true that "{{hypothesis}}"?
      Yes, no, or maybe? ||| {{ answer_choices[label] }} '
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: must be true
    reference: v0.1
  c4ed37ae-d7d7-4197-a725-ef2152fa3b1f: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: c4ed37ae-d7d7-4197-a725-ef2152fa3b1f
    jinja: 'Suppose {{premise}} Can we infer that "{{hypothesis}}"? Yes, no, or maybe?
      ||| {{ answer_choices[label] }} '
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: can we infer
    reference: Webson & Pavlick 2021
  ca24b93a-6265-462f-b140-e329c03d94fa: !Template
    answer_choices: Guaranteed ||| Possible ||| Impossible
    id: ca24b93a-6265-462f-b140-e329c03d94fa
    jinja: "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is\
      \ {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label]\
      \ }}"
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: guaranteed/possible/impossible
    reference: Bers et al.
  dbc68425-5c42-43ae-9748-70ce8c5a167e: !Template
    answer_choices: Always ||| Sometimes ||| Never
    id: dbc68425-5c42-43ae-9748-70ce8c5a167e
    jinja: Suppose it's true that {{premise}} Then, is "{{hypothesis}}" {{"always"}},
      {{"sometimes"}}, or {{"never"}} true? ||| {{ answer_choices[label] }}
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: always/sometimes/never
    reference: Bers et al.
  e5b7fdd7-fdff-4630-889b-3c7a052e5da0: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: e5b7fdd7-fdff-4630-889b-3c7a052e5da0
    jinja: "{{premise}} \n\nQuestion: Does this imply that \"{{hypothesis}}\"? Yes,\
      \ no, or maybe? ||| {{answer_choices[label]}}"
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: does this imply
    reference: v0.1
  e6f32b9c-7e0b-474a-a0d2-e84d20c22aba: !Template
    answer_choices: Always ||| Sometimes ||| Never
    id: e6f32b9c-7e0b-474a-a0d2-e84d20c22aba
    jinja: "{{premise}} \n\nKeeping in mind the above text, consider: {{hypothesis}}\
      \ Is this {{\"always\"}}, {{\"sometimes\"}}, or {{\"never\"}} correct? ||| {{\
      \ answer_choices[label] }}"
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: consider always/sometimes/never
    reference: Bers et al.
  ec249357-e672-4e7d-b8b6-d97ed7d090c5: !Template
    answer_choices: True ||| Inconclusive ||| False
    id: ec249357-e672-4e7d-b8b6-d97ed7d090c5
    jinja: '{{premise}} Based on that information, is the claim: "{{hypothesis}}"
      {{"true"}}, {{"false"}}, or {{"inconclusive"}}? ||| {{ answer_choices[label]
      }}'
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: claim true/false/inconclusive
    reference: Bers et al.
  ffa0a6f0-7186-4ccb-bb35-8b1affb747a0: !Template
    answer_choices: Yes ||| Maybe ||| No
    id: ffa0a6f0-7186-4ccb-bb35-8b1affb747a0
    jinja: 'Given {{premise}} Is it guaranteed true that "{{hypothesis}}"? Yes, no,
      or maybe? ||| {{ answer_choices[label] }} '
    metadata: !TemplateMetadata
      choices_in_prompt: true
      metrics:
      - Accuracy
      original_task: true
    name: guaranteed true
    reference: Webson & Pavlick 2021