March07's picture
add transferability information
683d0f6
raw
history blame
8.55 kB
[
{
"dataset": "cola",
"type": "task",
"origin_acc": 79.0,
"origin_prompt": "Review the sentence below and identify whether its grammar is 'Acceptable' or 'Unacceptable':",
"atk_acc": 74.5,
"atk_prompt": "Review the sentece below and identify whether its grammar is 'Acceptable' or 'Unacceptable':",
"transfer_ori_acc": 28.9,
"transfer_atk_acc": 42.0
},
{
"dataset": "cola",
"type": "role",
"origin_acc": 79.5,
"origin_prompt": "While performing grammar analysis, classify the grammar of the following sentence as 'acceptable' for correct grammar or 'unacceptable' for incorrect grammar:",
"atk_acc": 0.0,
"atk_prompt": "nhile performing grammr analysis, classify the rgammar of the fllowing sentenZe as 'acceptable' for corFect Xrammar or 'unacceptable' for incorrect gramUar:",
"transfer_ori_acc": 27.0,
"transfer_atk_acc": 42.5
},
{
"dataset": "mrpc",
"type": "task",
"origin_acc": 74.0,
"origin_prompt": "Evaluate whether the two provided sentences convey the same meaning by answering 'equivalent' or 'not_equivalent'.",
"atk_acc": 70.0,
"atk_prompt": "Evaluate whether the two provRided sentnces convey the same meaning by answering 'equivalent' or 'not_equivalent'. ",
"transfer_ori_acc": 48.77450980392157,
"transfer_atk_acc": 29.411764705882355
},
{
"dataset": "mrpc",
"type": "role",
"origin_acc": 76.0,
"origin_prompt": "In your capacity as a semantic assessment tool, evaluate the provided sentences and classify their meanings as 'equivalent' or 'not_equivalent'.",
"atk_acc": 53.0,
"atk_prompt": "nI your capacity as a sematnic asseJssment tol, evaluate the prvoided senRtences and classify their meainngs as 'equivalent' or 'not_equivalent'. ",
"transfer_ori_acc": 48.03921568627451,
"transfer_atk_acc": 27.696078431372552
},
{
"dataset": "qnli",
"type": "task",
"origin_acc": 78.0,
"origin_prompt": "Evaluate whether the given context supports the answer to the question by responding with 'entailment' or 'not_entailment'.",
"atk_acc": 50.0,
"atk_prompt": "Exaluate whetxher the iven contet Supports the anwser to the queston by respEonding with 'entailment' or 'not_entailment'. ",
"transfer_ori_acc": 26.5,
"transfer_atk_acc": 29.7
},
{
"dataset": "qnli",
"type": "role",
"origin_acc": 79.5,
"origin_prompt": "As a textual inference expert, analyze if the answer to the question can be deduced from the provided context and select 'entailment' or 'not_entailment'.",
"atk_acc": 19.5,
"atk_prompt": "As a textula inference expert, aInalyze if the answer to the question can be deducud from the provided context and Melect 'entailment' or 'not_entailment'. ",
"transfer_ori_acc": 32.5,
"transfer_atk_acc": 32.0
},
{
"dataset": "qqp",
"type": "task",
"origin_acc": 82.0,
"origin_prompt": "Assess whether the following statements are identical in meaning by answering 'equivalent' or 'not_equivalent'.",
"atk_acc": 45.0,
"atk_prompt": "Assess whethdr the followng sLtatements are identocal in meuaning by answerinJ 'equivalent' or 'not_equivalent'. ",
"transfer_ori_acc": 30.2,
"transfer_atk_acc": 23.0
},
{
"dataset": "qqp",
"type": "role",
"origin_acc": 82.5,
"origin_prompt": "As an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions.",
"atk_acc": 79.5,
"atk_prompt": "Ao an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions. ",
"transfer_ori_acc": 26.200000000000003,
"transfer_atk_acc": 12.2
},
{
"dataset": "rte",
"type": "task",
"origin_acc": 83.5,
"origin_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word.",
"atk_acc": 79.0,
"atk_prompt": "ArYe the folowing two sentenes entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word. ",
"transfer_ori_acc": 27.075812274368232,
"transfer_atk_acc": 7.581227436823104
},
{
"dataset": "rte",
"type": "role",
"origin_acc": 80.5,
"origin_prompt": "As an instrument for entailment evaluation, consider the two sentences and determine if their relationship is 'entailment' or 'not_entailment'. Respond with 'entailment' or 'not_entailment':",
"atk_acc": 78.5,
"atk_prompt": "IAs an instrument for entailment evaluatioRn, consider the otwo sentences and determine if their relationship is 'entailment' or 'not_entailment'. Respond with 'entailment' or 'not_entailment':",
"transfer_ori_acc": 24.90974729241877,
"transfer_atk_acc": 32.851985559566785
},
{
"dataset": "sst2",
"type": "task",
"origin_acc": 93.0,
"origin_prompt": "Determine the overall sentiment of this sentence, categorizing it as 'positive' or 'negative':",
"atk_acc": 88.5,
"atk_prompt": "Deetrmine the overall sentLment of this sentence, categorizing it as 'positive' or 'negative': ",
"transfer_ori_acc": 14.3348623853211,
"transfer_atk_acc": 6.422018348623854
},
{
"dataset": "sst2",
"type": "role",
"origin_acc": 93.0,
"origin_prompt": "Functioning as a sentiment identification tool, assess if the following expression is 'positive' or 'negative'. Classify:",
"atk_acc": 85.0,
"atk_prompt": "uFnctioning as a sentmient identification tool, assess if the folGowing expression is 'positive' or 'negative'. Classify: ",
"transfer_ori_acc": 38.646788990825684,
"transfer_atk_acc": 30.84862385321101
},
{
"dataset": "wnli",
"type": "task",
"origin_acc": 66.2,
"origin_prompt": "Examine the pair of sentences and determine if they exhibit entailment or not_entailment. Answer with either 'entailment' or 'not_entailment'.",
"atk_acc": 53.52,
"atk_prompt": "Examine the Bair of snetences and determine if they exhibit entailment or not_entailment. Ancwer with eMther 'entailment' or 'not_entailment'.",
"transfer_ori_acc": 33.80281690140845,
"transfer_atk_acc": 25.352112676056336
},
{
"dataset": "wnli",
"type": "role",
"origin_acc": 63.38,
"origin_prompt": "In the role of a semantic relationship analyzer, examine the connection between the given sentences and decide if they exhibit entailment or not_entailment. Answer with 'entailment' or 'not_entailment':",
"atk_acc": 53.52,
"atk_prompt": "IQn the role of a semantic relationship analyzer, eamine the connection between the given sentences and decide if they exhbit entailment or not_entailment. Answer with 'entailment' or 'not_entailment':",
"transfer_ori_acc": 43.66197183098591,
"transfer_atk_acc": 43.66197183098591
},
{
"dataset": "mmlu",
"type": "task",
"origin_acc": 62.5,
"origin_prompt": "Regarding the following multiple-choice question on {}, pick the correct answer from the options 'A', 'B', 'C', or 'D'.",
"atk_acc": 60.0,
"atk_prompt": "Regarding the following multiple-choice question on {}, pick the correct answeWr from the options 'A', 'B', 'C', or 'D'. ",
"transfer_ori_acc": 18.439716312056735,
"transfer_atk_acc": 22.5177304964539
},
{
"dataset": "mmlu",
"type": "role",
"origin_acc": 62.5,
"origin_prompt": "With your knowledge of {}, tackle the following multiple-choice question by choosing 'A', 'B', 'C', or 'D'.",
"atk_acc": 59.0,
"atk_prompt": "With your knowledge of {}, tackle the follQwing multiple-choice question by choosing 'A', 'B', 'C', or 'D'.",
"transfer_ori_acc": 22.872340425531913,
"transfer_atk_acc": 23.93617021276596
}
]