results_transfer/google-flan-t5-large_chatgpt/deepwordbug_0_shot.json

[
    {
        "dataset": "cola",
        "type": "task",
        "origin_acc": 77.9,
        "origin_prompt": "Examine the given sentence and decide if it is grammatically sound. Answer with either 'Acceptable' or 'Unacceptable':",
        "atk_acc": 31.1,
        "atk_prompt": "Exapine the gievn sentene and dVecide if it is grammktically sound. Answer with either 'Acceptable' or 'Unacceptable':",
        "transfer_ori_acc": 77.0,
        "transfer_atk_acc": 75.5
    },
    {
        "dataset": "cola",
        "type": "role",
        "origin_acc": 76.3,
        "origin_prompt": "As an instrument for grammar evaluation, consider the sentence and determine if its grammar is correct, responding with 'acceptable' for correct grammar or 'unacceptable' for incorrect grammar:",
        "atk_acc": 36.5,
        "atk_prompt": "sA an instrument for grammar evaluatiJn, cnsider the setnence and deterbmine if its grmamar is crorect, reYponding with 'acceptable' for corect gramOar or 'unacceptable' for incorGect rgammar:",
        "transfer_ori_acc": 79.5,
        "transfer_atk_acc": 80.0
    },
    {
        "dataset": "mrpc",
        "type": "task",
        "origin_acc": 82.35,
        "origin_prompt": "Compare the following sentences and determine if they share the same semantic meaning by responding with 'equivalent' or 'not_equivalent'.",
        "atk_acc": 76.96,
        "atk_prompt": "Compar the folhowing sentences and deWermine if they share the same semantic emaning by respondig with 'equivalent' or 'not_equivalent'. ",
        "transfer_ori_acc": 73.0,
        "transfer_atk_acc": 76.0
    },
    {
        "dataset": "mrpc",
        "type": "role",
        "origin_acc": 82.6,
        "origin_prompt": "In your capacity as a semantic comparison tool, examine the given sentences and decide if their meanings are identical, responding with 'equivalent' or 'not_equivalent'.",
        "atk_acc": 62.25,
        "atk_prompt": "In your capacity as a seantic coGparison tolo, exaine the given esentences and decide if their meanings are identiacl, resYonding with 'equivalent' or 'not_equivalent'. ",
        "transfer_ori_acc": 75.0,
        "transfer_atk_acc": 69.0
    },
    {
        "dataset": "qnli",
        "type": "task",
        "origin_acc": 89.8,
        "origin_prompt": "Evaluate whether the given context supports the answer to the question by responding with 'entailment' or 'not_entailment'.",
        "atk_acc": 38.6,
        "atk_prompt": "Evlauate whyther the giGven cotext supporzs the answeQ to the quePtion by rseponding with 'entailment' or 'not_entailment'. ",
        "transfer_ori_acc": 78.0,
        "transfer_atk_acc": 74.5
    },
    {
        "dataset": "qnli",
        "type": "role",
        "origin_acc": 89.4,
        "origin_prompt": "As a semantic interpreter, assess whether the provided context supports the answer to the given question and answer with 'entailment' or 'not_entailment'.",
        "atk_acc": 2.1,
        "atk_prompt": "sA a esmantic interpreter, aBssess whehter the proviedd contexYt spuports the answher to the gEven questin and answOr with 'entailment' or 'not_entailment'. ",
        "transfer_ori_acc": 68.0,
        "transfer_atk_acc": 65.0
    },
    {
        "dataset": "qqp",
        "type": "task",
        "origin_acc": 87.8,
        "origin_prompt": "Can these two statements be considered equal in meaning? Answer with 'equivalent' or 'not_equivalent'.",
        "atk_acc": 59.8,
        "atk_prompt": "Can these tow statYements be cnosidered qeual in maening? AnsweI with 'equivalent' or 'not_equivalent'. ",
        "transfer_ori_acc": 81.0,
        "transfer_atk_acc": 72.0
    },
    {
        "dataset": "qqp",
        "type": "role",
        "origin_acc": 86.9,
        "origin_prompt": "Functioning as a question similarity evaluation tool, analyze the given questions and decide if they share the same meaning, responding with 'equivalent' or 'not_equivalent'.",
        "atk_acc": 81.4,
        "atk_prompt": "Functioning as a question smiilarity eavluation toXl, analyze the givYen questions and decidae if they share the same meajning, responding with 'equivalent' or 'not_equivalent'. ",
        "transfer_ori_acc": 81.5,
        "transfer_atk_acc": 81.5
    },
    {
        "dataset": "rte",
        "type": "task",
        "origin_acc": 86.64,
        "origin_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word.",
        "atk_acc": 81.95,
        "atk_prompt": "Aer the following two Ysentences entailment or not_entailment? nswer me with \"entailment\" or \"not_entailment\", just one word. ",
        "transfer_ori_acc": 81.0,
        "transfer_atk_acc": 83.5
    },
    {
        "dataset": "rte",
        "type": "role",
        "origin_acc": 84.84,
        "origin_prompt": "While performing entailment analysis, classify the relationship between the provided sentences as 'entailment' or 'not_entailment':",
        "atk_acc": 82.67,
        "atk_prompt": "While performing entailment analysis, cassify the reiationship between the proviLded sentences as 'entailment' or 'not_entailment':",
        "transfer_ori_acc": 79.0,
        "transfer_atk_acc": 77.5
    },
    {
        "dataset": "sst2",
        "type": "task",
        "origin_acc": 94.84,
        "origin_prompt": "Analyze the tone of this statement and respond with either 'positive' or 'negative':",
        "atk_acc": 94.04,
        "atk_prompt": "Analze the tone of this sattement and respond with either 'positive' or 'negative': ",
        "transfer_ori_acc": 89.5,
        "transfer_atk_acc": 90.0
    },
    {
        "dataset": "sst2",
        "type": "role",
        "origin_acc": 95.3,
        "origin_prompt": "Serving as a sentiment evaluation model, determine if the given statement is 'positive' or 'negative'. Classify:",
        "atk_acc": 64.91,
        "atk_prompt": "Servign as a sentimBnt envaluation model, Qetermine if the Iiven statemen is 'positive' or 'negative'. Classhfy: ",
        "transfer_ori_acc": 92.0,
        "transfer_atk_acc": 91.0
    },
    {
        "dataset": "wnli",
        "type": "task",
        "origin_acc": 66.2,
        "origin_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word.",
        "atk_acc": 60.56,
        "atk_prompt": "Are the follHwing Ntwo sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one wor. ",
        "transfer_ori_acc": 66.19718309859155,
        "transfer_atk_acc": 64.7887323943662
    },
    {
        "dataset": "wnli",
        "type": "role",
        "origin_acc": 66.2,
        "origin_prompt": "As an entailment identification system, examine the connection between the following sentences and respond with 'entailment' or 'not_entailment':",
        "atk_acc": 57.75,
        "atk_prompt": "As an entailment identfiication zystem, examine the connection between the flolowing sentences and respond with 'entailment' or 'not_entailment':",
        "transfer_ori_acc": 52.112676056338024,
        "transfer_atk_acc": 50.70422535211267
    },
    {
        "dataset": "mmlu",
        "type": "task",
        "origin_acc": 45.74,
        "origin_prompt": "For each multiple-choice question about {}, identify the correct answer by selecting 'A', 'B', 'C', or 'D'.",
        "atk_acc": 42.2,
        "atk_prompt": "For each multiple-chotce question about {}, dentify the caorrect answedr by selccting 'A', 'B', 'C', or 'D'. ",
        "transfer_ori_acc": 59.5,
        "transfer_atk_acc": 63.5
    },
    {
        "dataset": "mmlu",
        "type": "role",
        "origin_acc": 44.86,
        "origin_prompt": "Utilizing your expertise in {}, answer the following multiple-choice question by picking 'A', 'B', 'C', or 'D'.",
        "atk_acc": 41.13,
        "atk_prompt": "Utilizing your expertise in {}, ansKer the following multiprle-choice question by ipcking 'A', 'B', 'C', or 'D'.",
        "transfer_ori_acc": 60.5,
        "transfer_atk_acc": 61.0
    },
    {
        "dataset": "squad_v2",
        "type": "task",
        "origin_acc": 87.55,
        "origin_prompt": "Find the correct answer in the context provided. If an answer cannot be found, please respond with 'unanswerable'.",
        "atk_acc": 85.32,
        "atk_prompt": "Find the Dcorrect answer in the context provided. GIf an answer cannot be found, please erspond with 'unanswerable'.",
        "transfer_ori_acc": 67.5632226901742,
        "transfer_atk_acc": 66.07909004721805
    },
    {
        "dataset": "squad_v2",
        "type": "role",
        "origin_acc": 88.05,
        "origin_prompt": "As an expert with a deep understanding of the context, find the best answer. If the context doesn't include an answer, say 'unanswerable'.",
        "atk_acc": 86.92,
        "atk_prompt": "As an expert with a deep understanding of the context, fHind the best answer. If the context doesn't include an answer, say 'unanswerable'.",
        "transfer_ori_acc": 64.66646500169827,
        "transfer_atk_acc": 63.33777694425634
    },
    {
        "dataset": "iwslt",
        "type": "task",
        "origin_acc": 0.14,
        "origin_prompt": "Perform a machine translation for the provided sentence, changing it from {} to {}:",
        "atk_acc": 0.08,
        "atk_prompt": "Perorm a nachine translatiou for the providde sentecne, changing it from {} to {}:",
        "transfer_ori_acc": 0.22160247790642423,
        "transfer_atk_acc": 0.22247429357782236
    },
    {
        "dataset": "iwslt",
        "type": "role",
        "origin_acc": 0.14,
        "origin_prompt": "Taking on the role of a certified translator, perform a translation of the given sentence from {} to {}:",
        "atk_acc": 0.07,
        "atk_prompt": "TakIing on the roel of a certifieFd trLanslator, erform a translaton of the guiven Ventence from {} to {}: ",
        "transfer_ori_acc": 0.22524167015355148,
        "transfer_atk_acc": 0.21936213420618
    },
    {
        "dataset": "un_multi",
        "type": "task",
        "origin_acc": 0.27,
        "origin_prompt": "Convert the following sentence from its original {} language to the target language {}:",
        "atk_acc": 0.12,
        "atk_prompt": "ConvHert the folloiwng setnence from its origianl {} langudage to the trget languaoe {}:",
        "transfer_ori_acc": 0.3252157829213561,
        "transfer_atk_acc": 0.32616092622927834
    },
    {
        "dataset": "un_multi",
        "type": "role",
        "origin_acc": 0.27,
        "origin_prompt": "As a qualified translator, convert the given sentence from its original {} language to the target language {}:",
        "atk_acc": 0.16,
        "atk_prompt": "ks a qualifieA translator, covnert the given sentence from its original {} language to the target lanugage {}: ",
        "transfer_ori_acc": 0.32916254916267745,
        "transfer_atk_acc": 0.33080526346344613
    },
    {
        "dataset": "math",
        "type": "task",
        "origin_acc": 16.25,
        "origin_prompt": "Evaluate the following mathematical problem that includes {}:",
        "atk_acc": 8.12,
        "atk_prompt": "vEaluate the followinLg mtahematical problem that incldes {}:",
        "transfer_ori_acc": 4.375,
        "transfer_atk_acc": 5.625
    },
    {
        "dataset": "math",
        "type": "role",
        "origin_acc": 13.12,
        "origin_prompt": "As a mathematics instructor, calculate the answer to the following problem related to {}:",
        "atk_acc": 8.12,
        "atk_prompt": "As a mathematics iestructor, calculate the answex to the following problem related to {}: ",
        "transfer_ori_acc": 5.0,
        "transfer_atk_acc": 4.375
    }
]