add transferability information

app.py

@@ -1,9 +1,19 @@
-import streamlit as st  
+import streamlit as st
 from parse import retrieve
+from transfer import retrieve_transfer
 
-  
-def main():  
-    st.title("PromptBench")  
+def main():
+    st.sidebar.title("Choose Function")
+    function_choice = st.sidebar.radio("", ["PromptBench", "Retrieve Transferability Information"])
+    
+    if function_choice == "PromptBench":
+        promptbench()
+
+    elif function_choice == "Retrieve Transferability Information":
+        retrieve_transferability_information()
+
+def promptbench():
+    st.title("PromptBench")
   
     model_name = st.selectbox(  
         "Select Model",  
@@ -47,5 +57,47 @@ def main():
             st.write("Attack prompt: {}".format(result["attack prompt"]))
             st.write("Attack acc: {}".format(result["attack acc"]))
   
-if __name__ == "__main__":  
-    main()  
+
+def retrieve_transferability_information():
+    st.title("Retrieve Transferability Information")
+    source_model_name = st.selectbox(  
+        "Select Source Model",  
+        options=["T5", "Vicuna", "UL2", "ChatGPT"],  
+        index=0,  
+    ) 
+
+    target_model_name = st.selectbox(
+        "Select Target Model",
+        options=["T5", "Vicuna", "UL2", "ChatGPT"],
+        index=0,
+    )
+
+    if source_model_name == target_model_name:
+        st.write("Source model and target model cannot be the same.")
+        return
+    
+    attack_name = st.selectbox(
+        "Select Attack",
+        options=[
+            "BertAttack", "CheckList", "DeepWordBug", "StressTest", "TextFooler", "TextBugger", "Semantic"
+        ],
+        index=0,
+    )
+
+    shot = st.selectbox(
+        "Select Shot",
+        options=[0, 3],
+        index=0,
+    )
+
+    data = retrieve_transfer(source_model_name, target_model_name, attack_name, shot)
+    for d in data:
+        st.write(f"Dataset: {d['dataset']}")
+        st.write(f"Prompt Type: {d['type']}-oriented")
+        st.write(f"Origin prompt: {d['origin_prompt']}")
+        st.write(f"Attack prompt: {d['atk_prompt']}")
+        st.write(f"Source model: origin acc: {d['origin_acc']}, attack acc: {d['atk_acc']}")
+        st.write(f"Target model: origin acc: {d['transfer_origin_acc']}, attack acc: {d['transfer_atk_acc']}")
+
+if __name__ == "__main__":
+    main()

results_transfer/chatgpt_google-flan-t5-large/bertattack_0_shot.json

@@ -0,0 +1,242 @@
+[
+    {
+        "dataset": "cola",
+        "type": "task",
+        "origin_acc": 78.5,
+        "origin_prompt": "Assess the grammatical structure of the given sentence and classify it as 'Acceptable' or 'Unacceptable':",
+        "atk_acc": 0.0,
+        "atk_prompt": "what the grammatical structure of the given sentence and consider it as 'Acceptable' or 'Unacceptable':",
+        "transfer_ori_acc": 75.5,
+        "transfer_atk_acc": 76.3
+    },
+    {
+        "dataset": "cola",
+        "type": "role",
+        "origin_acc": 80.0,
+        "origin_prompt": "While performing grammar analysis, classify the grammar of the following sentence as 'acceptable' for correct grammar or 'unacceptable' for incorrect grammar:",
+        "atk_acc": 0.0,
+        "atk_prompt": "While performing grammar analysis, classify the grammar of the following sentence as 'acceptable' for so grammar or 'unacceptable' for abnormal grammar:",
+        "transfer_ori_acc": 75.5,
+        "transfer_atk_acc": 73.3
+    },
+    {
+        "dataset": "mrpc",
+        "type": "task",
+        "origin_acc": 75.5,
+        "origin_prompt": "Examine the pair of sentences and decide if their meanings are identical by answering with 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 56.0,
+        "atk_prompt": "Examine the pairs of pronouns and evaluate if their meanings are different by answering with 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 82.1078431372549,
+        "transfer_atk_acc": 71.81372549019608
+    },
+    {
+        "dataset": "mrpc",
+        "type": "role",
+        "origin_acc": 75.5,
+        "origin_prompt": "In your capacity as a semantic assessment tool, evaluate the provided sentences and classify their meanings as 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 0.0,
+        "atk_prompt": "In your capacity as a semantic assessment tool, evaluate the provided tense and classify their constituents as 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 76.47058823529412,
+        "transfer_atk_acc": 79.41176470588235
+    },
+    {
+        "dataset": "qnli",
+        "type": "task",
+        "origin_acc": 79.0,
+        "origin_prompt": "Assess if the answer to the question can be logically concluded from the provided context by choosing 'entailment' or 'not_entailment'.",
+        "atk_acc": 0.0,
+        "atk_prompt": "and if the answer to the question can be logically concluded from the provided context by avoiding 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 84.8,
+        "transfer_atk_acc": 31.5
+    },
+    {
+        "dataset": "qnli",
+        "type": "role",
+        "origin_acc": 79.5,
+        "origin_prompt": "As a textual inference expert, analyze if the answer to the question can be deduced from the provided context and select 'entailment' or 'not_entailment'.",
+        "atk_acc": 0.0,
+        "atk_prompt": "As a textual inference expert, analysis if the answer to the question can be deduced from the provided citation and define 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 78.10000000000001,
+        "transfer_atk_acc": 60.5
+    },
+    {
+        "dataset": "qqp",
+        "type": "task",
+        "origin_acc": 79.5,
+        "origin_prompt": "Can these two statements be considered equal in meaning? Answer with 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 1.0,
+        "atk_prompt": "Can these two constructions be considered equal in meaning? object with 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 87.8,
+        "transfer_atk_acc": 68.89999999999999
+    },
+    {
+        "dataset": "qqp",
+        "type": "role",
+        "origin_acc": 82.5,
+        "origin_prompt": "As an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions.",
+        "atk_acc": 0.0,
+        "atk_prompt": "give an table for category comparison evaluation, consider the questions and determine if their concept is the same, responding with 'equivalent' for similarity questions or 'not_equivalent' for different knowledge. ",
+        "transfer_ori_acc": 87.3,
+        "transfer_atk_acc": 87.4
+    },
+    {
+        "dataset": "rte",
+        "type": "task",
+        "origin_acc": 82.5,
+        "origin_prompt": "Determine if the given pair of sentences displays entailment or not_entailment. Respond with 'entailment' or 'not_entailment'.",
+        "atk_acc": 55.5,
+        "atk_prompt": "determine if the following singular of sentence indicating entailment or not_entailment. contrast with 'entailment' or 'not_entailment'.",
+        "transfer_ori_acc": 84.11552346570397,
+        "transfer_atk_acc": 82.31046931407943
+    },
+    {
+        "dataset": "rte",
+        "type": "role",
+        "origin_acc": 81.5,
+        "origin_prompt": "Working as an entailment classifier, identify whether the given pair of sentences displays entailment or not_entailment. Respond with 'entailment' or 'not_entailment':",
+        "atk_acc": 37.5,
+        "atk_prompt": "at as an entailment classifier, know which the given type of verb displays entailment or not_entailment. object with 'entailment' or 'not_entailment':",
+        "transfer_ori_acc": 83.03249097472924,
+        "transfer_atk_acc": 81.2274368231047
+    },
+    {
+        "dataset": "sst2",
+        "type": "task",
+        "origin_acc": 90.0,
+        "origin_prompt": "Review this statement and decide whether it has a 'positive' or 'negative' sentiment:",
+        "atk_acc": 0.0,
+        "atk_prompt": "what this assertion and are whether it has a 'positive' or 'negative' sentiment: ",
+        "transfer_ori_acc": 95.06880733944955,
+        "transfer_atk_acc": 94.95412844036697
+    },
+    {
+        "dataset": "sst2",
+        "type": "role",
+        "origin_acc": 93.0,
+        "origin_prompt": "Functioning as a sentiment identification tool, assess if the following expression is 'positive' or 'negative'. Classify:",
+        "atk_acc": 76.0,
+        "atk_prompt": "from as a sentiments identification tool, assess if the same word is 'positive' or 'negative'. function: ",
+        "transfer_ori_acc": 94.83944954128441,
+        "transfer_atk_acc": 93.92201834862385
+    },
+    {
+        "dataset": "wnli",
+        "type": "task",
+        "origin_acc": 69.01,
+        "origin_prompt": "Identify whether the given pair of sentences demonstrates entailment or not_entailment. Answer with 'entailment' or 'not_entailment'.",
+        "atk_acc": 47.89,
+        "atk_prompt": "Identify whether the given individual of verbs demonstrates entailment or not_entailment. have with 'entailment' or 'not_entailment'.",
+        "transfer_ori_acc": 59.154929577464785,
+        "transfer_atk_acc": 52.112676056338024
+    },
+    {
+        "dataset": "wnli",
+        "type": "role",
+        "origin_acc": 60.56,
+        "origin_prompt": "Working as an entailment classifier, identify whether the given pair of sentences displays entailment or not_entailment. Respond with 'entailment' or 'not_entailment':",
+        "atk_acc": 47.89,
+        "atk_prompt": "Working as an entailment classifier, identify whether the given corpus of verbs displays entailment or not_entailment. act with 'entailment' or 'not_entailment':",
+        "transfer_ori_acc": 57.74647887323944,
+        "transfer_atk_acc": 53.52112676056338
+    },
+    {
+        "dataset": "mmlu",
+        "type": "task",
+        "origin_acc": 63.5,
+        "origin_prompt": "Regarding the following multiple-choice question on {}, pick the correct answer from the options 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 14.5,
+        "atk_prompt": "1 the first multiple-choice question on {}, remove the following answer from the options 'A', 'B', 'C', or 'D'. ",
+        "transfer_ori_acc": 44.32624113475177,
+        "transfer_atk_acc": 44.148936170212764
+    },
+    {
+        "dataset": "mmlu",
+        "type": "role",
+        "origin_acc": 62.0,
+        "origin_prompt": "Drawing upon your mastery of {}, please answer the multiple-choice question by selecting the correct option from 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 1.0,
+        "atk_prompt": "Drawing upon your mastery of {}, simply question the multiple-selecting question by selecting the wrong color from 'A', 'B', 'C', or 'D'.",
+        "transfer_ori_acc": 44.32624113475177,
+        "transfer_atk_acc": 25.354609929078016
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "task",
+        "origin_acc": 72.47,
+        "origin_prompt": "Discover the best answer based on the context. If the context doesn't include an answer, respond with 'unanswerable'.",
+        "atk_acc": 2.76,
+        "atk_prompt": "challenge the best answer based on the context. however the context doesn't include an answer, play with 'unanswerable'.",
+        "transfer_ori_acc": 87.54992770387507,
+        "transfer_atk_acc": 87.64992770387508
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "role",
+        "origin_acc": 65.63,
+        "origin_prompt": "Drawing on your expertise in the context, provide the most precise answer. If the answer is not in the context, respond with 'unanswerable'.",
+        "atk_acc": 1.86,
+        "atk_prompt": "Drawing on your technique in the frame, offer the most precise answer. saying the answer is not in the context, play with 'unanswerable'.",
+        "transfer_ori_acc": 88.04992770387507,
+        "transfer_atk_acc": 87.92265497660236
+    },
+    {
+        "dataset": "iwslt",
+        "type": "task",
+        "origin_acc": 0.22,
+        "origin_prompt": "Translate the provided sentence from {} to {} while maintaining the original meaning and context:",
+        "atk_acc": 0.08,
+        "atk_prompt": "Translate the designated sentence from {} to {} while defending the assumed meaning and context:",
+        "transfer_ori_acc": 0.12847010019593297,
+        "transfer_atk_acc": 0.13300283959211967
+    },
+    {
+        "dataset": "iwslt",
+        "type": "role",
+        "origin_acc": 0.22,
+        "origin_prompt": "Taking on the role of a certified translator, perform a translation of the given sentence from {} to {}:",
+        "atk_acc": 0.01,
+        "atk_prompt": "who on the role of a certified translated, request a translator of the defined sentence from {} to {}: ",
+        "transfer_ori_acc": 0.14266566803921382,
+        "transfer_atk_acc": 0.14227772806172803
+    },
+    {
+        "dataset": "un_multi",
+        "type": "task",
+        "origin_acc": 0.34,
+        "origin_prompt": "Translate the following text from the source language {} to the target language {}:",
+        "atk_acc": 0.18,
+        "atk_prompt": "Translate the by text from the source language {} to the english example {}:",
+        "transfer_ori_acc": 0.2655722295717415,
+        "transfer_atk_acc": 0.26924943953069924
+    },
+    {
+        "dataset": "un_multi",
+        "type": "role",
+        "origin_acc": 0.34,
+        "origin_prompt": "In the role of a language expert, perform a machine translation for the provided sentence, changing it from {} to {}:",
+        "atk_acc": 0.08,
+        "atk_prompt": "of the role of a language expertise, define a machine and for the presented sentence, changing it from {} to {}: ",
+        "transfer_ori_acc": 0.26923226069781947,
+        "transfer_atk_acc": 0.26037565925935324
+    },
+    {
+        "dataset": "math",
+        "type": "task",
+        "origin_acc": 6.25,
+        "origin_prompt": "Resolve the following mathematical question associated with {}:",
+        "atk_acc": 0.62,
+        "atk_prompt": "Resolve the following mathematical answer along with {}:",
+        "transfer_ori_acc": 9.375,
+        "transfer_atk_acc": 13.750000000000002
+    },
+    {
+        "dataset": "math",
+        "type": "role",
+        "origin_acc": 5.0,
+        "origin_prompt": "In the position of a mathematics coach, figure out the answer to this math challenge involving {}:",
+        "atk_acc": 0.0,
+        "atk_prompt": "In the position of a mathematics coach, develop out the theme to this math challenge involving {}: ",
+        "transfer_ori_acc": 13.750000000000002,
+        "transfer_atk_acc": 4.375
+    }
+]

results_transfer/chatgpt_google-flan-t5-large/bertattack_3_shot.json

@@ -0,0 +1,242 @@
+[
+    {
+        "dataset": "cola",
+        "type": "task",
+        "origin_acc": 81.0,
+        "origin_prompt": "Analyze the provided sentence and classify its grammatical correctness as 'Acceptable' or 'Unacceptable'.",
+        "atk_acc": 59.0,
+        "atk_prompt": "Analyze the reported speech and classify its inclusive correcting as 'Acceptable' or 'Unacceptable'. ",
+        "transfer_ori_acc": 77.10000000000001,
+        "transfer_atk_acc": 74.1
+    },
+    {
+        "dataset": "cola",
+        "type": "role",
+        "origin_acc": 81.0,
+        "origin_prompt": "In the role of a syntax analyzer, examine the grammar of the provided sentence and decide if it is correct, answering with 'acceptable' for grammatically correct sentences or 'unacceptable' for incorrect ones.",
+        "atk_acc": 75.5,
+        "atk_prompt": "using the art of a level analyzeg, examine the grammar of the provided sentence and decide if it is genuine, answering with 'acceptable' for grammatically correct sentences or 'unacceptable' for incorrect ones. ",
+        "transfer_ori_acc": 74.2,
+        "transfer_atk_acc": 74.0
+    },
+    {
+        "dataset": "mrpc",
+        "type": "task",
+        "origin_acc": 74.0,
+        "origin_prompt": "Compare the following sentences and determine if they share the same semantic meaning by responding with 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 47.5,
+        "atk_prompt": "Compare the following sentences and determine if they lack the same aspect marking by respectively with 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 80.3921568627451,
+        "transfer_atk_acc": 81.61764705882352
+    },
+    {
+        "dataset": "mrpc",
+        "type": "role",
+        "origin_acc": 71.5,
+        "origin_prompt": "In the role of a textual comparison specialist, examine the given sentences and decide if they share the same meaning, responding with 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 47.0,
+        "atk_prompt": "have the role of a textual comparison academic, examine the given sentences and determine if they share the same length, responding with 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 79.65686274509804,
+        "transfer_atk_acc": 76.22549019607843
+    },
+    {
+        "dataset": "qnli",
+        "type": "task",
+        "origin_acc": 84.0,
+        "origin_prompt": "Consider the context and question, and indicate if the answer can be logically deduced from the context by responding with 'entailment' or 'not_entailment'.",
+        "atk_acc": 71.0,
+        "atk_prompt": "put the contexts and question, and be if the answer can be logically deduced from the question by stating with 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 85.39999999999999,
+        "transfer_atk_acc": 78.9
+    },
+    {
+        "dataset": "qnli",
+        "type": "role",
+        "origin_acc": 84.0,
+        "origin_prompt": "As a textual inference expert, analyze if the answer to the question can be deduced from the provided context and select 'entailment' or 'not_entailment'.",
+        "atk_acc": 70.0,
+        "atk_prompt": "As a structural inference expert, analyze if the answer to the challenge can be deduced from the query audience and select 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 77.2,
+        "transfer_atk_acc": 82.8
+    },
+    {
+        "dataset": "qqp",
+        "type": "task",
+        "origin_acc": 82.5,
+        "origin_prompt": "Are the following two questions equivalent or not? Answer me with \"equivalent\" or \"not_equivalent\".",
+        "atk_acc": 78.5,
+        "atk_prompt": "Are the following versus answers equivalent or not? Answer me with \"equivalent\" or \"not_equivalent\". ",
+        "transfer_ori_acc": 87.9,
+        "transfer_atk_acc": 87.7
+    },
+    {
+        "dataset": "qqp",
+        "type": "role",
+        "origin_acc": 83.5,
+        "origin_prompt": "As an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions.",
+        "atk_acc": 78.0,
+        "atk_prompt": "and an instrument for question comparison analysis, consider the questions and establish if their topic is the same, responding with 'equivalent' for matching responses or 'not_equivalent' for between questions. ",
+        "transfer_ori_acc": 86.4,
+        "transfer_atk_acc": 86.5
+    },
+    {
+        "dataset": "rte",
+        "type": "task",
+        "origin_acc": 85.0,
+        "origin_prompt": "Review the two statements and categorize their relationship as either 'entailment' or 'not_entailment'.",
+        "atk_acc": 59.5,
+        "atk_prompt": "write the second statements and categorize their relationship as neither 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 84.83754512635379,
+        "transfer_atk_acc": 84.47653429602889
+    },
+    {
+        "dataset": "rte",
+        "type": "role",
+        "origin_acc": 83.5,
+        "origin_prompt": "While performing entailment analysis, classify the relationship between the provided sentences as 'entailment' or 'not_entailment'.",
+        "atk_acc": 79.0,
+        "atk_prompt": "was performing entailment analysis, classify the relationship between the first phrase as 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 83.39350180505414,
+        "transfer_atk_acc": 83.39350180505414
+    },
+    {
+        "dataset": "sst2",
+        "type": "task",
+        "origin_acc": 95.5,
+        "origin_prompt": "After examining the following expression, label its emotion as either 'positive' or 'negative'.",
+        "atk_acc": 94.5,
+        "atk_prompt": "After examining the following expression, label its emotion as either 'positive' or 'negative' Q47Jmd4lMV. ",
+        "transfer_ori_acc": 93.69266055045871,
+        "transfer_atk_acc": 93.92201834862385
+    },
+    {
+        "dataset": "sst2",
+        "type": "role",
+        "origin_acc": 97.5,
+        "origin_prompt": "In the capacity of a sentiment classifier, decide whether the given quote is 'positive' or 'negative'.",
+        "atk_acc": 96.5,
+        "atk_prompt": "In the capacity of a sentiment classifier, decide whether the given quote is 'positive' or 'negative' 9Q3S1F94fE. ",
+        "transfer_ori_acc": 94.95412844036697,
+        "transfer_atk_acc": 94.95412844036697
+    },
+    {
+        "dataset": "wnli",
+        "type": "task",
+        "origin_acc": 69.01,
+        "origin_prompt": "Review the two statements and categorize their relationship as either 'entailment' or 'not_entailment'.",
+        "atk_acc": 59.15,
+        "atk_prompt": "examine the paired actions and categorize their relationship as is 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 63.38028169014085,
+        "transfer_atk_acc": 66.19718309859155
+    },
+    {
+        "dataset": "wnli",
+        "type": "role",
+        "origin_acc": 66.2,
+        "origin_prompt": "In the role of a semantic relationship analyzer, examine the connection between the given sentences and decide if they exhibit entailment or not_entailment. Answer with 'entailment' or 'not_entailment'.",
+        "atk_acc": 56.34,
+        "atk_prompt": "In the role of a semantic relationship analyzer, define the connection between the given links and decide if they display entailment or not_entailment. Answer with 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 56.33802816901409,
+        "transfer_atk_acc": 66.19718309859155
+    },
+    {
+        "dataset": "mmlu",
+        "type": "task",
+        "origin_acc": 66.5,
+        "origin_prompt": "Please respond to the multiple-choice question about {} by selecting the appropriate answer: 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 11.5,
+        "atk_prompt": "at respond to the multiple-independent question about {} by selecting the inappropriate question: 'A', 'B', 'C', or 'D'. ",
+        "transfer_ori_acc": 45.21276595744681,
+        "transfer_atk_acc": 33.51063829787234
+    },
+    {
+        "dataset": "mmlu",
+        "type": "role",
+        "origin_acc": 65.5,
+        "origin_prompt": "Considering your familiarity with {}, attend to the following multiple-choice question by picking 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 52.5,
+        "atk_prompt": "all your ability with {}, attend to the following multiple-select question by picking 'A', 'B', 'C', or 'D'.",
+        "transfer_ori_acc": 46.98581560283688,
+        "transfer_atk_acc": 45.92198581560284
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "task",
+        "origin_acc": 72.25,
+        "origin_prompt": "Based on the given context, provide the best possible answer. If there's no answer available in the context, reply with 'unanswerable'.",
+        "atk_acc": 65.81,
+        "atk_prompt": "Based on the use context, provide the best possible answer. who there'tis no answer provided in the area, reply with 'unanswerable'.",
+        "transfer_ori_acc": 87.04992770387507,
+        "transfer_atk_acc": 87.54992770387507
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "role",
+        "origin_acc": 72.45,
+        "origin_prompt": "As a proficient expert in the given context, search for the most relevant answer. If the answer cannot be found, respond by saying 'unanswerable'.",
+        "atk_acc": 61.02,
+        "atk_prompt": "As a proficient expert in the given context, try for the most specific out. If the question lacks be search, end by as 'unanswerable'.",
+        "transfer_ori_acc": 87.31659437054175,
+        "transfer_atk_acc": 87.68023073417811
+    },
+    {
+        "dataset": "iwslt",
+        "type": "task",
+        "origin_acc": 0.24,
+        "origin_prompt": "Convert the following sentence from its original {} language to the target language {}.",
+        "atk_acc": 0.21,
+        "atk_prompt": "Convert the following sentence from its actual {} language to the net language {}. ",
+        "transfer_ori_acc": 0.2240211126734209,
+        "transfer_atk_acc": 0.22296244610321242
+    },
+    {
+        "dataset": "iwslt",
+        "type": "role",
+        "origin_acc": 0.24,
+        "origin_prompt": "Assuming the responsibilities of a professional translator, translate the subsequent text passage from {} to {}.",
+        "atk_acc": 0.17,
+        "atk_prompt": "what the responsibilities of a professional scholar, translate the subsequent text passage from {} to {}. ",
+        "transfer_ori_acc": 0.22030199520456395,
+        "transfer_atk_acc": 0.22001437607394084
+    },
+    {
+        "dataset": "un_multi",
+        "type": "task",
+        "origin_acc": 0.36,
+        "origin_prompt": "Translate the following text from the source language {} to the target language {}.",
+        "atk_acc": 0.35,
+        "atk_prompt": "Translate the following area from the standard language {} to the target language {}. ",
+        "transfer_ori_acc": 0.3090347352409074,
+        "transfer_atk_acc": 0.30893566132252137
+    },
+    {
+        "dataset": "un_multi",
+        "type": "role",
+        "origin_acc": 0.35,
+        "origin_prompt": "As a qualified translator, convert the given sentence from its original {} language to the target language {}.",
+        "atk_acc": 0.26,
+        "atk_prompt": "As a qualified proper, converts the target sentence from its original {} language to the english language {}. ",
+        "transfer_ori_acc": 0.31127691224033316,
+        "transfer_atk_acc": 0.306608167162218
+    },
+    {
+        "dataset": "math",
+        "type": "task",
+        "origin_acc": 22.5,
+        "origin_prompt": "Compute the result of this math task concerning {}.",
+        "atk_acc": 3.12,
+        "atk_prompt": "Compute the degree of this math task concerning {}. ",
+        "transfer_ori_acc": 16.25,
+        "transfer_atk_acc": 16.875
+    },
+    {
+        "dataset": "math",
+        "type": "role",
+        "origin_acc": 22.5,
+        "origin_prompt": "As a mathematics instructor, calculate the answer to the following problem related to {}.",
+        "atk_acc": 7.5,
+        "atk_prompt": "as a mathematics instructor, evaluate the reply to the following problem and to {}. ",
+        "transfer_ori_acc": 17.5,
+        "transfer_atk_acc": 16.25
+    }
+]

results_transfer/chatgpt_google-flan-t5-large/checklist_0_shot.json

@@ -0,0 +1,242 @@
+[
+    {
+        "dataset": "cola",
+        "type": "task",
+        "origin_acc": 78.5,
+        "origin_prompt": "Review the sentence below and identify whether its grammar is 'Acceptable' or 'Unacceptable':",
+        "atk_acc": 72.5,
+        "atk_prompt": "Review the sentence below and identify whether its grammar is 'Acceptable' or 'Unacceptable' zexKUTWJYG:",
+        "transfer_ori_acc": 76.5,
+        "transfer_atk_acc": 75.5
+    },
+    {
+        "dataset": "cola",
+        "type": "role",
+        "origin_acc": 79.0,
+        "origin_prompt": "While performing grammar analysis, classify the grammar of the following sentence as 'acceptable' for correct grammar or 'unacceptable' for incorrect grammar:",
+        "atk_acc": 45.0,
+        "atk_prompt": "While performing grammar analysis, classify the grammar of the following sentence as 'acceptable' for correct grammar or 'unacceptable' for incorrect grammar zexKUTWJYG:",
+        "transfer_ori_acc": 75.5,
+        "transfer_atk_acc": 74.1
+    },
+    {
+        "dataset": "mrpc",
+        "type": "task",
+        "origin_acc": 75.5,
+        "origin_prompt": "Evaluate whether the two provided sentences convey the same meaning by answering 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 72.0,
+        "atk_prompt": "Evaluate whether the two provided sentences convey the same meaning by answering 'equivalent' or 'not_equivalent' SvAp8RlOFn. ",
+        "transfer_ori_acc": 81.86274509803921,
+        "transfer_atk_acc": 82.35294117647058
+    },
+    {
+        "dataset": "mrpc",
+        "type": "role",
+        "origin_acc": 75.5,
+        "origin_prompt": "In your capacity as a semantic assessment tool, evaluate the provided sentences and classify their meanings as 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 73.5,
+        "atk_prompt": "In your capacity as a semantic assessment tool, evaluate the provided sentences and classify their meanings as 'equivalent' or 'not_equivalent' Kw6nrs57gH. ",
+        "transfer_ori_acc": 76.47058823529412,
+        "transfer_atk_acc": 74.75490196078431
+    },
+    {
+        "dataset": "qnli",
+        "type": "task",
+        "origin_acc": 78.0,
+        "origin_prompt": "Evaluate whether the given context supports the answer to the question by responding with 'entailment' or 'not_entailment'.",
+        "atk_acc": 75.5,
+        "atk_prompt": "Evaluate whether the given context supports the answer to the question by responding with 'entailment' or 'not_entailment' ItGDrrA1Sc. ",
+        "transfer_ori_acc": 90.4,
+        "transfer_atk_acc": 89.5
+    },
+    {
+        "dataset": "qnli",
+        "type": "role",
+        "origin_acc": 78.0,
+        "origin_prompt": "As a semantic researcher, evaluate whether the provided context supports the answer to the question and choose 'entailment' or 'not_entailment'.",
+        "atk_acc": 74.5,
+        "atk_prompt": "As a semantic researcher, evaluate whether the provided context supports the answer to the question and choose 'entailment' or 'not_entailment' 6a4Yn3RGVc. ",
+        "transfer_ori_acc": 89.1,
+        "transfer_atk_acc": 88.7
+    },
+    {
+        "dataset": "qqp",
+        "type": "task",
+        "origin_acc": 81.5,
+        "origin_prompt": "Assess whether the following statements are identical in meaning by answering 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 78.0,
+        "atk_prompt": "Assess whether the following statements are identical in meaning by answering 'equivalent' or 'not_equivalent' iia2yL9Uzm. ",
+        "transfer_ori_acc": 87.6,
+        "transfer_atk_acc": 87.7
+    },
+    {
+        "dataset": "qqp",
+        "type": "role",
+        "origin_acc": 83.0,
+        "origin_prompt": "As an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions.",
+        "atk_acc": 80.5,
+        "atk_prompt": "As an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions Cev0s49fbe. ",
+        "transfer_ori_acc": 87.3,
+        "transfer_atk_acc": 87.5
+    },
+    {
+        "dataset": "rte",
+        "type": "task",
+        "origin_acc": 82.5,
+        "origin_prompt": "Determine if the given pair of sentences displays entailment or not_entailment. Respond with 'entailment' or 'not_entailment'.",
+        "atk_acc": 80.0,
+        "atk_prompt": "Determine if the given pair of sentences displays entailment or not_entailment. Respond with 'entailment' or 'not_entailment' MSAddJ4D2a.",
+        "transfer_ori_acc": 84.11552346570397,
+        "transfer_atk_acc": 84.11552346570397
+    },
+    {
+        "dataset": "rte",
+        "type": "role",
+        "origin_acc": 81.5,
+        "origin_prompt": "As an instrument for entailment evaluation, consider the two sentences and determine if their relationship is 'entailment' or 'not_entailment'. Respond with 'entailment' or 'not_entailment':",
+        "atk_acc": 80.0,
+        "atk_prompt": "As an instrument for entailment evaluation, consider the two sentences and determine if their relationship is 'entailment' or 'not_entailment'. Respond with 'entailment' or 'not_entailment' 6a4Yn3RGVc:",
+        "transfer_ori_acc": 85.5595667870036,
+        "transfer_atk_acc": 85.92057761732852
+    },
+    {
+        "dataset": "sst2",
+        "type": "task",
+        "origin_acc": 93.0,
+        "origin_prompt": "Determine the overall sentiment of this sentence, categorizing it as 'positive' or 'negative':",
+        "atk_acc": 90.0,
+        "atk_prompt": "Determine the overall sentiment of this sentence, categorizing it as 'positive' or 'negative' AMsRIKZniY: ",
+        "transfer_ori_acc": 94.26605504587155,
+        "transfer_atk_acc": 94.38073394495413
+    },
+    {
+        "dataset": "sst2",
+        "type": "role",
+        "origin_acc": 92.5,
+        "origin_prompt": "Functioning as a sentiment identification tool, assess if the following expression is 'positive' or 'negative'. Classify:",
+        "atk_acc": 92.0,
+        "atk_prompt": "Functioning as a sentiment identification tool, assess if the following expression is 'positive' or 'negative'. Classify Cev0s49fbe: ",
+        "transfer_ori_acc": 94.83944954128441,
+        "transfer_atk_acc": 94.83944954128441
+    },
+    {
+        "dataset": "wnli",
+        "type": "task",
+        "origin_acc": 64.79,
+        "origin_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word.",
+        "atk_acc": 59.15,
+        "atk_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word zexKUTWJYG. ",
+        "transfer_ori_acc": 66.19718309859155,
+        "transfer_atk_acc": 66.19718309859155
+    },
+    {
+        "dataset": "wnli",
+        "type": "role",
+        "origin_acc": 63.38,
+        "origin_prompt": "In the role of a semantic relationship analyzer, examine the connection between the given sentences and decide if they exhibit entailment or not_entailment. Answer with 'entailment' or 'not_entailment':",
+        "atk_acc": 57.75,
+        "atk_prompt": "In the role of a semantic relationship analyzer, examine the connection between the given sentences and decide if they exhibit entailment or not_entailment. Answer with 'entailment' or 'not_entailment' 5XeflW1ZJc:",
+        "transfer_ori_acc": 54.929577464788736,
+        "transfer_atk_acc": 53.52112676056338
+    },
+    {
+        "dataset": "mmlu",
+        "type": "task",
+        "origin_acc": 64.0,
+        "origin_prompt": "Regarding the following multiple-choice question on {}, pick the correct answer from the options 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 58.0,
+        "atk_prompt": "Regarding the following multiple-choice question on {}, pick the correct answer from the options 'A', 'B', 'C', or 'D' Pbl3tYuoRo. ",
+        "transfer_ori_acc": 44.32624113475177,
+        "transfer_atk_acc": 44.50354609929078
+    },
+    {
+        "dataset": "mmlu",
+        "type": "role",
+        "origin_acc": 61.5,
+        "origin_prompt": "As a skilled person in the field of {}, please respond to the multiple-choice question by choosing 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 58.0,
+        "atk_prompt": "As a skilled person in the field of {}, please respond to the multiple-choice question by choosing 'A', 'B', 'C', or 'D' IhuBIhoPGc.",
+        "transfer_ori_acc": 44.680851063829785,
+        "transfer_atk_acc": 45.0354609929078
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "task",
+        "origin_acc": 69.16,
+        "origin_prompt": "Based on the given context, provide the best possible answer. If there's no answer available in the context, respond with 'unanswerable'.",
+        "atk_acc": 64.99,
+        "atk_prompt": "Based on the given context, provide the best possible answer. If there's no answer available in the context, respond with 'unanswerable' ItGDrrA1Sc.",
+        "transfer_ori_acc": 87.04992770387507,
+        "transfer_atk_acc": 87.54992770387507
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "role",
+        "origin_acc": 64.31,
+        "origin_prompt": "As an expert with a deep understanding of the context, find the best answer. If the context doesn't include an answer, say 'unanswerable'.",
+        "atk_acc": 60.2,
+        "atk_prompt": "As an expert with a deep understanding of the context, find the best answer. If the context doesn't include an answer, say 'unanswerable' d6ZQ3u0GBQ.",
+        "transfer_ori_acc": 88.04992770387507,
+        "transfer_atk_acc": 88.04992770387507
+    },
+    {
+        "dataset": "iwslt",
+        "type": "task",
+        "origin_acc": 0.22,
+        "origin_prompt": "Translate the provided sentence from {} to {} while maintaining the original meaning and context:",
+        "atk_acc": 0.21,
+        "atk_prompt": "Translate the provided sentence from {} to {} while maintaining the original meaning and context zq0DcZ5dnI:",
+        "transfer_ori_acc": 0.12847010019593297,
+        "transfer_atk_acc": 0.13452105947739143
+    },
+    {
+        "dataset": "iwslt",
+        "type": "role",
+        "origin_acc": 0.22,
+        "origin_prompt": "In the capacity of a proficient translator, please change the following sentence from {} to {}:",
+        "atk_acc": 0.21,
+        "atk_prompt": "In the capacity of a proficient translator, please change the following sentence from {} to SFPCqpiClT {}: ",
+        "transfer_ori_acc": 0.13904950536673938,
+        "transfer_atk_acc": 0.12181412545498613
+    },
+    {
+        "dataset": "un_multi",
+        "type": "task",
+        "origin_acc": 0.34,
+        "origin_prompt": "Perform a machine translation for the provided sentence, changing it from {} to {}:",
+        "atk_acc": 0.33,
+        "atk_prompt": "Perform a machine translation for the provided sentence, changing it from {} to Kw6nrs57gH {}:",
+        "transfer_ori_acc": 0.26814405340158265,
+        "transfer_atk_acc": 0.26255854470098927
+    },
+    {
+        "dataset": "un_multi",
+        "type": "role",
+        "origin_acc": 0.34,
+        "origin_prompt": "While acting as an experienced language expert, translate the following text from {} to {}:",
+        "atk_acc": 0.33,
+        "atk_prompt": "While acting as an experienced language expert, translate the following text from {} to Kw6nrs57gH {}: ",
+        "transfer_ori_acc": 0.26489202618745367,
+        "transfer_atk_acc": 0.25410191629878426
+    },
+    {
+        "dataset": "math",
+        "type": "task",
+        "origin_acc": 6.88,
+        "origin_prompt": "Calculate the answer to the following math query about {}:",
+        "atk_acc": 5.0,
+        "atk_prompt": "Calculate the answer to the following math query about 6a4Yn3RGVc {}:",
+        "transfer_ori_acc": 13.125,
+        "transfer_atk_acc": 11.875
+    },
+    {
+        "dataset": "math",
+        "type": "role",
+        "origin_acc": 5.0,
+        "origin_prompt": "Embodying a mathematics advisor, find the answer to the following math problem with {}:",
+        "atk_acc": 2.5,
+        "atk_prompt": "Embodying a mathematics advisor, find the answer to the following math problem with 9vGXgnbWB8 {}: ",
+        "transfer_ori_acc": 11.25,
+        "transfer_atk_acc": 12.5
+    }
+]

results_transfer/chatgpt_google-flan-t5-large/checklist_3_shot.json

@@ -0,0 +1,242 @@
+[
+    {
+        "dataset": "cola",
+        "type": "task",
+        "origin_acc": 81.0,
+        "origin_prompt": "Examine the sentence and decide if its grammar is 'Acceptable' or 'Unacceptable'.",
+        "atk_acc": 78.0,
+        "atk_prompt": "Examine the sentence and decide if its grammar is 'Acceptable' or 'Unacceptable' 9Q3S1F94fE. ",
+        "transfer_ori_acc": 76.3,
+        "transfer_atk_acc": 76.6
+    },
+    {
+        "dataset": "cola",
+        "type": "role",
+        "origin_acc": 81.5,
+        "origin_prompt": "Functioning as a grammar evaluation tool, analyze the given sentence and decide if it is grammatically correct, responding with 'acceptable' or 'unacceptable'.",
+        "atk_acc": 79.5,
+        "atk_prompt": "Functioning as a grammar evaluation tool, analyze the given sentence and decide if it is grammatically correct, responding with 'acceptable' or 'unacceptable' 6a4Yn3RGVc. ",
+        "transfer_ori_acc": 75.2,
+        "transfer_atk_acc": 76.3
+    },
+    {
+        "dataset": "mrpc",
+        "type": "task",
+        "origin_acc": 74.0,
+        "origin_prompt": "Assess if the two given sentences have equivalent meanings by selecting 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 68.5,
+        "atk_prompt": "Assess if the two given sentences have equivalent meanings by selecting 'equivalent' or 'not_equivalent' LkF0FZxMZ4. ",
+        "transfer_ori_acc": 82.84313725490196,
+        "transfer_atk_acc": 82.59803921568627
+    },
+    {
+        "dataset": "mrpc",
+        "type": "role",
+        "origin_acc": 72.0,
+        "origin_prompt": "As a linguistic comparator, review the following pair of sentences and determine their semantic equivalence by choosing 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 68.0,
+        "atk_prompt": "As a linguistic comparator, review the following pair of sentences and determine their semantic equivalence by choosing 'equivalent' or 'not_equivalent' bhrRSokrfa. ",
+        "transfer_ori_acc": 79.90196078431373,
+        "transfer_atk_acc": 80.63725490196079
+    },
+    {
+        "dataset": "qnli",
+        "type": "task",
+        "origin_acc": 84.5,
+        "origin_prompt": "Examine the context and question, and determine if the context logically implies the answer by selecting 'entailment' or 'not_entailment'.",
+        "atk_acc": 80.5,
+        "atk_prompt": "Examine the context and question, and determine if the context logically implies the answer by selecting 'entailment' or 'not_entailment' 9Q3S1F94fE. ",
+        "transfer_ori_acc": 86.5,
+        "transfer_atk_acc": 86.5
+    },
+    {
+        "dataset": "qnli",
+        "type": "role",
+        "origin_acc": 84.5,
+        "origin_prompt": "As a textual inference expert, analyze if the answer to the question can be deduced from the provided context and select 'entailment' or 'not_entailment'.",
+        "atk_acc": 81.0,
+        "atk_prompt": "As a textual inference expert, analyze if the answer to the question can be deduced from the provided context and select 'entailment' or 'not_entailment' YHQiDfFhw4. ",
+        "transfer_ori_acc": 77.2,
+        "transfer_atk_acc": 74.2
+    },
+    {
+        "dataset": "qqp",
+        "type": "task",
+        "origin_acc": 82.0,
+        "origin_prompt": "Can these two statements be considered equal in meaning? Answer with 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 79.0,
+        "atk_prompt": "Can these two statements be considered equal in meaning? Answer with 'equivalent' or 'not_equivalent' xikCjkMydH. ",
+        "transfer_ori_acc": 87.6,
+        "transfer_atk_acc": 87.8
+    },
+    {
+        "dataset": "qqp",
+        "type": "role",
+        "origin_acc": 83.0,
+        "origin_prompt": "In the capacity of a question assessment system, indicate if the meaning of the provided questions is the same, responding with 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 81.5,
+        "atk_prompt": "In the capacity of a question assessment system, indicate if the meaning of the provided questions is the same, responding with 'equivalent' or 'not_equivalent' 5XeflW1ZJc. ",
+        "transfer_ori_acc": 86.1,
+        "transfer_atk_acc": 85.9
+    },
+    {
+        "dataset": "rte",
+        "type": "task",
+        "origin_acc": 84.5,
+        "origin_prompt": "Review the two statements and categorize their relationship as either 'entailment' or 'not_entailment'.",
+        "atk_acc": 80.5,
+        "atk_prompt": "Review the two statements and categorize their relationship as either 'entailment' or 'not_entailment' 5XeflW1ZJc. ",
+        "transfer_ori_acc": 84.83754512635379,
+        "transfer_atk_acc": 85.5595667870036
+    },
+    {
+        "dataset": "rte",
+        "type": "role",
+        "origin_acc": 84.0,
+        "origin_prompt": "In your role as an entailment analysis tool, assess the relationship between the given sentences and classify it as 'entailment' or 'not_entailment'.",
+        "atk_acc": 80.5,
+        "atk_prompt": "In your role as an entailment analysis tool, assess the relationship between the given sentences and classify it as 'entailment' or 'not_entailment' bhrRSokrfa. ",
+        "transfer_ori_acc": 83.03249097472924,
+        "transfer_atk_acc": 83.03249097472924
+    },
+    {
+        "dataset": "sst2",
+        "type": "task",
+        "origin_acc": 96.0,
+        "origin_prompt": "Read the provided excerpt and choose between 'positive' and 'negative' to describe its sentiment.",
+        "atk_acc": 77.5,
+        "atk_prompt": "go the read excerpt and be between 'positive' and 'negative' to describing its mood. ",
+        "transfer_ori_acc": 94.72477064220183,
+        "transfer_atk_acc": 94.61009174311926
+    },
+    {
+        "dataset": "sst2",
+        "type": "role",
+        "origin_acc": 97.5,
+        "origin_prompt": "In the capacity of a sentiment classifier, decide whether the given quote is 'positive' or 'negative'.",
+        "atk_acc": 13.5,
+        "atk_prompt": "in the capacity of a sentiment classizer, chose neither the said quotes is 'positive' or 'negative'. ",
+        "transfer_ori_acc": 94.95412844036697,
+        "transfer_atk_acc": 94.95412844036697
+    },
+    {
+        "dataset": "wnli",
+        "type": "task",
+        "origin_acc": 69.01,
+        "origin_prompt": "Indicate if the connection between the following sentences is 'entailment' or 'not_entailment'.",
+        "atk_acc": 61.97,
+        "atk_prompt": "Indicate if the connection between the followping senZences is 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 64.7887323943662,
+        "transfer_atk_acc": 69.01408450704226
+    },
+    {
+        "dataset": "wnli",
+        "type": "role",
+        "origin_acc": 66.2,
+        "origin_prompt": "In your role as an entailment analysis tool, assess the relationship between the given sentences and classify it as 'entailment' or 'not_entailment'.",
+        "atk_acc": 57.75,
+        "atk_prompt": "In your role as an entailment analysis tool, asress the rlationship between the given sentences and classify it as 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 57.74647887323944,
+        "transfer_atk_acc": 54.929577464788736
+    },
+    {
+        "dataset": "mmlu",
+        "type": "task",
+        "origin_acc": 66.0,
+        "origin_prompt": "Please respond to the multiple-choice question about {} by selecting the appropriate answer: 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 63.0,
+        "atk_prompt": "Please respond to the multiple-choice question about {} by selecting the appropriate answer: 'A', 'B', 'C', or 'D' MQnugHcaoy. ",
+        "transfer_ori_acc": 45.21276595744681,
+        "transfer_atk_acc": 45.21276595744681
+    },
+    {
+        "dataset": "mmlu",
+        "type": "role",
+        "origin_acc": 65.5,
+        "origin_prompt": "As someone well-versed in {}, please address the multiple-choice question below by selecting 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 63.0,
+        "atk_prompt": "As someone well-versed in {}, please address the multiple-choice question below by selecting 'A', 'B', 'C', or 'D' mOJ9nEwT4f.",
+        "transfer_ori_acc": 46.63120567375886,
+        "transfer_atk_acc": 46.98581560283688
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "task",
+        "origin_acc": 73.25,
+        "origin_prompt": "Based on the given context, provide the best possible answer. If there's no answer available in the context, reply with 'unanswerable'.",
+        "atk_acc": 71.09,
+        "atk_prompt": "Based on the given context, provide the best possible answer. If there's no answer available in the context, reply with 'unanswerable' ItGDrrA1Sc.",
+        "transfer_ori_acc": 87.04992770387507,
+        "transfer_atk_acc": 87.04992770387507
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "role",
+        "origin_acc": 72.63,
+        "origin_prompt": "As a proficient expert in the given context, search for the most relevant answer. If the answer cannot be found, respond by saying 'unanswerable'.",
+        "atk_acc": 63.07,
+        "atk_prompt": "As a proficient expert in the given context, search for the most relevant answer. If the answer cannot be found, respond by saying 'unanswerable' XStsD2pUzu.",
+        "transfer_ori_acc": 87.31659437054175,
+        "transfer_atk_acc": 87.31659437054175
+    },
+    {
+        "dataset": "iwslt",
+        "type": "task",
+        "origin_acc": 0.24,
+        "origin_prompt": "Convert the following sentence from its original {} language to the target language {}.",
+        "atk_acc": 0.22,
+        "atk_prompt": "Convert the following sentence from its original {} language to the target language 5XeflW1ZJc {}. ",
+        "transfer_ori_acc": 0.2240211126734209,
+        "transfer_atk_acc": 0.22156249809369105
+    },
+    {
+        "dataset": "iwslt",
+        "type": "role",
+        "origin_acc": 0.24,
+        "origin_prompt": "In the role of a language expert, perform a machine translation for the provided sentence, changing it from {} to {}.",
+        "atk_acc": 0.23,
+        "atk_prompt": "In the role of a language expert, perform a machine translation for the provided sentence, changing it from {} to MQnugHcaoy {}. ",
+        "transfer_ori_acc": 0.22070346732323043,
+        "transfer_atk_acc": 0.2112357399871226
+    },
+    {
+        "dataset": "un_multi",
+        "type": "task",
+        "origin_acc": 0.35,
+        "origin_prompt": "Perform a machine translation for the provided sentence, changing it from {} to {}.",
+        "atk_acc": 0.34,
+        "atk_prompt": "Perform a machine translation for the provided sentence, changing it from {} to Kw6nrs57gH {}. ",
+        "transfer_ori_acc": 0.30870734220870066,
+        "transfer_atk_acc": 0.3125050772206132
+    },
+    {
+        "dataset": "un_multi",
+        "type": "role",
+        "origin_acc": 0.35,
+        "origin_prompt": "As a qualified translator, convert the given sentence from its original {} language to the target language {}.",
+        "atk_acc": 0.31,
+        "atk_prompt": "As a qualified translator, convert the given sentence from its original {} language to the target language 5XeflW1ZJc {}. ",
+        "transfer_ori_acc": 0.31127691224033316,
+        "transfer_atk_acc": 0.31337641480593154
+    },
+    {
+        "dataset": "math",
+        "type": "task",
+        "origin_acc": 23.12,
+        "origin_prompt": "Calculate the answer to the following math query about {}.",
+        "atk_acc": 19.38,
+        "atk_prompt": "Calculate the answer to the following math query about 5XeflW1ZJc {}. ",
+        "transfer_ori_acc": 16.25,
+        "transfer_atk_acc": 17.5
+    },
+    {
+        "dataset": "math",
+        "type": "role",
+        "origin_acc": 22.5,
+        "origin_prompt": "As a mathematics instructor, calculate the answer to the following problem related to {}.",
+        "atk_acc": 21.25,
+        "atk_prompt": "As a mathematics instructor, calculate the answer to the following problem related to AMsRIKZniY {}. ",
+        "transfer_ori_acc": 17.5,
+        "transfer_atk_acc": 17.5
+    }
+]

results_transfer/chatgpt_google-flan-t5-large/deepwordbug_0_shot.json

@@ -0,0 +1,242 @@
+[
+    {
+        "dataset": "cola",
+        "type": "task",
+        "origin_acc": 79.0,
+        "origin_prompt": "Review the sentence below and identify whether its grammar is 'Acceptable' or 'Unacceptable':",
+        "atk_acc": 74.5,
+        "atk_prompt": "Review the sentece below and identify whether its grammar is 'Acceptable' or 'Unacceptable':",
+        "transfer_ori_acc": 76.5,
+        "transfer_atk_acc": 73.4
+    },
+    {
+        "dataset": "cola",
+        "type": "role",
+        "origin_acc": 79.5,
+        "origin_prompt": "While performing grammar analysis, classify the grammar of the following sentence as 'acceptable' for correct grammar or 'unacceptable' for incorrect grammar:",
+        "atk_acc": 0.0,
+        "atk_prompt": "nhile performing grammr analysis, classify the rgammar of the fllowing sentenZe as 'acceptable' for corFect Xrammar or 'unacceptable' for incorrect gramUar:",
+        "transfer_ori_acc": 75.5,
+        "transfer_atk_acc": 72.5
+    },
+    {
+        "dataset": "mrpc",
+        "type": "task",
+        "origin_acc": 74.0,
+        "origin_prompt": "Evaluate whether the two provided sentences convey the same meaning by answering 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 70.0,
+        "atk_prompt": "Evaluate whether the two provRided sentnces convey the same meaning by answering 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 81.86274509803921,
+        "transfer_atk_acc": 78.43137254901961
+    },
+    {
+        "dataset": "mrpc",
+        "type": "role",
+        "origin_acc": 76.0,
+        "origin_prompt": "In your capacity as a semantic assessment tool, evaluate the provided sentences and classify their meanings as 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 53.0,
+        "atk_prompt": "nI your capacity as a sematnic asseJssment tol, evaluate the prvoided senRtences and classify their meainngs as 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 76.47058823529412,
+        "transfer_atk_acc": 78.67647058823529
+    },
+    {
+        "dataset": "qnli",
+        "type": "task",
+        "origin_acc": 78.0,
+        "origin_prompt": "Evaluate whether the given context supports the answer to the question by responding with 'entailment' or 'not_entailment'.",
+        "atk_acc": 50.0,
+        "atk_prompt": "Exaluate whetxher the iven contet Supports the anwser to the queston by respEonding with 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 90.4,
+        "transfer_atk_acc": 50.1
+    },
+    {
+        "dataset": "qnli",
+        "type": "role",
+        "origin_acc": 79.5,
+        "origin_prompt": "As a textual inference expert, analyze if the answer to the question can be deduced from the provided context and select 'entailment' or 'not_entailment'.",
+        "atk_acc": 19.5,
+        "atk_prompt": "As a textula inference expert, aInalyze if the answer to the question can be deducud from the provided context and Melect 'entailment' or 'not_entailment'. ",
+        "transfer_ori_acc": 78.10000000000001,
+        "transfer_atk_acc": 81.6
+    },
+    {
+        "dataset": "qqp",
+        "type": "task",
+        "origin_acc": 82.0,
+        "origin_prompt": "Assess whether the following statements are identical in meaning by answering 'equivalent' or 'not_equivalent'.",
+        "atk_acc": 45.0,
+        "atk_prompt": "Assess whethdr the followng sLtatements are identocal in meuaning by answerinJ 'equivalent' or 'not_equivalent'. ",
+        "transfer_ori_acc": 87.6,
+        "transfer_atk_acc": 85.0
+    },
+    {
+        "dataset": "qqp",
+        "type": "role",
+        "origin_acc": 82.5,
+        "origin_prompt": "As an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions.",
+        "atk_acc": 79.5,
+        "atk_prompt": "Ao an instrument for question comparison evaluation, consider the questions and determine if their meaning is the same, responding with 'equivalent' for similar questions or 'not_equivalent' for different questions. ",
+        "transfer_ori_acc": 87.3,
+        "transfer_atk_acc": 86.9
+    },
+    {
+        "dataset": "rte",
+        "type": "task",
+        "origin_acc": 83.5,
+        "origin_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word.",
+        "atk_acc": 79.0,
+        "atk_prompt": "ArYe the folowing two sentenes entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word. ",
+        "transfer_ori_acc": 88.08664259927798,
+        "transfer_atk_acc": 83.39350180505414
+    },
+    {
+        "dataset": "rte",
+        "type": "role",
+        "origin_acc": 80.5,
+        "origin_prompt": "As an instrument for entailment evaluation, consider the two sentences and determine if their relationship is 'entailment' or 'not_entailment'. Respond with 'entailment' or 'not_entailment':",
+        "atk_acc": 78.5,
+        "atk_prompt": "IAs an instrument for entailment evaluatioRn, consider the otwo sentences and determine if their relationship is 'entailment' or 'not_entailment'. Respond with 'entailment' or 'not_entailment':",
+        "transfer_ori_acc": 85.5595667870036,
+        "transfer_atk_acc": 85.92057761732852
+    },
+    {
+        "dataset": "sst2",
+        "type": "task",
+        "origin_acc": 93.0,
+        "origin_prompt": "Determine the overall sentiment of this sentence, categorizing it as 'positive' or 'negative':",
+        "atk_acc": 88.5,
+        "atk_prompt": "Deetrmine the overall sentLment of this sentence, categorizing it as 'positive' or 'negative': ",
+        "transfer_ori_acc": 94.26605504587155,
+        "transfer_atk_acc": 94.72477064220183
+    },
+    {
+        "dataset": "sst2",
+        "type": "role",
+        "origin_acc": 93.0,
+        "origin_prompt": "Functioning as a sentiment identification tool, assess if the following expression is 'positive' or 'negative'. Classify:",
+        "atk_acc": 85.0,
+        "atk_prompt": "uFnctioning as a sentmient identification tool, assess if the folGowing expression is 'positive' or 'negative'. Classify: ",
+        "transfer_ori_acc": 94.83944954128441,
+        "transfer_atk_acc": 93.34862385321101
+    },
+    {
+        "dataset": "wnli",
+        "type": "task",
+        "origin_acc": 66.2,
+        "origin_prompt": "Examine the pair of sentences and determine if they exhibit entailment or not_entailment. Answer with either 'entailment' or 'not_entailment'.",
+        "atk_acc": 53.52,
+        "atk_prompt": "Examine the Bair of snetences and determine if they exhibit entailment or not_entailment. Ancwer with eMther 'entailment' or 'not_entailment'.",
+        "transfer_ori_acc": 61.97183098591549,
+        "transfer_atk_acc": 63.38028169014085
+    },
+    {
+        "dataset": "wnli",
+        "type": "role",
+        "origin_acc": 63.38,
+        "origin_prompt": "In the role of a semantic relationship analyzer, examine the connection between the given sentences and decide if they exhibit entailment or not_entailment. Answer with 'entailment' or 'not_entailment':",
+        "atk_acc": 53.52,
+        "atk_prompt": "IQn the role of a semantic relationship analyzer, eamine the connection between the given sentences and decide if they exhbit entailment or not_entailment. Answer with 'entailment' or 'not_entailment':",
+        "transfer_ori_acc": 54.929577464788736,
+        "transfer_atk_acc": 54.929577464788736
+    },
+    {
+        "dataset": "mmlu",
+        "type": "task",
+        "origin_acc": 62.5,
+        "origin_prompt": "Regarding the following multiple-choice question on {}, pick the correct answer from the options 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 60.0,
+        "atk_prompt": "Regarding the following multiple-choice question on {}, pick the correct answeWr from the options 'A', 'B', 'C', or 'D'. ",
+        "transfer_ori_acc": 44.32624113475177,
+        "transfer_atk_acc": 44.680851063829785
+    },
+    {
+        "dataset": "mmlu",
+        "type": "role",
+        "origin_acc": 62.5,
+        "origin_prompt": "With your knowledge of {}, tackle the following multiple-choice question by choosing 'A', 'B', 'C', or 'D'.",
+        "atk_acc": 59.0,
+        "atk_prompt": "With your knowledge of {}, tackle the follQwing multiple-choice question by choosing 'A', 'B', 'C', or 'D'.",
+        "transfer_ori_acc": 43.97163120567376,
+        "transfer_atk_acc": 43.08510638297872
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "task",
+        "origin_acc": 71.87,
+        "origin_prompt": "Discover the best answer based on the context. If the context doesn't include an answer, respond with 'unanswerable'.",
+        "atk_acc": 56.61,
+        "atk_prompt": "DiscoveMr the best answr bassd on the Rontext. Iof the cnotext doesn't include an answer, resCpond with 'unanswerable'.",
+        "transfer_ori_acc": 87.54992770387507,
+        "transfer_atk_acc": 87.08932164326902
+    },
+    {
+        "dataset": "squad_v2",
+        "type": "role",
+        "origin_acc": 64.5,
+        "origin_prompt": "As an expert with a deep understanding of the context, find the best answer. If the context doesn't include an answer, say 'unanswerable'.",
+        "atk_acc": 36.52,
+        "atk_prompt": "zs an exprt with a deep understanding of the context, find the best answr. If the context doesn't include an anCswer, asy 'unanswerable'.",
+        "transfer_ori_acc": 88.04992770387507,
+        "transfer_atk_acc": 87.08932164326902
+    },
+    {
+        "dataset": "iwslt",
+        "type": "task",
+        "origin_acc": 0.22,
+        "origin_prompt": "Translate the provided sentence from {} to {} while maintaining the original meaning and context:",
+        "atk_acc": 0.21,
+        "atk_prompt": "Translate the provideMd sentence from {} to {} while mainOaining the original meaning and context:",
+        "transfer_ori_acc": 0.12847010019593297,
+        "transfer_atk_acc": 0.13470031324495513
+    },
+    {
+        "dataset": "iwslt",
+        "type": "role",
+        "origin_acc": 0.22,
+        "origin_prompt": "In the capacity of a proficient translator, please change the following sentence from {} to {}:",
+        "atk_acc": 0.21,
+        "atk_prompt": "In the capacity of a proficient tanslator, plnase change the following senlence from {} to {}: ",
+        "transfer_ori_acc": 0.13904950536673938,
+        "transfer_atk_acc": 0.1211819424209054
+    },
+    {
+        "dataset": "un_multi",
+        "type": "task",
+        "origin_acc": 0.34,
+        "origin_prompt": "Translate the following text from the source language {} to the target language {}:",
+        "atk_acc": 0.33,
+        "atk_prompt": "Translate the following text from the source language {} to the Ptarget langugae {}:",
+        "transfer_ori_acc": 0.2655722295717415,
+        "transfer_atk_acc": 0.24955021119352877
+    },
+    {
+        "dataset": "un_multi",
+        "type": "role",
+        "origin_acc": 0.34,
+        "origin_prompt": "In the role of a language expert, perform a machine translation for the provided sentence, changing it from {} to {}:",
+        "atk_acc": 0.32,
+        "atk_prompt": "In the role of a language expert, pErform a macRhine yranslation for the provided sentence, cZanging it from {} to {}: ",
+        "transfer_ori_acc": 0.26923226069781947,
+        "transfer_atk_acc": 0.2529020268653232
+    },
+    {
+        "dataset": "math",
+        "type": "task",
+        "origin_acc": 6.88,
+        "origin_prompt": "Resolve the following mathematical question associated with {}:",
+        "atk_acc": 5.62,
+        "atk_prompt": "Resolve the following mathematical quXestion assocated with {}:",
+        "transfer_ori_acc": 9.375,
+        "transfer_atk_acc": 11.875
+    },
+    {
+        "dataset": "math",
+        "type": "role",
+        "origin_acc": 5.0,
+        "origin_prompt": "Embodying a mathematics advisor, find the answer to the following math problem with {}:",
+        "atk_acc": 2.5,
+        "atk_prompt": "Embodying a mathematics advisor, ind the anwer to the fpollowing math problem with {}: ",
+        "transfer_ori_acc": 11.25,
+        "transfer_atk_acc": 12.5
+    }
+]