File size: 5,583 Bytes
683d0f6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
[
    {
        "dataset": "cola",
        "type": "translation",
        "origin_acc": 78.0,
        "origin_prompt": "Assess the following sentence and determine if it is grammatically correct. Respond with 'Acceptable' or 'Unacceptable':",
        "atk_acc": 74.6,
        "atk_prompt": "As a tool for classifying valid sentences, determine whether  'acceptable' or 'unacceptable' is based on its grammatical rules.",
        "transfer_ori_acc": 75.5,
        "transfer_atk_acc": 75.0
    },
    {
        "dataset": "mrpc",
        "type": "translation",
        "origin_acc": 82.6,
        "origin_prompt": "As a semantic comparison expert, evaluate the given pair of sentences and determine if they are 'equivalent' or 'not_equivalent'.",
        "atk_acc": 77.7,
        "atk_prompt": "Determine whether a given pair of sentences is 'equivalent' or 'not_equivalent', and whether they are semantically identical.",
        "transfer_ori_acc": 73.0,
        "transfer_atk_acc": 75.0
    },
    {
        "dataset": "qnli",
        "type": "translation",
        "origin_acc": 89.8,
        "origin_prompt": "Evaluate whether the given context supports the answer to the question by responding with 'entailment' or 'not_entailment'.",
        "atk_acc": 53.1,
        "atk_prompt": "Answer with 'entailment' or 'not_entailment', inferring from the given context.",
        "transfer_ori_acc": 78.5,
        "transfer_atk_acc": 74.5
    },
    {
        "dataset": "qqp",
        "type": "translation",
        "origin_acc": 88.1,
        "origin_prompt": "Are the following two questions equivalent or not? Answer me with \"equivalent\" or \"not_equivalent\".",
        "atk_acc": 70.9,
        "atk_prompt": "As a tool for determining the equivalence of problems, the problems are examined and their similarity is classified as 'equivalent' or 'not_equivalent'.",
        "transfer_ori_acc": 81.0,
        "transfer_atk_acc": 2.0
    },
    {
        "dataset": "rte",
        "type": "translation",
        "origin_acc": 86.64,
        "origin_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word.",
        "atk_acc": 80.51,
        "atk_prompt": "Determinate the semantic connections of a given sentence and classify it as 'entailment' or 'not_entailment'.",
        "transfer_ori_acc": 81.0,
        "transfer_atk_acc": 53.0
    },
    {
        "dataset": "sst2",
        "type": "translation",
        "origin_acc": 95.41,
        "origin_prompt": "In the role of a sentiment analysis tool, respond with 'positive' or 'negative' to classify this statement:",
        "atk_acc": 92.78,
        "atk_prompt": "Classify what you're trying to convey in this sentence as 'positive' if it's positive, and 'negative' if it's negative.",
        "transfer_ori_acc": 92.5,
        "transfer_atk_acc": 90.0
    },
    {
        "dataset": "wnli",
        "type": "translation",
        "origin_acc": 66.2,
        "origin_prompt": "Are the following two sentences entailment or not_entailment? Answer me with \"entailment\" or \"not_entailment\", just one word.",
        "atk_acc": 49.3,
        "atk_prompt": "Examine the paraphrases of a given sentence and classify them 'entailment' or 'not_entailment'.",
        "transfer_ori_acc": 66.19718309859155,
        "transfer_atk_acc": 50.70422535211267
    },
    {
        "dataset": "mmlu",
        "type": "translation",
        "origin_acc": 45.74,
        "origin_prompt": "For each multiple-choice question about {}, identify the correct answer by selecting 'A', 'B', 'C', or 'D'.",
        "atk_acc": 41.84,
        "atk_prompt": "Choose the correct answer for '{}' from 'A', 'B', 'C', or 'D' in the multiple choice problem involving,",
        "transfer_ori_acc": 59.5,
        "transfer_atk_acc": 56.00000000000001
    },
    {
        "dataset": "squad_v2",
        "type": "translation",
        "origin_acc": 88.18,
        "origin_prompt": "As a proficient expert in the given context, search for the most relevant answer. If the answer cannot be found, respond by saying 'unanswerable'.",
        "atk_acc": 84.51,
        "atk_prompt": "Please give the cause of the incident mentioned in the context. If the cause is not clear, respond with 'unanswerable'.",
        "transfer_ori_acc": 60.4580727188865,
        "transfer_atk_acc": 66.50307424062444
    },
    {
        "dataset": "iwslt",
        "type": "translation",
        "origin_acc": 0.14,
        "origin_prompt": "Please perform a translation of the given sentence, converting it from {} to {}:",
        "atk_acc": 0.12,
        "atk_prompt": "Please translate the sentences below into {}, then {}.",
        "transfer_ori_acc": 0.21840008360286153,
        "transfer_atk_acc": 0.11831480990391464
    },
    {
        "dataset": "un_multi",
        "type": "translation",
        "origin_acc": 0.27,
        "origin_prompt": "Convert the following sentence from its original {} language to the target language {}:",
        "atk_acc": 0.26,
        "atk_prompt": "Please translate the next sentence from {} to {}.",
        "transfer_ori_acc": 0.3249349632796426,
        "transfer_atk_acc": 0.3358246755929457
    },
    {
        "dataset": "math",
        "type": "translation",
        "origin_acc": 16.25,
        "origin_prompt": "Evaluate the following mathematical problem that includes {}:",
        "atk_acc": 10.62,
        "atk_prompt": "Compute the result of {}.",
        "transfer_ori_acc": 4.375,
        "transfer_atk_acc": 6.875000000000001
    }
]