|
|
supported_templates = [ |
|
|
"DIRECT", |
|
|
"MEANING", |
|
|
"INDIRECT", |
|
|
"FS-DIRECT", |
|
|
"FS-DIRECT_MAJ", |
|
|
"FS-DIRECT_FR", |
|
|
"FS-DIRECT_MAJ_FR", |
|
|
"FS-DIRECT_FR_MIN", |
|
|
"NETWORK" |
|
|
] |
|
|
|
|
|
def print_templates(): |
|
|
print("Supported templates:") |
|
|
for tmplt in supported_templates: |
|
|
print(tmplt) |
|
|
|
|
|
def check_template_name(name): |
|
|
assert name in supported_templates, f""+name+" not support, supported templates: "+str(supported_templates) |
|
|
|
|
|
|
|
|
def make_prompt(name, reference, hypothese, model, tokenizer, device): |
|
|
use_fs = False |
|
|
if "FS-" in name: |
|
|
name = name.split("-")[1] |
|
|
use_fs = True |
|
|
|
|
|
match name: |
|
|
case "DIRECT": |
|
|
template = [ |
|
|
{"role": "user", "content": "You will receive two sentences A and B." |
|
|
+" Do these two sentences mean the same thing?" |
|
|
+" Answer with only one word \"yes\" or \"no\"."}, |
|
|
{"role": "assistant", "content": "Please provide the sentences for me to evaluate."}, |
|
|
] |
|
|
|
|
|
if use_fs: |
|
|
template.append({"role": "user", "content": "A: \""+"Amrozi accused his brother , whom he called \" the witness \" , of deliberately distorting his evidence ."+"\"; B: \""+"Amrozi accused his brother , whom he disparagingly referred to as 'the liar witness', of intentionally twisting his testimony ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Pennmakkal is an Indian Malayalam film from 1966 , produced by J. Sasikumar and directed by KP Kottarakkara ."+"\"; B: \""+"The Indian Malayalam film 'Pennmakkal', released in 1966, was produced by J. Sasikumar and directed by KP Kottarakkara ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Yes"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Sorkin , who faces charges of conspiracy to obstruct justice and lying to a grand jury , was to have been tried separately ."+"\"; B: \""+"Despite being accused of conspiring to obstruct justice and perjury , Sorkin was supposed to stand trial on his own ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Gilroy police and FBI agents described Gehring as cooperative , but said Saturday that he had revealed nothing about what had happened to the children ."+"\"; B: \""+"Although Gilroy police and FBI agents reported that Gehring was cooperative , he hadn't disclosed any information about the children's whereabouts or what had happened to them as of Saturday ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Whereas '' e `` the electric charge of the particle and A is the magnetic vector potential of the electromagnetic field ."+"\"; B: \""+"The electric charge of the particle is denoted by ''e'', and the magnetic vector potential of the electromagnetic field is denoted by 'A' ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Yes"}) |
|
|
template.append({"role": "user", "content": "A: \""+"The Jidanul River is a tributary of the Jiul de Vest River in Romania ."+"\"; B: \""+"The Jidanul River is a mere insignificant stream that flows into the grand Jiul de Vest River in Romania ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
|
|
|
template.append({"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "Yes"}) |
|
|
no.append({"role": "assistant", "content": "No"}) |
|
|
return yes,no |
|
|
|
|
|
case "NETWORK": |
|
|
template = [ |
|
|
{"role": "user", "content": "You will receive two sentences A and B." |
|
|
+" Do these two sentences express the same network policy?" |
|
|
+" Answer with only one word \"Yes\" or \"No\"."}, |
|
|
{"role": "assistant", "content": "Please provide the sentences for me to evaluate."}, |
|
|
] |
|
|
|
|
|
|
|
|
template.append({"role": "user", "content": "A: \""+"Everquest is blocked by the University firewall"+"\"; B: \""+"Everquest is not allowed by the University firewall"+"\""}) |
|
|
template.append({"role": "assistant", "content": "Yes"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Quotas for students are 5000 Megabyte per hr download and 2000 Megabyte per hour upload"+"\"; B: \""+"Students have a download quota of 5000 MB per hour and an upload quota of 2000 MB per hour"+"\""}) |
|
|
template.append({"role": "assistant", "content": "Yes"}) |
|
|
template.append({"role": "user", "content": "A: \""+"from endpoint('guests') to endpoint('servers') for group('host') block traffic('any')"+"\"; B: \""+"RHIT-OPEN will work in labs, classrooms and residence halls but does not allow the user to access the internal Rose-Hulman network"+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Housing does not normally limit the amount of bandwidth"+"\"; B: \""+"There is no bandwidth limit for the dorms"+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"for endpoint('university') add middlebox('firewall') allow traffic('H323 video conferencing')"+"\"; B: \""+"H323 video conferencing is allowed by the University firewall"+"\""}) |
|
|
template.append({"role": "assistant", "content": "Yes"}) |
|
|
template.append({"role": "user", "content": "A: \""+"H323 video conferencing is allowed by the University firewall"+"\"; B: \""+"The network firewall does not block H.323 video conferencing"+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
|
|
|
|
|
|
|
|
|
template.append({"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "Yes"}) |
|
|
no.append({"role": "assistant", "content": "No"}) |
|
|
return yes,no |
|
|
|
|
|
case "DIRECT_FR": |
|
|
template = [ |
|
|
{"role": "user", "content": "Tu vas recevoir deux phrases, A et B." |
|
|
+" Est-ce que ces deux phrases veulent dire la même chose ?" |
|
|
+" Réponds avec un seul mot : \"oui\" ou \"non\"."}, |
|
|
{"role": "assistant", "content": "S'il te plaît, fournis-moi les deux phrases que je dois évaluer."}, |
|
|
] |
|
|
|
|
|
if use_fs: |
|
|
template.append({"role": "user", "content": "A: \""+"Amrozi a accusé son frère, qu'il appelait \"le témoin\", d'avoir délibérément déformé ses preuves ."+"\"; B: \""+"Amrozi a accusé son frère, qu'il désignait de manière péjorative comme \"le témoin menteur\", d'avoir intentionnellement falsifié son témoignage."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Pennmakkal est un film indien en malayalam de 1966, produit par J. Sasikumar et réalisé par KP Kottarakkara ."+"\"; B: \""+"Le film indien en malayalam 'Pennmakkal', sorti en 1966, a été produit par J. Sasikumar et réalisé par KP Kottarakkara ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Oui"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Sorkin, qui fait face à des accusations de complot pour entraver la justice et de faux témoignage devant un grand jury, devait être jugé séparément ."+"\"; B: \""+"Malgré les accusations de complot pour entraver la justice et de parjure, Sorkin devait être jugé seul ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"La police de Gilroy et les agents du FBI ont décrit Gehring comme coopératif, mais ont déclaré samedi qu'il n'avait révélé aucune information sur ce qui était arrivé aux enfants ."+"\"; B: \""+"Bien que la police de Gilroy et les agents du FBI aient rapporté que Gehring était coopératif, il n'avait pas divulgué d'informations sur le lieu où se trouvaient les enfants ou sur ce qui leur était arrivé samedi ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Dans lequel ''e'' représente la charge électrique de la particule et A est le vecteur du potentiel magnétique du champ électromagnétique ."+"\"; B: \""+"La charge électrique de la particule est désignée par ''e'', et le vecteur du potentiel magnétique du champ électromagnétique est désigné par 'A' ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Oui"}) |
|
|
template.append({"role": "user", "content": "A: \""+"La rivière Jidanul est un affluent de la rivière Jiul de Vest en Roumanie ."+"\"; B: \""+"La rivière Jidanul est un simple ruisseau insignifiant qui se jette dans la grande rivière Jiul de Vest en Roumanie ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
|
|
|
template.append({"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "Oui"}) |
|
|
no.append({"role": "assistant", "content": "Non"}) |
|
|
return yes,no |
|
|
|
|
|
case "DIRECT_FR_MIN": |
|
|
template = [ |
|
|
{"role": "user", "content": "Tu vas recevoir deux phrases, A et B." |
|
|
+" Est-ce que ces deux phrases veulent dire la même chose ?" |
|
|
+" Réponds avec un seul mot : \"oui\" ou \"non\"."}, |
|
|
{"role": "assistant", "content": "S'il te plaît, fournis-moi les deux phrases que je dois évaluer."}, |
|
|
] |
|
|
|
|
|
if use_fs: |
|
|
template.append({"role": "user", "content": "A: \""+"Amrozi a accusé son frère, qu'il appelait \"le témoin\", d'avoir délibérément déformé ses preuves ."+"\"; B: \""+"Amrozi a accusé son frère, qu'il désignait de manière péjorative comme \"le témoin menteur\", d'avoir intentionnellement falsifié son témoignage."+"\""}) |
|
|
template.append({"role": "assistant", "content": "non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Pennmakkal est un film indien en malayalam de 1966, produit par J. Sasikumar et réalisé par KP Kottarakkara ."+"\"; B: \""+"Le film indien en malayalam 'Pennmakkal', sorti en 1966, a été produit par J. Sasikumar et réalisé par KP Kottarakkara ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "oui"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Sorkin, qui fait face à des accusations de complot pour entraver la justice et de faux témoignage devant un grand jury, devait être jugé séparément ."+"\"; B: \""+"Malgré les accusations de complot pour entraver la justice et de parjure, Sorkin devait être jugé seul ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"La police de Gilroy et les agents du FBI ont décrit Gehring comme coopératif, mais ont déclaré samedi qu'il n'avait révélé aucune information sur ce qui était arrivé aux enfants ."+"\"; B: \""+"Bien que la police de Gilroy et les agents du FBI aient rapporté que Gehring était coopératif, il n'avait pas divulgué d'informations sur le lieu où se trouvaient les enfants ou sur ce qui leur était arrivé samedi ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Dans lequel ''e'' représente la charge électrique de la particule et A est le vecteur du potentiel magnétique du champ électromagnétique ."+"\"; B: \""+"La charge électrique de la particule est désignée par ''e'', et le vecteur du potentiel magnétique du champ électromagnétique est désigné par 'A' ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "oui"}) |
|
|
template.append({"role": "user", "content": "A: \""+"La rivière Jidanul est un affluent de la rivière Jiul de Vest en Roumanie ."+"\"; B: \""+"La rivière Jidanul est un simple ruisseau insignifiant qui se jette dans la grande rivière Jiul de Vest en Roumanie ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "non"}) |
|
|
|
|
|
template.append({"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "oui"}) |
|
|
no.append({"role": "assistant", "content": "non"}) |
|
|
return yes,no |
|
|
|
|
|
case "DIRECT_MAJ_FR": |
|
|
template = [ |
|
|
{"role": "user", "content": "Tu vas recevoir deux phrases, A et B." |
|
|
+" Est-ce que ces deux phrases veulent dire la même chose ?" |
|
|
+" Réponds avec un seul mot : \"Oui\" ou \"Non\"."}, |
|
|
{"role": "assistant", "content": "S'il te plaît, fournis-moi les deux phrases que je dois évaluer."}, |
|
|
] |
|
|
|
|
|
if use_fs: |
|
|
template.append({"role": "user", "content": "A: \""+"Amrozi a accusé son frère, qu'il appelait \"le témoin\", d'avoir délibérément déformé ses preuves ."+"\"; B: \""+"Amrozi a accusé son frère, qu'il désignait de manière péjorative comme \"le témoin menteur\", d'avoir intentionnellement falsifié son témoignage."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Pennmakkal est un film indien en malayalam de 1966, produit par J. Sasikumar et réalisé par KP Kottarakkara ."+"\"; B: \""+"Le film indien en malayalam 'Pennmakkal', sorti en 1966, a été produit par J. Sasikumar et réalisé par KP Kottarakkara ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Oui"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Sorkin, qui fait face à des accusations de complot pour entraver la justice et de faux témoignage devant un grand jury, devait être jugé séparément ."+"\"; B: \""+"Malgré les accusations de complot pour entraver la justice et de parjure, Sorkin devait être jugé seul ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"La police de Gilroy et les agents du FBI ont décrit Gehring comme coopératif, mais ont déclaré samedi qu'il n'avait révélé aucune information sur ce qui était arrivé aux enfants ."+"\"; B: \""+"Bien que la police de Gilroy et les agents du FBI aient rapporté que Gehring était coopératif, il n'avait pas divulgué d'informations sur le lieu où se trouvaient les enfants ou sur ce qui leur était arrivé samedi ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Dans lequel ''e'' représente la charge électrique de la particule et A est le vecteur du potentiel magnétique du champ électromagnétique ."+"\"; B: \""+"La charge électrique de la particule est désignée par ''e'', et le vecteur du potentiel magnétique du champ électromagnétique est désigné par 'A' ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Oui"}) |
|
|
template.append({"role": "user", "content": "A: \""+"La rivière Jidanul est un affluent de la rivière Jiul de Vest en Roumanie ."+"\"; B: \""+"La rivière Jidanul est un simple ruisseau insignifiant qui se jette dans la grande rivière Jiul de Vest en Roumanie ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Non"}) |
|
|
|
|
|
template.append({"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "Oui"}) |
|
|
no.append({"role": "assistant", "content": "Non"}) |
|
|
return yes,no |
|
|
|
|
|
case "DIRECT_MAJ": |
|
|
template = [ |
|
|
{"role": "user", "content": "You will receive two sentences A and B." |
|
|
+" Do these two sentences mean the same thing?" |
|
|
+" Answer with only one word \"Yes\" or \"No\"."}, |
|
|
{"role": "assistant", "content": "Please provide the sentences for me to evaluate."}, |
|
|
] |
|
|
|
|
|
if use_fs: |
|
|
template.append({"role": "user", "content": "A: \""+"Amrozi accused his brother , whom he called \" the witness \" , of deliberately distorting his evidence ."+"\"; B: \""+"Amrozi accused his brother , whom he disparagingly referred to as 'the liar witness', of intentionally twisting his testimony ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Pennmakkal is an Indian Malayalam film from 1966 , produced by J. Sasikumar and directed by KP Kottarakkara ."+"\"; B: \""+"The Indian Malayalam film 'Pennmakkal', released in 1966, was produced by J. Sasikumar and directed by KP Kottarakkara ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Yes"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Sorkin , who faces charges of conspiracy to obstruct justice and lying to a grand jury , was to have been tried separately ."+"\"; B: \""+"Despite being accused of conspiring to obstruct justice and perjury , Sorkin was supposed to stand trial on his own ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Gilroy police and FBI agents described Gehring as cooperative , but said Saturday that he had revealed nothing about what had happened to the children ."+"\"; B: \""+"Although Gilroy police and FBI agents reported that Gehring was cooperative , he hadn't disclosed any information about the children's whereabouts or what had happened to them as of Saturday ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
template.append({"role": "user", "content": "A: \""+"Whereas '' e `` the electric charge of the particle and A is the magnetic vector potential of the electromagnetic field ."+"\"; B: \""+"The electric charge of the particle is denoted by ''e'', and the magnetic vector potential of the electromagnetic field is denoted by 'A' ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "Yes"}) |
|
|
template.append({"role": "user", "content": "A: \""+"The Jidanul River is a tributary of the Jiul de Vest River in Romania ."+"\"; B: \""+"The Jidanul River is a mere insignificant stream that flows into the grand Jiul de Vest River in Romania ."+"\""}) |
|
|
template.append({"role": "assistant", "content": "No"}) |
|
|
|
|
|
template.append({"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "Yes"}) |
|
|
no.append({"role": "assistant", "content": "No"}) |
|
|
return yes,no |
|
|
|
|
|
case "INDIRECT": |
|
|
template = [ |
|
|
{"role": "user", "content": "You will receive two sentences A and B." |
|
|
+" Do these two sentences mean the same thing?"}, |
|
|
{"role": "assistant", "content": "Please provide the sentences for me to evaluate."}, |
|
|
{"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""} |
|
|
] |
|
|
encodeds = tokenizer.apply_chat_template( |
|
|
template, |
|
|
return_tensors="pt", |
|
|
padding=False |
|
|
).to(device) |
|
|
explication = model.generate( |
|
|
encodeds, |
|
|
max_new_tokens=1000, |
|
|
do_sample=False, |
|
|
pad_token_id=tokenizer.eos_token_id |
|
|
).to(device) |
|
|
explication = tokenizer.batch_decode(explication)[0] |
|
|
explication = explication.replace("</s>",'').split("[/INST]")[-1] |
|
|
|
|
|
template.append({"role": "assistant", "content": explication}) |
|
|
template.append({"role": "user", "content": "Summarize your answer with only one word \"yes\" or \"no\"."}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "Yes"}) |
|
|
no.append({"role": "assistant", "content": "No"}) |
|
|
|
|
|
return yes,no |
|
|
|
|
|
case "MEANING": |
|
|
template = [ |
|
|
{"role": "user", "content": "You will receive two sentences A and B." |
|
|
+" Explain the meaning of each sentence."}, |
|
|
{"role": "assistant", "content": "Please provide the sentences for me to evaluate."}, |
|
|
{"role": "user", "content": "A: \""+reference+"\"; B: \""+hypothese+"\""} |
|
|
] |
|
|
encodeds = tokenizer.apply_chat_template( |
|
|
template, |
|
|
return_tensors="pt", |
|
|
padding=False |
|
|
).to(device) |
|
|
explication = model.generate( |
|
|
encodeds, |
|
|
max_new_tokens=1000, |
|
|
do_sample=False, |
|
|
pad_token_id=tokenizer.eos_token_id |
|
|
).to(device) |
|
|
explication = tokenizer.batch_decode(explication)[0] |
|
|
explication = explication.replace("</s>",'').split("[/INST]")[-1] |
|
|
|
|
|
template.append({"role": "assistant", "content": explication}) |
|
|
template.append({"role": "user", "content": "Do these two sentences mean the same thing?" |
|
|
+" Summarize your answer with only one word \"yes\" or \"no\"."}) |
|
|
|
|
|
yes = [v for v in template] |
|
|
no = [v for v in template] |
|
|
yes.append({"role": "assistant", "content": "Yes"}) |
|
|
no.append({"role": "assistant", "content": "No"}) |
|
|
|
|
|
return yes,no |
|
|
|
|
|
|
