Update scripts/eval_mteb.py

scripts/eval_mteb.py

@@ -119,7 +119,19 @@ CMTEB_TASK_LIST = ['TNews', 'IFlyTek', 'MultilingualSentiment', 'JDReview', 'Onl
                    'T2Retrieval', 'MMarcoRetrieval', 'DuRetrieval', 'CovidRetrieval', 'CmedqaRetrieval', 'EcomRetrieval', 'MedicalRetrieval', 'VideoRetrieval',
                    'ATEC', 'BQ', 'LCQMC', 'PAWSX', 'STSB', 'AFQMC', 'QBQTC', 'STS22']
 
+MTEB_PL = [
+    "CBD","PolEmo2.0-IN","PolEmo2.0-OUT","AllegroReviews","PAC","MassiveIntentClassification","MassiveScenarioClassification",
+    "SICK-E-PL","PPC","CDSC-E","PSC","8TagsClustering","SICK-R-PL","CDSC-R","STS22",
+    "ArguAna-PL","DBPedia-PL","FiQA-PL","HotpotQA-PL","MSMARCO-PL","NFCorpus-PL","NQ-PL","Quora-PL","SCIDOCS-PL","SciFact-PL","TRECCOVID-PL"
+]
 
+MTEB_FR = [
+    "AmazonReviewsClassification","MasakhaNEWSClassification","MassiveIntentClassification",
+    "MassiveScenarioClassification","MTOPDomainClassification","MTOPIntentClassification","OpusparcusPC","PawsX",
+    "AlloProfClusteringP2P","AlloProfClusteringS2S","HALClusteringS2S","MasakhaNEWSClusteringP2P","MasakhaNEWSClusteringS2S","MLSUMClusteringP2P","MLSUMClusteringS2S",
+    "SyntecReranking","AlloprofReranking","AlloprofRetrieval","BSARDRetrieval","SyntecRetrieval","XPQARetrieval","MintakaRetrieval",
+    "SummEvalFr","STSBenchmarkMultilingualSTS","STS22","SICKFr"
+]
 
 logging.basicConfig(
     level=logging.INFO,
@@ -136,7 +148,6 @@ def get_detailed_instruct(task_description: str) -> str:
 
 def get_task_def_by_task_name_and_type(task_name: str, task_type: str, default_instruct='Given a web search query, retrieve relevant passages that answer the query') -> str:
     if task_type in ['STS']:
-        # return "Given a premise, retrieve a hypothesis that is entailed by the premise."
         return "Retrieve semantically similar text"
 
     if task_type in ['Summarization']:
@@ -166,6 +177,13 @@ def get_task_def_by_task_name_and_type(task_name: str, task_type: str, default_i
             'JDReview': 'Classify the customer review for iPhone on e-commerce platform into positive or negative',
             'OnlineShopping': 'Classify the customer review for online shopping into positive or negative',
             'Waimai': 'Classify the customer review from a food takeaway platform into positive or negative',
+            # MTEB-pl eval instructions
+            "CBD":"Classify the sentiment of polish tweet reviews",
+            "PolEmo2.0-IN": "Classify the sentiment of in-domain (medicine and hotels) online reviews",
+            "PolEmo2.0-OUT":"Classify the sentiment of out-of-domain (products and school) online reviews",
+            "AllegroReviews": "Classify the sentiment of reviews from e-commerce marketplace Allegro",
+            "PAC": "Classify the sentence into one of the two types: \"BEZPIECZNE_POSTANOWIENIE_UMOWNE\" and \"KLAUZULA_ABUZYWNA\"",
+
         }
         return task_name_to_instruct[task_name]
 
@@ -187,6 +205,16 @@ def get_task_def_by_task_name_and_type(task_name: str, task_type: str, default_i
             'CLSClusteringP2P': 'Identify the main category of scholar papers based on the titles and abstracts',
             'ThuNewsClusteringS2S': 'Identify the topic or theme of the given news articles based on the titles',
             'ThuNewsClusteringP2P': 'Identify the topic or theme of the given news articles based on the titles and contents',
+            # MTEB-fr eval instructions
+            "AlloProfClusteringP2P": "Identify the main category of Allo Prof document based on the titles and descriptions",
+            "AlloProfClusteringS2S": "Identify the main category of Allo Prof document based on the titles",
+            "HALClusteringS2S": "Identify the main category of academic passage based on the titles and contents",
+            "MasakhaNEWSClusteringP2P": "Identify the topic or theme of the given news articles based on the titles and contents",
+            "MasakhaNEWSClusteringS2S": "Identify the topic or theme of the given news articles based on the titles",
+            "MLSUMClusteringP2P": "Identify the topic or theme of the given articles based on the titles and contents",
+            "MLSUMClusteringS2S":  "Identify the topic or theme of the given articles based on the titles",
+            # MTEB-pl eval instructions
+            "8TagsClustering": "Identify of headlines from social media posts in Polish  into 8 categories: film, history, food, medicine, motorization, work, sport and technology",
         }
         return task_name_to_instruct[task_name]
 
@@ -206,6 +234,16 @@ def get_task_def_by_task_name_and_type(task_name: str, task_type: str, default_i
             'CMedQAv2': 'Given a Chinese community medical question, retrieve replies that best answer the question',
             'Ocnli': 'Retrieve semantically similar text.',
             'Cmnli': 'Retrieve semantically similar text.',
+            # MTEB-fr eval instructions
+            "AlloprofReranking": "Given a question, retrieve passages that answer the question",
+            "OpusparcusPC":"Retrieve semantically similar text",
+            "PawsX":"Retrieve semantically similar text",
+            "SyntecReranking": "Given a question, retrieve passages that answer the question",
+            # MTEB-pl eval instructions
+            "SICK-E-PL": "Retrieve semantically similar text",
+            "PPC": "Retrieve semantically similar text",
+            "CDSC-E": "Retrieve semantically similar text",
+            "PSC": "Retrieve semantically similar text",
         }
         return task_name_to_instruct[task_name]
 
@@ -237,6 +275,24 @@ def get_task_def_by_task_name_and_type(task_name: str, task_type: str, default_i
             'EcomRetrieval': 'Given a user query from an e-commerce website, retrieve description sentences of relevant products',
             'MedicalRetrieval': 'Given a medical question, retrieve user replies that best answer the question',
             'VideoRetrieval': 'Given a video search query, retrieve the titles of relevant videos',
+            # MTEB-fr eval instructions
+            "AlloprofRetrieval": "Given a question, retrieve passages that answer the question",
+            "BSARDRetrieval": "Given a question, retrieve passages that answer the question",
+            "SyntecRetrieval": "Given a question, retrieve passages that answer the question",
+            "XPQARetrieval": "Given a question, retrieve passages that answer the question",
+            "MintakaRetrieval": "Given a question, retrieve passages that answer the question",
+            # MTEB-pl eval instructions 
+            "ArguAna-PL": "Given a claim, find documents that refute the claim",
+            "DBPedia-PL": "Given a query, retrieve relevant entity descriptions from DBPedia",
+            "FiQA-PL": "Given a financial question, retrieve user replies that best answer the question",
+            "HotpotQA-PL": "Given a multi-hop question, retrieve documents that can help answer the question",
+            "MSMARCO-PL": "Given a web search query, retrieve relevant passages that answer the query",
+            "NFCorpus-PL": "Given a question, retrieve relevant documents that best answer the question",
+            "NQ-PL": "Given a question, retrieve Wikipedia passages that answer the question",
+            "Quora-PL": "Given a question, retrieve questions that are semantically equivalent to the given question",
+            "SCIDOCS-PL": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper",
+            "SciFact-PL": "Given a scientific claim, retrieve documents that support or refute the claim",
+            "TRECCOVID-PL": "Given a query on COVID-19, retrieve documents that answer the query"
         }
 
         # add lower case keys to match some beir names
@@ -629,9 +685,14 @@ def main(args):
     elif args.task == 'cmteb':
         task_names = CMTEB_TASK_LIST
         lang = ['zh','zh-CN']
+    elif args.task == 'mteb-fr':
+        tas_names = MTEB_FR
+        lang = ['fr']
+    elif args.task == 'mteb-pl':
+        lang = ['pl']
     else:
         task_names = [args.task]
-        lang = ['en','zh','zh-CN']
+        lang = ['en','zh','zh-CN','pl','fr']
     for task in task_names:
         evaluation = MTEB(tasks=[task], task_langs=lang)
         task_cls = evaluation.tasks[0]