upload

CERerankingEvaluator_results.csv

@@ -0,0 +1,127 @@
+epoch,steps,MRR@10
+0,5000,0.5650095238095237
+0,10000,0.5849968253968254
+0,15000,0.6097650793650794
+0,20000,0.6246285714285715
+0,25000,0.6100253968253967
+0,30000,0.6270730158730159
+0,35000,0.6138888888888888
+0,40000,0.6240317460317462
+0,45000,0.6327619047619049
+0,50000,0.619631746031746
+0,55000,0.5871142857142856
+0,60000,0.6175809523809525
+0,65000,0.6081968253968254
+0,70000,0.6151301587301587
+0,75000,0.6093269841269842
+0,80000,0.6032571428571428
+0,85000,0.6138063492063491
+0,90000,0.6156952380952381
+0,95000,0.6303523809523809
+0,100000,0.6061523809523809
+0,105000,0.6133174603174603
+0,110000,0.6226063492063493
+0,115000,0.6176349206349206
+0,120000,0.6104761904761905
+0,125000,0.6332253968253967
+0,130000,0.6289523809523808
+0,135000,0.6181809523809524
+0,140000,0.6399841269841271
+0,145000,0.623073015873016
+0,150000,0.5963587301587302
+0,155000,0.6157301587301588
+0,160000,0.613120634920635
+0,165000,0.6089936507936508
+0,170000,0.6203301587301587
+0,175000,0.6171269841269841
+0,180000,0.5939269841269841
+0,185000,0.6417873015873015
+0,190000,0.6164476190476191
+0,195000,0.6215841269841269
+0,200000,0.6298984126984126
+0,205000,0.6030507936507936
+0,210000,0.6084730158730158
+0,215000,0.6092730158730159
+0,220000,0.5939650793650793
+0,225000,0.6124190476190475
+0,230000,0.6039269841269841
+0,235000,0.6253301587301587
+0,240000,0.634904761904762
+0,245000,0.6317015873015873
+0,250000,0.6196603174603175
+0,255000,0.6287396825396825
+0,260000,0.6095746031746031
+0,265000,0.6263492063492063
+0,270000,0.6171079365079365
+0,275000,0.6289523809523809
+0,280000,0.6202634920634921
+0,285000,0.6255301587301587
+0,290000,0.5993841269841268
+0,295000,0.6191841269841271
+0,300000,0.6203396825396825
+0,305000,0.6128412698412699
+0,310000,0.6090825396825398
+0,315000,0.5950539682539682
+0,320000,0.5990444444444444
+0,325000,0.6042412698412698
+0,330000,0.5960190476190476
+0,335000,0.6106222222222223
+0,340000,0.6055968253968255
+0,345000,0.5984095238095238
+0,350000,0.6142984126984128
+0,355000,0.6137746031746032
+0,360000,0.6018412698412698
+0,365000,0.6123079365079365
+0,370000,0.6130285714285715
+0,375000,0.6008412698412698
+0,380000,0.6020698412698412
+0,385000,0.6100222222222222
+0,390000,0.5971650793650793
+0,395000,0.5941968253968255
+0,400000,0.5871428571428571
+0,405000,0.6100190476190476
+0,410000,0.5903174603174602
+0,415000,0.5988317460317459
+0,420000,0.6132380952380952
+0,425000,0.6144412698412698
+0,430000,0.5980888888888888
+0,435000,0.5973746031746032
+0,440000,0.595384126984127
+0,445000,0.5871714285714286
+0,450000,0.6012412698412699
+0,455000,0.5873047619047618
+0,460000,0.595584126984127
+0,465000,0.5804285714285713
+0,470000,0.5887619047619047
+0,475000,0.5872761904761904
+0,480000,0.5871396825396825
+0,485000,0.5907174603174602
+0,490000,0.5880412698412699
+0,495000,0.5807968253968254
+0,500000,0.5909746031746032
+0,505000,0.5912984126984128
+0,510000,0.5942761904761905
+0,515000,0.5840222222222223
+0,520000,0.5852380952380952
+0,525000,0.582784126984127
+0,530000,0.5916190476190476
+0,535000,0.5777269841269841
+0,540000,0.582120634920635
+0,545000,0.5746634920634921
+0,550000,0.5746444444444445
+0,555000,0.5632444444444444
+0,560000,0.5799650793650795
+0,565000,0.5932507936507936
+0,570000,0.5816190476190476
+0,575000,0.5838857142857143
+0,580000,0.5859650793650794
+0,585000,0.5843968253968255
+0,590000,0.5840634920634921
+0,595000,0.5958285714285714
+0,600000,0.5842857142857142
+0,605000,0.5892507936507937
+0,610000,0.5914507936507937
+0,615000,0.5953968253968254
+0,620000,0.5925174603174603
+0,625000,0.5890857142857143
+0,-1,0.5890857142857143

README.md

@@ -0,0 +1,34 @@
+# Cross-Encoder for MS Marco
+
+This model uses [TinyBERT](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/TinyBERT), a tiny BERT model with only 6 layers. The base model is: General_TinyBERT_v2(6layer-768dim)
+
+It was trained on [MS Marco Passage Ranking](https://github.com/microsoft/MSMARCO-Passage-Ranking) task.
+
+The model can be used for Information Retrieval: Given a query, encode the query will all possible passages (e.g. retrieved with ElasticSearch). Then sort the passages in a decreasing order. See [SBERT.net Information Retrieval](https://github.com/UKPLab/sentence-transformers/tree/master/examples/applications/information-retrieval) for more details. The training code is available here: [SBERT.net Training MS Marco](https://github.com/UKPLab/sentence-transformers/tree/master/examples/training/ms_marco)
+
+## Usage and Performance
+
+Pre-trained models can be used like this:
+```
+from sentence_transformers import CrossEncoder
+model = CrossEncoder('model_name', max_length=512)
+scores = model.predict([('Query', 'Paragraph1'), ('Query', 'Paragraph2') , ('Query', 'Paragraph3')])
+```
+
+In the following table, we provide various pre-trained Cross-Encoders together with their performance on the [TREC Deep Learning 2019](https://microsoft.github.io/TREC-2019-Deep-Learning/) and the [MS Marco Passage Reranking](https://github.com/microsoft/MSMARCO-Passage-Ranking/) dataset. 
+
+
+| Model-Name        | NDCG@10 (TREC DL 19) | MRR@10 (MS Marco Dev)  | Docs / Sec (BertTokenizerFast) | Docs / Sec |
+| ------------- |:-------------| -----| --- | --- |
+| cross-encoder/ms-marco-TinyBERT-L-2  | 67.43 | 30.15  | 9000 | 780
+| cross-encoder/ms-marco-TinyBERT-L-4  | 68.09 | 34.50  | 2900 | 760
+| cross-encoder/ms-marco-TinyBERT-L-6 |  69.57 | 36.13  | 680 | 660
+| cross-encoder/ms-marco-electra-base | 71.99 | 36.41 | 340 | 340
+| *Other models* | | | |
+| nboost/pt-tinybert-msmarco | 63.63 | 28.80 | 2900 | 760
+| nboost/pt-bert-base-uncased-msmarco | 70.94 | 34.75 | 340 | 340|
+| nboost/pt-bert-large-msmarco | 73.36 | 36.48 | 100 | 100 |
+| Capreolus/electra-base-msmarco | 71.23 | | 340 | 340 |
+| amberoad/bert-multilingual-passage-reranking-msmarco | 68.40 | | 330 | 330 
+ 
+ Note: Runtime was computed on a V100 GPU. A bottleneck for smaller models is the standard Python tokenizer from Huggingface v3. Replacing it with the fast tokenizer based on Rust, the throughput is significantly improved:

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "nreimers/TinyBERT_L-6_H-768_v2",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cf70691b32e33dc1a57845ceeeb81bc70cfec62a0d584d063f13576403f2759
+size 267871721

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": "/home/ukp-reimers/.cache/torch/transformers/68bd39c5d90b3f2d8da930bf3efe6ad55d21ae39cfbf97d37817cce8149bf2f3.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d", "tokenizer_file": null, "name_or_path": "nreimers/TinyBERT_L-6_H-768_v2"}

train_script.py

@@ -0,0 +1,193 @@
+from torch.utils.data import DataLoader
+from sentence_transformers import LoggingHandler
+from sentence_transformers.cross_encoder import CrossEncoder
+from sentence_transformers.cross_encoder.evaluation import CEBinaryClassificationEvaluator
+from sentence_transformers import InputExample
+import logging
+from datetime import datetime
+import gzip
+import sys
+import numpy as np
+import os
+from shutil import copyfile
+import csv
+import tqdm
+
+#### Just some code to print debug information to stdout
+logging.basicConfig(format='%(asctime)s - %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S',
+                    level=logging.INFO,
+                    handlers=[LoggingHandler()])
+#### /print debug information to stdout
+
+
+#Define our Cross-Encoder
+model_name = sys.argv[1] #'google/electra-small-discriminator'
+train_batch_size = 32
+num_epochs = 1
+model_save_path = 'output/training_ms-marco_cross-encoder-'+model_name.replace("/", "-")+'-'+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+#We set num_labels=1, which predicts a continous score between 0 and 1
+model = CrossEncoder(model_name, num_labels=1, max_length=512)
+
+
+# Write self to path
+os.makedirs(model_save_path, exist_ok=True)
+
+train_script_path = os.path.join(model_save_path, 'train_script.py')
+copyfile(__file__, train_script_path)
+with open(train_script_path, 'a') as fOut:
+    fOut.write("\n\n# Script was called via:\n#python " + " ".join(sys.argv))
+
+
+corpus = {}
+queries = {}
+
+#### Read train file
+with gzip.open('../data/collection.tsv.gz', 'rt') as fIn:
+    for line in fIn:
+        pid, passage = line.strip().split("\t")
+        corpus[pid] = passage
+
+with open('../data/queries.train.tsv', 'r') as fIn:
+    for line in fIn:
+        qid, query = line.strip().split("\t")
+        queries[qid] = query
+
+
+
+pos_neg_ration = (4+1)
+cnt = 0
+train_samples = []
+dev_samples = {}
+
+num_dev_queries = 125
+num_max_dev_negatives = 200
+
+with gzip.open('../data/qidpidtriples.rnd-shuf.train-eval.tsv.gz', 'rt') as fIn:
+    for line in fIn:
+        qid, pos_id, neg_id = line.strip().split()
+
+        if qid not in dev_samples and len(dev_samples) < num_dev_queries:
+            dev_samples[qid] = {'query': queries[qid], 'positive': set(), 'negative': set()}
+
+        if qid in dev_samples:
+            dev_samples[qid]['positive'].add(corpus[pos_id])
+
+            if len(dev_samples[qid]['negative']) < num_max_dev_negatives:
+                dev_samples[qid]['negative'].add(corpus[neg_id])
+
+with gzip.open('../data/qidpidtriples.rnd-shuf.train.tsv.gz', 'rt') as fIn:
+    for line in tqdm.tqdm(fIn, unit_scale=True):
+        cnt += 1
+        qid, pos_id, neg_id = line.strip().split()
+        query = queries[qid]
+        if (cnt % pos_neg_ration) == 0:
+            passage = corpus[pos_id]
+            label = 1
+        else:
+            passage = corpus[neg_id]
+            label = 0
+
+        train_samples.append(InputExample(texts=[query, passage], label=label))
+
+        if len(train_samples) >= 2e7:
+            break
+
+
+
+train_dataloader = DataLoader(train_samples, shuffle=True, batch_size=train_batch_size)
+
+# We add an evaluator, which evaluates the performance during training
+
+class CERerankingEvaluator:
+    def __init__(self, samples, mrr_at_k: int = 10, name: str = ''):
+        self.samples = samples
+        self.name = name
+        self.mrr_at_k = mrr_at_k
+
+        if isinstance(self.samples, dict):
+            self.samples = list(self.samples.values())
+
+        self.csv_file = "CERerankingEvaluator" + ("_" + name if name else '') + "_results.csv"
+        self.csv_headers = ["epoch", "steps", "MRR@{}".format(mrr_at_k)]
+
+    def __call__(self, model, output_path: str = None, epoch: int = -1, steps: int = -1) -> float:
+        if epoch != -1:
+            if steps == -1:
+                out_txt = " after epoch {}:".format(epoch)
+            else:
+                out_txt = " in epoch {} after {} steps:".format(epoch, steps)
+        else:
+            out_txt = ":"
+
+        logging.info("CERerankingEvaluator: Evaluating the model on " + self.name + " dataset" + out_txt)
+
+        all_mrr_scores = []
+        num_queries = 0
+        num_positives = []
+        num_negatives = []
+        for instance in self.samples:
+            query = instance['query']
+            positive = list(instance['positive'])
+            negative = list(instance['negative'])
+            docs = positive + negative
+            is_relevant = [True]*len(positive) + [False]*len(negative)
+
+            if len(positive) == 0 or len(negative) == 0:
+                continue
+
+            num_queries += 1
+            num_positives.append(len(positive))
+            num_negatives.append(len(negative))
+
+            model_input = [[query, doc] for doc in docs]
+            pred_scores = model.predict(model_input, convert_to_numpy=True, show_progress_bar=False)
+            pred_scores_argsort = np.argsort(-pred_scores)  #Sort in decreasing order
+
+            mrr_score = 0
+            for rank, index in enumerate(pred_scores_argsort[0:self.mrr_at_k]):
+                if is_relevant[index]:
+                    mrr_score = 1 / (rank+1)
+
+            all_mrr_scores.append(mrr_score)
+
+        mean_mrr = np.mean(all_mrr_scores)
+        logging.info("Queries: {} \t Positives: Min {:.1f}, Mean {:.1f}, Max {:.1f} \t Negatives: Min {:.1f}, Mean {:.1f}, Max {:.1f}".format(num_queries, np.min(num_positives), np.mean(num_positives), np.max(num_positives), np.min(num_negatives), np.mean(num_negatives), np.max(num_negatives)))
+        logging.info("MRR@{}: {:.2f}".format(self.mrr_at_k, mean_mrr*100))
+
+        if output_path is not None:
+            csv_path = os.path.join(output_path, self.csv_file)
+            output_file_exists = os.path.isfile(csv_path)
+            with open(csv_path, mode="a" if output_file_exists else 'w', encoding="utf-8") as f:
+                writer = csv.writer(f)
+                if not output_file_exists:
+                    writer.writerow(self.csv_headers)
+
+                writer.writerow([epoch, steps, mean_mrr])
+
+        return mean_mrr
+
+
+evaluator = CERerankingEvaluator(dev_samples)
+
+# Configure the training
+warmup_steps = 5000
+logging.info("Warmup-steps: {}".format(warmup_steps))
+
+
+# Train the model
+model.fit(train_dataloader=train_dataloader,
+          evaluator=evaluator,
+          epochs=num_epochs,
+          evaluation_steps=5000,
+          warmup_steps=warmup_steps,
+          output_path=model_save_path,
+          use_amp=True)
+
+#Save latest model
+model.save(model_save_path+'-latest')
+
+
+# Script was called via:
+#python train_cross-encoder.py nreimers/TinyBERT_L-6_H-768_v2