initial commit

.gitattributes

@@ -0,0 +1,4 @@
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+vocab.txt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+.venv
+venv

README.md

@@ -0,0 +1,83 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+
+---
+
+# metarank/ce-esci-MiniLM-L6-v2
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 384 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+A [cross-encoder/ms-marco-MiniLM-L-6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) model fine-tuned on 
+[Amazon ESCI dataset](https://github.com/amazon-science/esci-data).
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('metarank/esci-MiniLM-L6-v2')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 769 with parameters:
+```
+{'batch_size': 128, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.MultipleNegativesRankingLoss.MultipleNegativesRankingLoss` with parameters:
+  ```
+  {'scale': 20.0, 'similarity_fct': 'cos_sim'}
+  ```
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 1,
+    "evaluation_steps": 0,
+    "evaluator": "NoneType",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 1000,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: BertModel 
+  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+  (2): Normalize()
+)
+```
+
+## Citing & Authors
+
+* Roman Grebennikov

config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "cross-encoder/ms-marco-MiniLM-L-12-v2",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "sbert_ce_default_activation_function": "torch.nn.modules.linear.Identity",
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.4",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

finetune.py

@@ -0,0 +1,99 @@
+from sentence_transformers import SentenceTransformer, LoggingHandler, util, models, evaluation, losses, InputExample, CrossEncoder
+from torch import nn
+import csv
+from torch.utils.data import DataLoader, Dataset
+import torch
+from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator, SentenceEvaluator, SimilarityFunction, RerankingEvaluator
+from sentence_transformers.cross_encoder.evaluation import CERerankingEvaluator
+import logging
+import json
+import random
+import gzip
+
+model_name = 'cross-encoder/ms-marco-MiniLM-L-12-v2'
+
+train_batch_size = 32
+max_seq_length = 128
+num_epochs = 1
+warmup_steps = 1000
+model_save_path = '.'
+lr = 2e-5
+
+class ESCIDataset(Dataset):
+    def __init__(self, input):
+        self.queries = []
+        self.posneg = []
+        with gzip.open(input) as jsonfile:
+            for line in jsonfile.readlines():
+                query = json.loads(line)
+                for doc in query['e']:
+                    self.queries.append(InputExample(texts=[query['query'], doc['title']], label=1.0))
+                for doc in query['s']:
+                    self.queries.append(InputExample(texts=[query['query'], doc['title']], label=0.1))
+                for doc in query['c']:
+                    self.queries.append(InputExample(texts=[query['query'], doc['title']], label=0.01))
+                for doc in query['i']:
+                    self.queries.append(InputExample(texts=[query['query'], doc['title']], label=0.0))
+
+    def __getitem__(self, item):
+        return self.queries[item]
+
+    def __len__(self):
+        return len(self.queries)
+
+class ESCIEvalDataset(Dataset):
+    def __init__(self, input):
+        self.queries = []
+        with gzip.open(input) as jsonfile:
+            for line in jsonfile.readlines():
+                query = json.loads(line)
+                if len(query['e']) > 0 and len(query['i']) > 0:
+                    for p in query['e']:
+                        positive = p['title']
+                        for n in query['i']:
+                            negative = n['title']
+                            self.queries.append(InputExample(texts=[query['query'], positive, negative]))
+
+    def __getitem__(self, item):
+        return self.queries[item]
+
+    def __len__(self):
+        return len(self.queries)
+
+model = CrossEncoder(model_name, num_labels=1)
+model.max_seq_length = max_seq_length
+
+
+train_dataset = ESCIDataset(input='train-small.json.gz')
+eval_dataset = ESCIEvalDataset(input='test-small.json.gz')
+train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=train_batch_size)
+
+samples = {}
+for query in eval_dataset.queries:
+    qstr = query.texts[0]
+    sample = samples.get(qstr, {'query': qstr})
+    positive = sample.get('positive', [])
+    positive.append(query.texts[1])
+    sample['positive'] = positive
+    negative = sample.get('negative', [])
+    negative.append(query.texts[2])
+    sample['negative'] = negative
+    samples[qstr] = sample
+
+evaluator = CERerankingEvaluator(samples=samples,name='esci')
+
+# Train the model
+
+model.fit(train_dataloader=train_dataloader,
+          epochs=num_epochs,
+          warmup_steps=warmup_steps,
+          use_amp=True,
+          optimizer_params = {'lr': lr},
+          evaluator=evaluator,
+#          evaluation_steps=1000,
+          output_path=model_save_path
+          )
+
+# Save the model
+
+model.save(model_save_path)

onnx_convert.py

@@ -0,0 +1,18 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+max_seq_length=128
+
+model = AutoModel.from_pretrained(".")
+model.eval()
+
+inputs = {"input_ids": torch.ones(1, max_seq_length, dtype=torch.int64),
+        "attention_mask": torch.ones(1, max_seq_length, dtype=torch.int64),
+        "token_type_ids": torch.ones(1, max_seq_length, dtype=torch.int64)}
+
+symbolic_names = {0: 'batch_size', 1: 'max_seq_len'}
+
+torch.onnx.export(model, args=tuple(inputs.values()), f='pytorch_model.onnx', export_params=True, 
+    input_names=['input_ids', 'attention_mask', 'token_type_ids'], output_names=['last_hidden_state'],
+    dynamic_axes={'input_ids': symbolic_names, 'attention_mask': symbolic_names, 'token_type_ids': symbolic_names})
+

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dcfb2efa8e9be4d55c8353e38f61ccfd7223e0bfc2f24ab8af495b2cbbc8bc3
+size 133514357

pytorch_model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb0312525f025d18e7013477ed8c389ad104591fdbfda838599762dad8608acb
+size 133694712

requirements.txt

@@ -0,0 +1,4 @@
+sentence-transformers==2.2.2
+torch==2.0.0
+onnx==1.13.1
+huggingface_hub==0.13.3

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

test-small.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb557251b12addb55d94af30120d121dfa6391e58bcc4a9aee0f1d35cc2ea1c8
+size 8522018

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": "/home/shutty/.cache/huggingface/hub/models--cross-encoder--ms-marco-MiniLM-L-12-v2/snapshots/97f7dcbdd6ab58fe7f44368c795fc5200b48fcbe/special_tokens_map.json",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train-small.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c7c14a8910a3a6c09421a08a84cfc0e74fd198d0aaf43ab2c39250a8ae4e4dd
+size 19430577

vocab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3
+size 231508