Update from abhijith

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/README.md

@@ -0,0 +1,125 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+---
+
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
+model = AutoModel.from_pretrained('{MODEL_NAME}')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 360 with parameters:
+```
+{'batch_size': 16, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.CosineSimilarityLoss.CosineSimilarityLoss` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 25,
+    "evaluation_steps": 1000,
+    "evaluator": "sentence_transformers.evaluation.EmbeddingSimilarityEvaluator.EmbeddingSimilarityEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'transformers.optimization.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 900,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: DistilBertModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "distilbert-base-uncased",
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertModel"
+  ],
+  "attention_dropout": 0.1,
+  "dim": 768,
+  "dropout": 0.1,
+  "hidden_dim": 3072,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "pad_token_id": 0,
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.12.2",
+  "vocab_size": 30522
+}

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.1.0",
+    "transformers": "4.12.2",
+    "pytorch": "1.9.0+cu111"
+  }
+}

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/eval/similarity_evaluation_sts-dev_results.csv

@@ -0,0 +1,26 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+0,-1,0.8163493363975831,0.8116940708348979,0.7658013920447797,0.7697873650737942,0.7674640795570262,0.7717451226508814,0.745492317503698,0.7539935677114966
+1,-1,0.8614026571069544,0.8591406081526577,0.8298348503580933,0.8345466932194419,0.8296245349477835,0.8342102262014122,0.7920515486603191,0.7969402272045083
+2,-1,0.8618134875126271,0.8605375938661779,0.8213170965525093,0.8263611531760017,0.8214650271402293,0.8261975854476088,0.7887949162035451,0.7964090346675804
+3,-1,0.8634197902971296,0.8607618761739855,0.8341299977853398,0.8363887497986677,0.8340604434040638,0.8361277150674712,0.7980681607799465,0.803406000344134
+4,-1,0.863122950170882,0.8609880069884007,0.8307545750440299,0.834219626563763,0.8307970810282259,0.833664930451917,0.8008657322937442,0.8053629294550337
+5,-1,0.8671087662041486,0.8656601721344199,0.8374766706189534,0.8397804471223802,0.8373593730835684,0.8396130235194438,0.8079563799437387,0.81199989772631
+6,-1,0.8673750526221179,0.8664622652146434,0.8364617350151492,0.8391372740812266,0.8363341742890351,0.8388068432485042,0.8028729049090223,0.8065793573407668
+7,-1,0.868664560814998,0.8676612368010613,0.8353442271710472,0.8378016272531964,0.8352042090201878,0.837460156086736,0.8041474265301363,0.8096866167725159
+8,-1,0.8708500423119241,0.8707622414917114,0.8389260810332955,0.8428263592235371,0.838597610726961,0.8423192779127409,0.8077553170534958,0.8134911663811426
+9,-1,0.868670549812233,0.8681357930599544,0.8362426235145598,0.8395830644240622,0.836180442973115,0.8394080952987203,0.8018577331839877,0.8049367273777921
+10,-1,0.869727559169721,0.8692491888461141,0.8413297598839373,0.8445145037659781,0.8408525937290308,0.844064609188387,0.8060500949036249,0.8102521064861072
+11,-1,0.8712658401444207,0.870597262730171,0.8410508492568923,0.844599016279742,0.8409011722435854,0.844206377219091,0.8074086238628271,0.8134068097280129
+12,-1,0.8714091544169701,0.8710710459752833,0.8388966539823731,0.842368020080169,0.8385817944453019,0.8420208070661935,0.8071589055422911,0.8133493221439073
+13,-1,0.8731685907848233,0.8724367728964338,0.8414684333670919,0.8450051998337139,0.8411359352717475,0.8446814433828159,0.810374742482743,0.8167604526088117
+14,-1,0.872479182206766,0.8722495104260328,0.8412985814385678,0.8449107307687457,0.8409104274525241,0.8444828649541364,0.8078627353062178,0.8137716968238501
+15,-1,0.8722978834316985,0.8722476891360518,0.8409776739252801,0.8449029008308228,0.8406903172170344,0.844311555227525,0.8073518718377992,0.8134541780880624
+16,-1,0.8721259915298777,0.871430363953986,0.842477125776224,0.8457993765376614,0.842009008097188,0.8452269241858195,0.8081429474982276,0.8129876820479176
+17,-1,0.8729087271389145,0.8722637915746657,0.8412936860553671,0.845222340552767,0.8410582300748964,0.8450865538438198,0.8049382742640098,0.8109777849222035
+18,-1,0.8730833199939736,0.8726610320619757,0.8431539029230263,0.8465233582161567,0.842854601372054,0.846033573180642,0.8092990370642487,0.8150161613710208
+19,-1,0.8733457806006462,0.8726056619170518,0.8421153259926785,0.8459631417750871,0.8417275796844319,0.8453932969380689,0.8092018900975103,0.8141700752126543
+20,-1,0.872123074058248,0.8720272832899972,0.8409041186255461,0.8448433232489174,0.8405059194075603,0.8442802410097946,0.8093009049859391,0.8151467689471793
+21,-1,0.8731753819639653,0.8730178302369295,0.8417064621264401,0.8458489157177124,0.841384295232393,0.8451731412306015,0.8093189904821696,0.8158943007662893
+22,-1,0.873336796719464,0.8732291232008476,0.8412216129751707,0.8453967979683806,0.8408729344560504,0.844828721064394,0.8079538647653599,0.8149804129604891
+23,-1,0.8734203003775499,0.8730765216721835,0.8418727587844642,0.84602676993282,0.841492943267561,0.8454618745256882,0.8090278128959401,0.8156011297457241
+24,-1,0.8735196671255344,0.8732526613598429,0.8420646273973388,0.8463672755055822,0.841664775838457,0.8456246804116927,0.8093761550180399,0.8158899324358765

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0c72df68efda954130d412360e500b41c3f8a4093dd51ec9ef220b9d0180042
+size 265488185

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/similarity_evaluation_sts-test_results.csv

@@ -0,0 +1,2 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+-1,-1,0.8331925270347311,0.8347647455338306,0.8251037290355692,0.8241453139228918,0.8245985792677243,0.8235965663644278,0.7461993566563806,0.7413172671100948

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

training_stsbenchmark_distilbert-base-uncased-2021-11-02_09-48-17/tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "distilbert-base-uncased", "tokenizer_class": "DistilBertTokenizer"}