init

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -0,0 +1,125 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+---
+
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
+model = AutoModel.from_pretrained('{MODEL_NAME}')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 329 with parameters:
+```
+{'batch_size': 32, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.CosineSimilarityLoss.CosineSimilarityLoss` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 4,
+    "evaluation_steps": 32,
+    "evaluator": "sentence_transformers.evaluation.EmbeddingSimilarityEvaluator.EmbeddingSimilarityEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'transformers.optimization.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 132,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 512, 'do_lower_case': True}) with Transformer model: RobertaModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "output/training_nli_by_MNRloss_klue-roberta-base-2022-04-04_05-37-06/",
+  "architectures": [
+    "RobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tokenizer_class": "BertTokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.17.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 32000
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.0",
+    "transformers": "4.17.0",
+    "pytorch": "1.10.0+cu111"
+  }
+}

eval/similarity_evaluation_sts-dev_results.csv

@@ -0,0 +1,45 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+0,32,0.9364174362360864,0.9094216546694657,0.92582788694544,0.9017441643624815,0.9254401438588308,0.901526619591415,0.9195836305019299,0.8955015298062975
+0,64,0.9387752275339513,0.9007437600357805,0.9225384658462545,0.8939835328003457,0.9224253234587552,0.8938864660272327,0.9312168927716451,0.8944727091686603
+0,96,0.9468690622759517,0.9052071491166803,0.9315989701714651,0.8982278196123462,0.9312254058599362,0.8975479505188085,0.9408992391966308,0.8995410650319291
+0,128,0.9513973438569597,0.9104143353257175,0.9390391178916682,0.903455540814021,0.9386622769013641,0.9027070675096522,0.9456786886784108,0.9037294612393371
+0,160,0.9539637583127887,0.9114323825634303,0.9441187648063655,0.9069916361798683,0.9437786368251059,0.9058798379458675,0.9503494076269636,0.9054814833363348
+0,192,0.9539124421909819,0.911112897783568,0.9421967634011064,0.9056968302261439,0.9419330172603467,0.9051007534608151,0.9493618507616046,0.9041584421873551
+0,224,0.9551687995499981,0.9112834950543767,0.9438365083172408,0.9058067242890087,0.9434752751503982,0.9051846556830871,0.9496683457982172,0.9041114340546021
+0,256,0.9554804203829234,0.9128381398473069,0.9485515866450848,0.9098893455661841,0.9483374653467044,0.9096530126998181,0.9507678704498552,0.9029032435696672
+0,288,0.9562482086511792,0.9100148438780354,0.9448471974002534,0.9054040497059701,0.9446359378799348,0.9052784862655212,0.9492478983062795,0.8995691031757613
+0,320,0.9561487187902168,0.9096581597799543,0.945520667765122,0.9057972506628995,0.9454266663505844,0.9056164597716856,0.950265539798975,0.9004332001920656
+0,-1,0.9571052120202281,0.9111794329549592,0.9461612080839833,0.906620440561665,0.9459869566928014,0.9065482515307142,0.9513097386083957,0.9018320455077192
+1,32,0.9589284376986155,0.9155851465996718,0.9483045904404686,0.9108832918913888,0.9480921463786799,0.910821728479482,0.9531169742770288,0.906775887608312
+1,64,0.9570876996908693,0.9122643223095513,0.9469297025827347,0.9077753172683088,0.946887598589024,0.9078164669106759,0.9526850308960527,0.9035609405886594
+1,96,0.9595173522480765,0.9185526235805104,0.949525873118562,0.9136694625933475,0.9493670154270055,0.9134103248142119,0.9541430512629039,0.910460139118153
+1,128,0.9594034804425732,0.9177448302127982,0.9510410538205278,0.9140668054196116,0.9510066453124948,0.9139974774237459,0.9543353984825856,0.9093598532345999
+1,160,0.9584134650306313,0.9172432054959888,0.9461025929414694,0.9105820950017797,0.9461444449624696,0.9103345148701154,0.9516067076326292,0.9082593020895156
+1,192,0.9593122412588952,0.9177322113427256,0.9490222867938638,0.9126346356767571,0.9488741413840257,0.9123628373436895,0.9523702097361784,0.9074511942035157
+1,224,0.9587664740779075,0.918486641668888,0.9488150974469091,0.9126174277822926,0.9487367568966477,0.9125062642535294,0.9522711523458558,0.9075626798355667
+1,256,0.9604776700687462,0.9199549021488536,0.9507702520105745,0.9145412067206465,0.950627800866083,0.9143870291398981,0.9523122158141124,0.9075605425855164
+1,288,0.9599948071138408,0.9167965164426752,0.949718666634352,0.9114113669685069,0.949585599134688,0.9113439336978632,0.9534018869898268,0.9063316617011595
+1,320,0.9597816801503579,0.9164429645030677,0.9503627500328624,0.9123222978028438,0.9502739985277567,0.9121922779686731,0.953720293466597,0.906147244305871
+1,-1,0.9598848308181257,0.9151735175421271,0.9499376525952087,0.9111668072052281,0.9498483507883388,0.9109439306773873,0.9538155052652163,0.905184329790349
+2,32,0.9611249795383279,0.917696166089834,0.9511940498317587,0.9137962765524426,0.9511085346287841,0.91331617212849,0.955393707646168,0.9088780321895962
+2,64,0.9616877811715796,0.9192479081637024,0.9523860863690279,0.9148297771613778,0.9523215186092309,0.9145050515739642,0.9556432386327292,0.9097805693912004
+2,96,0.9604781269002473,0.9157677867440812,0.9495904511222646,0.9113192681649256,0.9495279215398057,0.9109291366628556,0.9542327564577004,0.9064814169932403
+2,128,0.9611133334720325,0.9187520489352939,0.9522764505040499,0.9147749930763146,0.9521882757981746,0.9145110464845659,0.9557172940043167,0.9097987284377261
+2,160,0.9615751896241393,0.9199000346954663,0.9518281640003243,0.9153267256925522,0.9516685292510961,0.9150998702417457,0.9548557351807491,0.909208279006306
+2,192,0.9613341797906176,0.9191231442963551,0.9518160789845664,0.91428573334009,0.9516872478411317,0.9141634288270228,0.9550004692835815,0.9085085245917373
+2,224,0.9616900120618316,0.919838634229465,0.9525592426357835,0.9150848867546918,0.9524253655928929,0.9148058202556733,0.9552728837262433,0.9089769103200213
+2,256,0.9614442434723803,0.9192568512668169,0.9516094663271528,0.9143671042094657,0.9515011443642336,0.9139930210300242,0.9548639890929828,0.908475069608324
+2,288,0.9618484932169079,0.920681574750307,0.9513790593502466,0.9152344298375478,0.9513181683023832,0.914947792016543,0.9538819302951755,0.9083427382093829
+2,320,0.9615814166173311,0.9194168911286099,0.9511747679699945,0.9142165758694942,0.9510632043548606,0.9138110440408215,0.9536774502549238,0.9071540974992867
+2,-1,0.9618899204830091,0.9197523219160592,0.9514990187584786,0.9144950284775409,0.9513724249074965,0.9141933958011308,0.9541000305950149,0.907493882362763
+3,32,0.9615725200294286,0.9196699542618352,0.9500532438609471,0.9139447661680752,0.9499245880971301,0.9136385217304753,0.954052601801799,0.9082290205910208
+3,64,0.9622641779564153,0.9208518537072761,0.9518685142228873,0.9157354368701474,0.9517278605856678,0.9154265739222907,0.9549183732412461,0.9091011626106166
+3,96,0.9625325524437279,0.9211031170102869,0.9520056696349597,0.9157857115091826,0.9518802639360843,0.9155718273472446,0.9549038242948721,0.9093585420847464
+3,128,0.962412207234652,0.9210918320267806,0.9521220082987237,0.9157850028819498,0.9519936696864797,0.9155149931694914,0.9549815974380843,0.9092520130538757
+3,160,0.9625864612298495,0.9212151369556961,0.9524239137731138,0.9161263224728557,0.9522914047396616,0.9158954350467772,0.9551869771522131,0.9097375363919628
+3,192,0.9627922108913867,0.9214139694219735,0.9523087980474476,0.9163644932226866,0.9521701958703442,0.9158994404958961,0.9550326043881759,0.9095774055845672
+3,224,0.9630076661645971,0.9217946235113581,0.9525463989407941,0.9166150706332703,0.952410780891146,0.9161885755647432,0.9555716681641571,0.9102728493194283
+3,256,0.9630675850486086,0.9222451399096487,0.952555889993494,0.9168340478166045,0.9524244870974917,0.9164701544694059,0.9553434910563082,0.9104731937749312
+3,288,0.9630349696372591,0.9222882373298693,0.952605827051247,0.9168340478166045,0.9524710416836056,0.9165576301434459,0.9551684193943721,0.9101838578652106
+3,320,0.9630215630333796,0.9222556859503128,0.9526062420847365,0.9168406528287275,0.9524752171203408,0.9165423624476087,0.9551697992335885,0.9101982615663468
+3,-1,0.9630212619359664,0.9222594034012261,0.9526033726229892,0.9168338697124335,0.952472766628758,0.9165486567247956,0.9551685746586317,0.910192641811339

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2147a9742efe60296c22416c409fb7cd04b8ddf8ebf9852b6fe79d8b7becf5fd
+size 442554737

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": true
+}

similarity_evaluation_sts-test_results.csv

@@ -0,0 +1,2 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+-1,-1,0.8900689868339492,0.8892573547643868,0.8875096311088903,0.882654606412235,0.8867271453049435,0.8817527389246668,0.8786014218947737,0.8734982195607274

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "do_basic_tokenize": true, "never_split": null, "bos_token": "[CLS]", "eos_token": "[SEP]", "model_max_length": 512, "special_tokens_map_file": "/root/.cache/huggingface/transformers/9d0c87e44b00acfbfbae931b2e4068eb6311a0c3e71e23e5400bdf57cab4bfbf.70c17d6e4d492c8f24f5bb97ab56c7f272e947112c6faf9dd846da42ba13eb23", "name_or_path": "output/training_nli_by_MNRloss_klue-roberta-base-2022-04-04_05-37-06/", "tokenizer_class": "BertTokenizer"}