Add new SentenceTransformer model.

.gitattributes

@@ -30,3 +30,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

README.md

@@ -0,0 +1,127 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+
+---
+
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
+model = AutoModel.from_pretrained('{MODEL_NAME}')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 11898 with parameters:
+```
+{'batch_size': 64, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.MSELoss.MSELoss` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 5,
+    "evaluation_steps": 1000,
+    "evaluator": "sentence_transformers.evaluation.SequentialEvaluator.SequentialEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "eps": 1e-06,
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 10000,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: XLMRobertaModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+  "architectures": [
+    "XLMRobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.2",
+    "transformers": "4.21.1",
+    "pytorch": "1.12.1+cu102"
+  }
+}

eval/mse_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv

@@ -0,0 +1,15 @@
+epoch,steps,MSE
+0,1000,0.09204786620102823
+0,2000,0.08162449230439961
+0,3000,0.07583714905194938
+0,4000,0.07227175519801676
+0,5000,0.06990365218371153
+0,6000,0.06809314945712686
+0,7000,0.06643632077611983
+0,8000,0.06556767039000988
+0,9000,0.06475779809989035
+0,10000,0.06360668339766562
+0,11000,0.0628375040832907
+0,-1,0.06280211382545531
+1,1000,0.061956571880728006
+1,2000,0.061449845088645816

eval/similarity_evaluation_STS.en-en.txt_results.csv

@@ -0,0 +1,15 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+0,1000,0.8293205844619254,0.8419787459033459,0.6794723390345381,0.6755838576503266,0.6873264253805316,0.6802846430734095,0.5835118931200658,0.6811414629857533
+0,2000,0.8392547412093986,0.8531016445052206,0.7949954282862768,0.789470177026649,0.7990578438745793,0.7926245351247616,0.6037096488371273,0.6793743920404666
+0,3000,0.8440305704195653,0.8590774732882158,0.8384963401666484,0.8337122980725012,0.8396253453417017,0.833892964451506,0.6258701335552779,0.679714967384718
+0,4000,0.8455026897990299,0.8613800086291485,0.8569539373110111,0.8519661370933485,0.8564639789206823,0.8518119940763678,0.6537428335174124,0.6858026555640757
+0,5000,0.8449729472713324,0.8616717656114137,0.8626858626651913,0.8568245251023716,0.8611406568485954,0.8570251801020322,0.6704330864524474,0.695898830978032
+0,6000,0.8409848384757502,0.8576725038940407,0.8635957827579509,0.856077258207084,0.8618113883489208,0.8543390170031278,0.6756650613031925,0.6972069324413366
+0,7000,0.8401698762558922,0.8601514772519164,0.8640678580302994,0.8574368688082323,0.8622511906951611,0.8562191005344302,0.6815197298817903,0.699455652265119
+0,8000,0.8362533929441562,0.8562898294998277,0.8640951977766022,0.8585889052718008,0.8617517010923053,0.8553223033807749,0.6798684454902298,0.6973779889065645
+0,9000,0.8391277129089909,0.8588452977713673,0.8651653142353873,0.8602133650966393,0.8631801846419191,0.8575671792390465,0.697210858023822,0.7185690019741673
+0,10000,0.8367924301102642,0.8574979878598532,0.866188861356406,0.8610674942331255,0.8641034381716037,0.8581483868242702,0.7067120087052444,0.7259790142892193
+0,11000,0.8349516318951811,0.857133195532884,0.8659704546340554,0.8604878242341061,0.8636599641909686,0.8577282413939463,0.6995912775700288,0.7206827986085228
+0,-1,0.831226376686493,0.8539830657968331,0.8613179607488938,0.8572615839809427,0.8591360541705144,0.8541706513137574,0.7063382811746434,0.7254470094625329
+1,1000,0.8323427897537439,0.8547841482092713,0.8624871875056543,0.8567364982921757,0.860175446962519,0.8537055314869578,0.719176042516346,0.7408747650398845
+1,2000,0.8299299745229648,0.8540257338140025,0.8605322920042283,0.8555479441562552,0.8584797330282151,0.8524277973511882,0.71705446492131,0.738827853405416

eval/similarity_evaluation_STS.zh-tw-zh-tw.txt_results.csv

@@ -0,0 +1,15 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+0,1000,0.7629035906621146,0.7842249006495091,0.6364538956940605,0.6778165271861408,0.6345425314339904,0.6757520820715905,0.4553958475246221,0.5255485892665057
+0,2000,0.7728511131886094,0.7925252371910556,0.7658483887580377,0.7777408297401831,0.7640118586220203,0.7766239245385841,0.5772313479729435,0.6113671931928488
+0,3000,0.7717732641920682,0.791849290937038,0.7869922776113134,0.7931611737552232,0.7854899603821509,0.7921327806295796,0.6171805011997781,0.6381881563508299
+0,4000,0.7724162686622553,0.7923886389766995,0.790915871059518,0.7954684307582307,0.7896909884887545,0.7945062083581826,0.6448460305114796,0.6608874972414434
+0,5000,0.7684965594506217,0.789283168797404,0.7886069782120985,0.7932858207677319,0.787740338731014,0.7924920118454298,0.6564664964056413,0.6713145555362676
+0,6000,0.7714017927595153,0.7911862415524744,0.7916391258065215,0.7960746801386601,0.7906596467745164,0.7952187711994523,0.6690895386313114,0.680620292642491
+0,7000,0.7678474502411792,0.7898492178203845,0.7896621734201098,0.7945579434594102,0.7884672224588058,0.7932743430076358,0.6762158598785418,0.687038119490821
+0,8000,0.7593730377523233,0.7844028886921589,0.7849030508330415,0.7911690273786037,0.7837849870407047,0.7898803390908001,0.6715549278768641,0.6822155385641808
+0,9000,0.7639119935275679,0.788172982702887,0.787013146306845,0.7925746422340025,0.7858035596456088,0.790980941487723,0.6786354862920582,0.6899100048896932
+0,10000,0.764337988821277,0.7871166639806947,0.7870095215051055,0.7921322416626823,0.7858631069321597,0.790981571786479,0.6826810617410292,0.6925308871297546
+0,11000,0.7593929462609824,0.7855116210087307,0.7832003934991564,0.7892605222622866,0.7820261439746802,0.7878934022844406,0.6814442199982291,0.6927030963239129
+0,-1,0.7589438927950686,0.7844964480115055,0.7837516178200493,0.7897560726543602,0.7827171436580161,0.788543609024197,0.6829991777770029,0.6927768300912257
+1,1000,0.7569817070645236,0.7822452756968289,0.7824767122551421,0.7881050700425756,0.7813848935252531,0.7870529702501794,0.6816892630925534,0.691691777734857
+1,2000,0.7545730587369087,0.7809866043899912,0.7824811256864898,0.7884474222727358,0.7814010859740413,0.7871373275387907,0.6800608547341389,0.6905093056947363

eval/translation_evaluation_TED2020-en-zh-tw-dev.tsv.gz_results.csv

@@ -0,0 +1,15 @@
+epoch,steps,src2trg,trg2src
+0,1000,0.907,0.875
+0,2000,0.922,0.903
+0,3000,0.918,0.898
+0,4000,0.919,0.901
+0,5000,0.918,0.899
+0,6000,0.913,0.892
+0,7000,0.907,0.894
+0,8000,0.915,0.899
+0,9000,0.912,0.893
+0,10000,0.912,0.897
+0,11000,0.918,0.894
+0,-1,0.916,0.889
+1,1000,0.915,0.897
+1,2000,0.915,0.897

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14559fa0504e6c8461f4c47d060fca6d92d7c6c5fe9fc8c07a697380e69d95ed
+size 1112244081

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 128,
+  "do_lower_case": false
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b60b6b43406a48bf3638526314f3d232d97058bc93472ff2de930d43686fa441
+size 17082913

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}