Modified validation and training for linktransformer model

.gitattributes

@@ -35,3 +35,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text
+.git/lfs/objects/bf/e9/bfe950b415bc9506b72a2f73eed59afd4840841deae18772931c069d89d51f23 filter=lfs diff=lfs merge=lfs -text
+.git/lfs/objects/b6/0b/b60b6b43406a48bf3638526314f3d232d97058bc93472ff2de930d43686fa441 filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -3,5 +3,7 @@
   "pooling_mode_cls_token": false,
   "pooling_mode_mean_tokens": true,
   "pooling_mode_max_tokens": false,
-  "pooling_mode_mean_sqrt_len_tokens": false
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false
 }

LT_training_config.json

@@ -1,19 +1,26 @@
 {
   "model_save_dir": "models",
   "model_save_name": "linkage_multi_aliases",
-  "opt_model_description": "This model was trained on a dataset consisting of company aliases from wiki data using the LinkTransformer framework. \n                               It was trained for 100 epochs using other defaults that can be found in the repo's LinkTransformer config file - LT_training_config.json \n  ",
+  "opt_model_description": "This model was trained on a dataset consisting of company aliases from wiki data using the LinkTransformer framework. \n                               It was trained for 70 epochs using other defaults that can be found in the repo's LinkTransformer config file - LT_training_config.json \n  ",
   "opt_model_lang": [
+    "de",
     "en",
+    "zh",
+    "ja",
+    "hi",
+    "ar",
+    "bn",
+    "pt",
+    "ru",
     "es",
     "fr",
-    "de",
-    "ja",
-    "zh"
+    "ko"
   ],
   "train_batch_size": 64,
-  "num_epochs": 100,
+  "num_epochs": 70,
   "warm_up_perc": 1,
-  "learning_rate": 2e-06,
+  "learning_rate": 2e-05,
+  "loss_type": "supcon",
   "val_perc": 0.2,
   "wandb_names": {
     "project": "linkage",
@@ -23,10 +30,11 @@
   },
   "add_pooling_layer": false,
   "large_val": true,
-  "eval_steps_perc": 0.1,
+  "eval_steps_perc": 0.5,
   "test_at_end": true,
   "save_val_test_pickles": true,
   "val_query_prop": 0.5,
+  "loss_params": {},
   "eval_type": "retrieval",
   "training_dataset": "dataframe",
   "base_model_path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",

README.md

@@ -1,12 +1,18 @@
 ---
 pipeline_tag: sentence-similarity
 language: 
+- de
 - en
+- zh
+- ja
+- hi
+- ar
+- bn
+- pt
+- ru
 - es
 - fr
-- de
-- ja
-- zh
+- ko
 tags:
 - linktransformer
 - sentence-transformers
@@ -17,23 +23,29 @@ tags:
 
 # dell-research-harvard/lt-wikidata-comp-multi
 
-This is a [LinkTransformer](https://github.com/dell-research-harvard/linktransformer) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class. 
+This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class. 
 It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.
 Notwithstanding that, it can be used for any sentence similarity task within the sentence-transformers framework as well. 
 It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
 Take a look at the documentation of [sentence-transformers](https://www.sbert.net/index.html) if you want to use this model for more than what we support in our applications. 
 
 
-This model has been fine-tuned on the model : sentence-transformers/paraphrase-multilingual-mpnet-base-v2. It is pretrained for the language : - en
+This model has been fine-tuned on the model : sentence-transformers/paraphrase-multilingual-mpnet-base-v2. It is pretrained for the language : - de
+- en
+- zh
+- ja
+- hi
+- ar
+- bn
+- pt
+- ru
 - es
 - fr
-- de
-- ja
-- zh.
+- ko.
 
 
 This model was trained on a dataset consisting of company aliases from wiki data using the LinkTransformer framework. 
-                               It was trained for 100 epochs using other defaults that can be found in the repo's LinkTransformer config file - LT_training_config.json 
+                               It was trained for 70 epochs using other defaults that can be found in the repo's LinkTransformer config file - LT_training_config.json 
   
 
 ## Usage (LinkTransformer)
@@ -107,7 +119,7 @@ The model was trained with the parameters:
 
 **DataLoader**:
 
-`torch.utils.data.dataloader.DataLoader` of length 2268 with parameters:
+`torch.utils.data.dataloader.DataLoader` of length 5966 with parameters:
 ```
 {'batch_size': 64, 'sampler': 'torch.utils.data.dataloader._InfiniteConstantSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
 ```
@@ -119,17 +131,17 @@ The model was trained with the parameters:
 Parameters of the fit()-Method:
 ```
 {
-    "epochs": 100,
-    "evaluation_steps": 22680,
+    "epochs": 70,
+    "evaluation_steps": 2983,
     "evaluator": "sentence_transformers.evaluation.SequentialEvaluator.SequentialEvaluator",
     "max_grad_norm": 1,
     "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
     "optimizer_params": {
-        "lr": 2e-06
+        "lr": 2e-05
     },
     "scheduler": "WarmupLinear",
     "steps_per_epoch": null,
-    "warmup_steps": 226800,
+    "warmup_steps": 417620,
     "weight_decay": 0.01
 }
 ```
@@ -139,10 +151,20 @@ Parameters of the fit()-Method:
 
 LinkTransformer(
   (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: XLMRobertaModel 
-  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False})
 )
 ```
 
 ## Citing & Authors
 
-<!--- Describe where people can find more information -->
+```
+@misc{arora2023linktransformer,
+                  title={LinkTransformer: A Unified Package for Record Linkage with Transformer Language Models},
+                  author={Abhishek Arora and Melissa Dell},
+                  year={2023},
+                  eprint={2309.00789},
+                  archivePrefix={arXiv},
+                  primaryClass={cs.CL}
+                }
+
+```

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "models/linkage_multi_aliases/",
+  "_name_or_path": "models/linkage_multi_aliases",
   "architectures": [
     "XLMRobertaModel"
   ],
@@ -22,7 +22,7 @@
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
-  "transformers_version": "4.31.0",
+  "transformers_version": "4.35.1",
   "type_vocab_size": 1,
   "use_cache": true,
   "vocab_size": 250002

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3d23a8c3194fddc392984009fb03fbfd1ce072746d1ced463d0feb407bd0059
+size 1112197096

special_tokens_map.json

@@ -1,7 +1,25 @@
 {
-  "bos_token": "<s>",
-  "cls_token": "<s>",
-  "eos_token": "</s>",
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
@@ -9,7 +27,25 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<pad>",
-  "sep_token": "</s>",
-  "unk_token": "<unk>"
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b60b6b43406a48bf3638526314f3d232d97058bc93472ff2de930d43686fa441
-size 17082913
+oid sha256:c835b069d7b8cd02b400e6247b83bc1840ab12bb1628d5b2e03c8d728de75558
+size 17082941

tokenizer_config.json

@@ -1,19 +1,61 @@
 {
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "cls_token": "<s>",
   "eos_token": "</s>",
-  "mask_token": {
-    "__type": "AddedToken",
-    "content": "<mask>",
-    "lstrip": true,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "mask_token": "<mask>",
+  "max_length": 128,
   "model_max_length": 512,
+  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "</s>",
+  "stride": 0,
   "tokenizer_class": "XLMRobertaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }