Updated model with better training and evaluation. Test and val data included as pickle files. Older Legacy files were removed to avoid confusion.

.gitattributes

@@ -1,41 +1,6 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
 model.safetensors filter=lfs diff=lfs merge=lfs -text
+.git/lfs/objects/97/f4/97f4992b52a0621c15f4b731a5d0aa12f93a8136dfefe9da6d15efe200de2025 filter=lfs diff=lfs merge=lfs -text
 .git/lfs/objects/10/5b/105b8df0ad58b8f0280d55ee92bd528df20c93351ed578775f427b8022581ec0 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/d8/9e/d89eb57d2145088f1078af291109f8d3033eccffb4526b2562deb93d4239263a filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/8f/13/8f1325becf59c50fad5dca8ed3851bb8f7710dea27f965f2162c8236eb0d4184 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/99/44/994418f47770d3b34ecf225929a10feb591a4d18025804cf2051af4827e5e4ec filter=lfs diff=lfs merge=lfs -text
+test_data.pickle filter=lfs diff=lfs merge=lfs -text
+val_data.pickle filter=lfs diff=lfs merge=lfs -text
+sentencepiece.bpe.model filter=lfs diff=lfs merge=lfs -text

Information-Retrieval_evaluation_eval_results.csv

@@ -0,0 +1,6 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.7432432432432432,0.9054054054054054,0.9459459459459459,0.9594594594594594,0.7432432432432432,0.7274774774774774,0.3018018018018018,0.8896396396396395,0.19189189189189185,0.9346846846846846,0.09729729729729729,0.9481981981981981,0.8277027027027029,0.8510838939890354,0.8163342935605461
+0,0,0.7432432432432432,0.9054054054054054,0.9459459459459459,0.9594594594594594,0.7432432432432432,0.7274774774774774,0.3018018018018018,0.8896396396396395,0.19189189189189185,0.9346846846846846,0.09729729729729729,0.9481981981981981,0.8277027027027029,0.8510838939890354,0.8163342935605461
+0,0,0.3772455089820359,0.564870259481038,0.6946107784431138,0.8483033932135728,0.3772455089820359,0.3772455089820359,0.1882900864936793,0.564870259481038,0.13892215568862273,0.6946107784431138,0.08483033932135728,0.8483033932135728,0.5086089725311275,0.5891511561097007,0.5148648590536671
+0,0,0.3772455089820359,0.564870259481038,0.6946107784431138,0.8483033932135728,0.3772455089820359,0.3772455089820359,0.1882900864936793,0.564870259481038,0.13892215568862273,0.6946107784431138,0.08483033932135728,0.8483033932135728,0.5086089725311275,0.5891511561097007,0.5148648590536671
+0,0,0.3772455089820359,0.564870259481038,0.6946107784431138,0.8483033932135728,0.3772455089820359,0.3772455089820359,0.1882900864936793,0.564870259481038,0.13892215568862273,0.6946107784431138,0.08483033932135728,0.8483033932135728,0.5086089725311275,0.5891511561097007,0.5148648590536671

Information-Retrieval_evaluation_test_results.csv

@@ -0,0 +1,5 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.4947565543071161,0.6910112359550562,0.804119850187266,0.9108614232209737,0.4947565543071161,0.4534398965578741,0.24269662921348314,0.6424451578384162,0.17617977528089887,0.7637154449794898,0.10408239700374532,0.8804012841091494,0.6226166696391399,0.6721939572015778,0.6041564193432677
+0,0,0.49887640449438203,0.7138576779026217,0.8307116104868913,0.9258426966292135,0.49887640449438203,0.4541889602282861,0.2545568039950063,0.6692460317460317,0.18337078651685393,0.7918958444801142,0.10606741573033708,0.8973154093097913,0.6346632185958015,0.68553627425209,0.6149989655561907
+0,0,0.6821192052980133,0.890728476821192,0.9370860927152318,0.9668874172185431,0.6821192052980133,0.6821192052980133,0.29690949227373065,0.890728476821192,0.18741721854304633,0.9370860927152318,0.0966887417218543,0.9668874172185431,0.7943918847892356,0.8374009368138124,0.796834833672582
+0,0,0.7185430463576159,0.9238410596026491,0.956953642384106,0.9801324503311258,0.7185430463576159,0.7185430463576159,0.30794701986754963,0.9238410596026491,0.19139072847682115,0.956953642384106,0.09801324503311257,0.9801324503311258,0.8196100073583515,0.8596148051808452,0.8212238276807813

README.md

@@ -10,7 +10,7 @@ tags:
 
 ---
 
-# dell-research-harvard/lt-un-data-fine-industry-fr
+# {MODEL_NAME}
 
 This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class. 
 It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "models/linkage_un_data_fr_fine_industry",
+  "_name_or_path": "dangvantuan/sentence-camembert-large",
   "architectures": [
     "CamembertModel"
   ],

special_tokens_map.json

@@ -3,27 +3,9 @@
     "<s>NOTUSED",
     "</s>NOTUSED"
   ],
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "cls_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
@@ -31,25 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "sep_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
 }

pytorch_model.bin → test_data.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:105b8df0ad58b8f0280d55ee92bd528df20c93351ed578775f427b8022581ec0
-size 1346772393
+oid sha256:cadd8cf5f67df026618234c3f3a709d22fc6d454829cdc08d307c49ad9b1c859
+size 44811

tokenizer_config.json

@@ -66,17 +66,10 @@
   "cls_token": "<s>",
   "eos_token": "</s>",
   "mask_token": "<mask>",
-  "max_length": 514,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_to_multiple_of": null,
   "pad_token": "<pad>",
-  "pad_token_type_id": 0,
-  "padding_side": "right",
   "sep_token": "</s>",
   "sp_model_kwargs": {},
-  "stride": 0,
   "tokenizer_class": "CamembertTokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }

val_data.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:626904ea85eb65a3f91a22272d4a0b234d2aa2ca4f8502a1c7a70355b17b5aaa
+size 169613