Updated model with better training and evaluation. Test and val data included as pickle files. Older Legacy files were removed to avoid confusion.

.gitattributes

@@ -1,35 +1,4 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text
+sentencepiece.bpe.model filter=lfs diff=lfs merge=lfs -text
+test_data.pickle filter=lfs diff=lfs merge=lfs -text
+val_data.pickle filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

LT_training_config.json

@@ -0,0 +1,29 @@
+{
+  "model_save_dir": "models",
+  "model_save_name": "lt-historicjapanesecompanies-comp-prod-ind_onlinecontrastive_full",
+  "opt_model_description": "This model was trained on a dataset of historic Japanese companies, products, industry, addresses, and shareholders. Take a look at our paper for more details. The task is to link indices of japanese companies",
+  "opt_model_lang": "ja",
+  "train_batch_size": 64,
+  "num_epochs": 50,
+  "warm_up_perc": 1,
+  "learning_rate": 2e-05,
+  "loss_type": "onlinecontrastive",
+  "val_perc": 0.2,
+  "wandb_names": {
+    "id": "econabhishek",
+    "run": "lt-historicjapanesecompanies-comp-prod-ind_onlinecontrastive_full",
+    "project": "linkage",
+    "entity": "econabhishek"
+  },
+  "add_pooling_layer": false,
+  "large_val": true,
+  "eval_steps_perc": 0.5,
+  "test_at_end": true,
+  "save_val_test_pickles": true,
+  "val_query_prop": 0.5,
+  "loss_params": {},
+  "eval_type": "classification",
+  "training_dataset": "dataframe",
+  "base_model_path": "oshizo/sbert-jsnli-luke-japanese-base-lite",
+  "best_model_path": "models/lt-historicjapanesecompanies-comp-prod-ind_onlinecontrastive_full"
+}

README.md

@@ -0,0 +1,146 @@
+---
+pipeline_tag: sentence-similarity
+language: 
+- ja
+tags:
+- linktransformer
+- sentence-transformers
+- sentence-similarity
+- tabular-classification
+
+---
+
+# {MODEL_NAME}
+
+This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class. 
+It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.
+Notwithstanding that, it can be used for any sentence similarity task within the sentence-transformers framework as well. 
+It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+Take a look at the documentation of [sentence-transformers](https://www.sbert.net/index.html) if you want to use this model for more than what we support in our applications. 
+
+
+This model has been fine-tuned on the model : oshizo/sbert-jsnli-luke-japanese-base-lite. It is pretrained for the language : - ja.
+
+
+This model was trained on a dataset of historic Japanese companies, products, industry, addresses, and shareholders. Take a look at our paper for more details. The task is to link indices of japanese companies
+
+## Usage (LinkTransformer)
+
+Using this model becomes easy when you have [LinkTransformer](https://github.com/dell-research-harvard/linktransformer) installed:
+
+```
+pip install -U linktransformer
+```
+
+Then you can use the model like this:
+
+```python
+import linktransformer as lt
+import pandas as pd
+
+##Load the two dataframes that you want to link. For example, 2 dataframes with company names that are written differently
+df1=pd.read_csv("data/df1.csv") ###This is the left dataframe with key CompanyName for instance
+df2=pd.read_csv("data/df2.csv") ###This is the right dataframe with key CompanyName for instance
+
+###Merge the two dataframes on the key column!
+df_merged = lt.merge(df1, df2, on="CompanyName", how="inner")
+
+##Done! The merged dataframe has a column called "score" that contains the similarity score between the two company names
+
+```
+
+
+## Training your own LinkTransformer model
+Any Sentence Transformers can be used as a backbone by simply adding a pooling layer.  Any other transformer on HuggingFace can also be used by specifying the option add_pooling_layer==True
+The model was trained using SupCon loss. 
+Usage can be found in the package docs. 
+The training config can be found in the repo with the name LT_training_config.json
+To replicate the training, you can download the file and specify the path in the config_path argument of the training function. You can also override the config by specifying the training_args argument.
+Here is an example. 
+
+
+```python
+
+##Consider the example in the paper that has a dataset of Mexican products and their tariff codes from 1947 and 1948 and we want train a model to link the two tariff codes.
+saved_model_path = train_model(
+        model_path="hiiamsid/sentence_similarity_spanish_es",
+        dataset_path=dataset_path,
+        left_col_names=["description47"],
+        right_col_names=['description48'],
+        left_id_name=['tariffcode47'],
+        right_id_name=['tariffcode48'],
+        log_wandb=False,
+        config_path=LINKAGE_CONFIG_PATH,
+        training_args={"num_epochs": 1}
+    )
+
+```
+
+
+You can also use this package for deduplication (clusters a df on the supplied key column). Merging a fine class (like product) to a coarse class (like HS code) is also possible.
+Read our paper and the documentation for more!
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+You can evaluate the model using the [LinkTransformer](https://github.com/dell-research-harvard/linktransformer) package's inference functions.
+We have provided a few datasets in the package for you to try out. We plan to host more datasets on Huggingface and our website (Coming soon) that you can take a look at.
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 45 with parameters:
+```
+{'batch_size': 64, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`linktransformer.modified_sbert.losses.OnlineContrastiveLoss_wandb` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 50,
+    "evaluation_steps": 23,
+    "evaluator": "sentence_transformers.evaluation.SequentialEvaluator.SequentialEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 2250,
+    "weight_decay": 0.01
+}
+```
+
+
+
+
+LinkTransformer(
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: LukeModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+)
+```
+
+## Citing & Authors
+
+```
+@misc{arora2023linktransformer,
+                  title={LinkTransformer: A Unified Package for Record Linkage with Transformer Language Models},
+                  author={Abhishek Arora and Melissa Dell},
+                  year={2023},
+                  eprint={2309.00789},
+                  archivePrefix={arXiv},
+                  primaryClass={cs.CL}
+                }
+
+```

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<ent2>": 32771,
+  "<ent>": 32770
+}

binary_classification_evaluation_eval_results.csv

@@ -0,0 +1,2 @@
+epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
+0,0,0.9382022471910112,0.8532869815826416,0.7636363636363636,0.7,0.84,0.8262333869934082,0.8184099257358792,0.9382022471910112,84.1578369140625,0.7572815533980584,0.7358490566037735,0.78,89.29296112060547,0.8270661393011416,0.9438202247191011,4.08632755279541,0.7872340425531915,0.8409090909090909,0.74,4.08632755279541,0.8277502672701681,0.8848314606741573,44.88762283325195,0.5925925925925927,0.5517241379310345,0.64,43.54036331176758,0.5417359362559628

binary_classification_evaluation_test_results.csv

@@ -0,0 +1,2 @@
+epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
+0,0,0.9831460674157303,0.7418056726455688,0.9433962264150944,0.9803921568627451,0.9090909090909091,0.7362334728240967,0.9536854893174564,0.9803370786516854,117.0570297241211,0.9320388349514563,1.0,0.8727272727272727,117.0570297241211,0.9474351573268464,0.9775280898876404,5.338754653930664,0.923076923076923,0.9795918367346939,0.8727272727272727,5.437343597412109,0.9464839308275074,0.9606741573033708,42.41117858886719,0.8771929824561402,0.847457627118644,0.9090909090909091,41.67987823486328,0.9307436646625807

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "/home/abhishekarora/.cache/torch/sentence_transformers/oshizo_sbert-jsnli-luke-japanese-base-lite/",
+  "architectures": [
+    "LukeModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bert_model_name": "models/luke-japanese/hf_xlm_roberta",
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "cls_entity_prediction": false,
+  "entity_emb_size": 256,
+  "entity_vocab_size": 4,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "luke",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.35.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "use_entity_aware_attention": true,
+  "vocab_size": 32772
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.2.2",
+    "transformers": "4.25.1",
+    "pytorch": "1.13.0+cu116"
+  }
+}

entity_vocab.json

@@ -0,0 +1,6 @@
+{
+  "[MASK2]": 3,
+  "[MASK]": 0,
+  "[PAD]": 2,
+  "[UNK]": 1
+}

eval/binary_classification_evaluation_eval_results.csv

@@ -0,0 +1,101 @@
+epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
+0,23,0.9382022471910112,0.840080738067627,0.7735849056603773,0.7321428571428571,0.82,0.8183544278144836,0.8235726722373042,0.9382022471910112,85.76151275634766,0.7722772277227723,0.7647058823529411,0.78,90.44619750976562,0.8334727580500052,0.9438202247191011,4.150282382965088,0.7872340425531915,0.8409090909090909,0.74,4.150282382965088,0.8295236120734805,0.8904494382022472,43.70259475708008,0.6095238095238096,0.5818181818181818,0.64,42.006656646728516,0.5567817419766898
+0,-1,0.9382022471910112,0.813146710395813,0.78,0.78,0.78,0.797511100769043,0.8303248592676866,0.9438202247191011,93.32714080810547,0.7916666666666667,0.8260869565217391,0.76,93.32714080810547,0.8459640727515764,0.9438202247191011,4.261557579040527,0.7916666666666667,0.8260869565217391,0.76,4.3601250648498535,0.8471509938288564,0.901685393258427,38.66168975830078,0.6346153846153846,0.6111111111111112,0.66,38.05706024169922,0.5806093097883482
+1,23,0.9466292134831461,0.7603912949562073,0.8041237113402062,0.8297872340425532,0.78,0.7603912949562073,0.8395191619567781,0.949438202247191,97.29362487792969,0.8125,0.8478260869565217,0.78,98.465087890625,0.8553024238307882,0.949438202247191,4.441499710083008,0.8043478260869565,0.8809523809523809,0.74,4.526854991912842,0.8592247049166957,0.9101123595505618,33.11018371582031,0.6776859504132231,0.5774647887323944,0.82,31.435935974121094,0.6158494750968696
+1,-1,0.9466292134831461,0.7351434230804443,0.8118811881188118,0.803921568627451,0.82,0.7351434230804443,0.8606496416343485,0.9578651685393258,100.54653930664062,0.8387096774193548,0.9069767441860465,0.78,100.54653930664062,0.8695940380141225,0.9578651685393258,4.66744327545166,0.8351648351648352,0.926829268292683,0.76,4.66744327545166,0.8722519462813986,0.9185393258426966,32.856346130371094,0.7079646017699115,0.6349206349206349,0.8,30.166812896728516,0.6441181217662475
+2,23,0.949438202247191,0.7490123510360718,0.826923076923077,0.7962962962962963,0.86,0.7283334732055664,0.8781740970465758,0.9606741573033708,100.11785125732422,0.8478260869565217,0.9285714285714286,0.78,100.11785125732422,0.8813285965561884,0.9606741573033708,4.6823272705078125,0.8478260869565217,0.9285714285714286,0.78,4.6823272705078125,0.886271124069354,0.9241573033707865,33.71270751953125,0.743362831858407,0.6666666666666666,0.84,30.854446411132812,0.6779590587002511
+2,-1,0.9550561797752809,0.773331880569458,0.826923076923077,0.7962962962962963,0.86,0.7285211086273193,0.8908238050704685,0.9606741573033708,99.71263885498047,0.8478260869565217,0.9285714285714286,0.78,101.50850677490234,0.8868426339380894,0.9606741573033708,4.590607643127441,0.8510638297872342,0.9090909090909091,0.8,4.769964218139648,0.8949396000225113,0.9297752808988764,34.16911315917969,0.7522935779816514,0.6949152542372882,0.82,32.40364456176758,0.6853486948283198
+3,23,0.9578651685393258,0.7916141748428345,0.8484848484848485,0.8571428571428571,0.84,0.7613775134086609,0.8995930548874173,0.9606741573033708,97.85294342041016,0.8510638297872342,0.9090909090909091,0.8,101.95211791992188,0.8978259663044791,0.9662921348314607,4.588953495025635,0.8666666666666667,0.975,0.78,4.588953495025635,0.9008506038714967,0.9297752808988764,36.435245513916016,0.7706422018348624,0.711864406779661,0.84,35.1168212890625,0.7099683324789272
+3,-1,0.9578651685393258,0.7639147639274597,0.8484848484848485,0.8571428571428571,0.84,0.7639147639274597,0.9019985695411536,0.9634831460674157,98.48260498046875,0.8602150537634408,0.9302325581395349,0.8,100.23257446289062,0.9023077309965295,0.9662921348314607,4.519453048706055,0.8666666666666667,0.975,0.78,4.519453048706055,0.9048488877031652,0.9297752808988764,36.021888732910156,0.7706422018348624,0.711864406779661,0.84,35.57328796386719,0.7070033555936083
+4,23,0.9578651685393258,0.7749492526054382,0.8484848484848485,0.8571428571428571,0.84,0.7749492526054382,0.9036814965664277,0.9634831460674157,98.31718444824219,0.8602150537634408,0.9302325581395349,0.8,100.60600280761719,0.9045463152971923,0.9662921348314607,4.680278778076172,0.8723404255319149,0.9318181818181818,0.82,4.680278778076172,0.9060246426807765,0.9297752808988764,38.67071533203125,0.7572815533980584,0.7358490566037735,0.78,38.67071533203125,0.7192534717243018
+4,-1,0.9606741573033708,0.7528070211410522,0.8541666666666666,0.8913043478260869,0.82,0.7528070211410522,0.9074608523311473,0.9634831460674157,102.06263732910156,0.8602150537634408,0.9302325581395349,0.8,104.72052001953125,0.9076502590424322,0.9662921348314607,4.820178985595703,0.8695652173913043,0.9523809523809523,0.8,4.820178985595703,0.9111976412447476,0.9382022471910112,36.30040740966797,0.7843137254901961,0.7692307692307693,0.8,36.12192153930664,0.7430597431705827
+5,23,0.9662921348314607,0.7873730659484863,0.8695652173913043,0.9523809523809523,0.8,0.7873730659484863,0.9110235656408233,0.9662921348314607,97.65350341796875,0.8695652173913043,0.9523809523809523,0.8,102.03347778320312,0.9118109214284268,0.9662921348314607,4.481189727783203,0.8749999999999999,0.9130434782608695,0.84,4.793779373168945,0.9131460894610325,0.9410112359550562,37.05552673339844,0.796116504854369,0.7735849056603774,0.82,37.05552673339844,0.7543030477861441
+5,-1,0.9662921348314607,0.7745945453643799,0.8723404255319149,0.9318181818181818,0.82,0.7745945453643799,0.9075423250076066,0.9662921348314607,98.09512329101562,0.8695652173913043,0.9523809523809523,0.8,102.76459503173828,0.912138935521147,0.9691011235955056,4.813371658325195,0.8865979381443299,0.9148936170212766,0.86,4.813371658325195,0.9139851441420692,0.9269662921348315,37.337860107421875,0.7387387387387387,0.6721311475409836,0.82,35.12767028808594,0.7138718162074226
+6,23,0.9662921348314607,0.784274160861969,0.8723404255319149,0.9318181818181818,0.82,0.784274160861969,0.9118521377996901,0.9691011235955056,101.94235229492188,0.8791208791208791,0.975609756097561,0.8,101.94235229492188,0.9130350388468179,0.9691011235955056,4.776327610015869,0.8842105263157894,0.9333333333333333,0.84,4.776327610015869,0.9137060198672389,0.9185393258426966,38.715240478515625,0.7222222222222223,0.6724137931034483,0.78,37.50344467163086,0.7138343581656884
+6,-1,0.9662921348314607,0.8082194328308105,0.8723404255319149,0.9318181818181818,0.82,0.7982444167137146,0.9110783025545418,0.9691011235955056,101.05641174316406,0.8817204301075269,0.9534883720930233,0.82,101.05641174316406,0.9144142077966783,0.9662921348314607,4.619779109954834,0.8749999999999999,0.9130434782608695,0.84,4.701949119567871,0.9132565323482589,0.9241573033707865,41.05711364746094,0.7378640776699028,0.7169811320754716,0.76,40.46123504638672,0.747586027106742
+7,23,0.9691011235955056,0.7589374780654907,0.8865979381443299,0.9148936170212766,0.86,0.7554340362548828,0.9099528353589518,0.9719101123595506,104.18718719482422,0.8936170212765958,0.9545454545454546,0.84,104.18718719482422,0.9168950332603157,0.9719101123595506,4.781330585479736,0.8958333333333334,0.9347826086956522,0.86,4.793957710266113,0.9167010742683522,0.9213483146067416,37.08013916015625,0.7272727272727273,0.6197183098591549,0.88,34.64673614501953,0.7344805981128737
+7,-1,0.9691011235955056,0.812466561794281,0.8865979381443299,0.9148936170212766,0.86,0.7876336574554443,0.9172867398241914,0.9775280898876404,102.20433044433594,0.9148936170212766,0.9772727272727273,0.86,102.20433044433594,0.9206815593869297,0.9775280898876404,4.724061965942383,0.9166666666666666,0.9565217391304348,0.88,4.724061965942383,0.9195469902138874,0.9213483146067416,44.55482482910156,0.736842105263158,0.65625,0.84,41.272239685058594,0.7550655950758518
+8,23,0.9719101123595506,0.8115893602371216,0.8936170212765958,0.9545454545454546,0.84,0.7980576157569885,0.9135481512991681,0.9747191011235955,101.14024353027344,0.9032258064516129,0.9767441860465116,0.84,101.14024353027344,0.9193863298039925,0.9747191011235955,4.755157470703125,0.9052631578947369,0.9555555555555556,0.86,4.755157470703125,0.9181353646725379,0.9185393258426966,45.63719940185547,0.7017543859649122,0.625,0.8,42.213356018066406,0.7281805419403353
+8,-1,0.9719101123595506,0.8089303374290466,0.8913043478260869,0.9761904761904762,0.82,0.8089303374290466,0.9138948181620618,0.9775280898876404,104.31477355957031,0.9148936170212766,0.9772727272727273,0.86,104.31477355957031,0.9174999910739636,0.9747191011235955,4.748935222625732,0.9032258064516129,0.9767441860465116,0.84,4.748935222625732,0.9151090974125788,0.9185393258426966,45.224365234375,0.7413793103448276,0.6515151515151515,0.86,42.3104248046875,0.7486241281720253
+9,23,0.9747191011235955,0.7951878905296326,0.9052631578947369,0.9555555555555556,0.86,0.7951878905296326,0.9139730659867744,0.9775280898876404,101.84993743896484,0.9148936170212766,0.9772727272727273,0.86,101.84993743896484,0.9173949469894259,0.9747191011235955,4.700490951538086,0.9072164948453608,0.9361702127659575,0.88,4.769707679748535,0.9165808939266344,0.9297752808988764,46.13153076171875,0.7394957983193275,0.6376811594202898,0.88,43.018436431884766,0.7770313601018408
+9,-1,0.9719101123595506,0.8215473890304565,0.8936170212765958,0.9545454545454546,0.84,0.8125100135803223,0.9154544522894188,0.9747191011235955,101.68478393554688,0.9052631578947369,0.9555555555555556,0.86,103.7650146484375,0.9187658099739966,0.9747191011235955,4.732616901397705,0.9052631578947369,0.9555555555555556,0.86,4.7795915603637695,0.9176036977654254,0.9269662921348315,51.09953308105469,0.7413793103448276,0.6515151515151515,0.86,47.369140625,0.7842836873984204
+10,23,0.9719101123595506,0.7904789447784424,0.8913043478260869,0.9761904761904762,0.82,0.7904789447784424,0.9104640493111918,0.9747191011235955,105.47246551513672,0.9032258064516129,0.9767441860465116,0.84,105.47246551513672,0.9156862395315475,0.9719101123595506,4.841980457305908,0.8979591836734694,0.9166666666666666,0.88,5.029376029968262,0.9137205264213775,0.9241573033707865,41.21728515625,0.7610619469026547,0.6825396825396826,0.86,41.21728515625,0.76297419114765
+10,-1,0.9691011235955056,0.8140467405319214,0.8817204301075269,0.9534883720930233,0.82,0.8050681352615356,0.9115121778673786,0.9719101123595506,102.23046112060547,0.8979591836734694,0.9166666666666666,0.88,110.34585571289062,0.9160244975300187,0.9719101123595506,4.854016304016113,0.8958333333333334,0.9347826086956522,0.86,4.959338188171387,0.9138280538000375,0.9213483146067416,44.2801628112793,0.7586206896551725,0.6666666666666666,0.88,43.71131896972656,0.7551586897876873
+11,23,0.9719101123595506,0.8020027279853821,0.8913043478260869,0.9761904761904762,0.82,0.8020027279853821,0.9124019303449254,0.9719101123595506,103.39257049560547,0.8958333333333334,0.9347826086956522,0.86,105.9890365600586,0.9171029720445024,0.9719101123595506,4.849503517150879,0.9,0.9,0.9,5.132312774658203,0.9163350271170014,0.9269662921348315,42.503814697265625,0.7547169811320756,0.7142857142857143,0.8,42.37981414794922,0.7699831722369624
+11,-1,0.9691011235955056,0.7768244743347168,0.8842105263157894,0.9333333333333333,0.84,0.7582629323005676,0.903285395527375,0.9747191011235955,107.10214233398438,0.9052631578947369,0.9555555555555556,0.86,107.10214233398438,0.913066918067066,0.9775280898876404,4.986804962158203,0.9183673469387755,0.9375,0.9,4.986804962158203,0.9135626470552293,0.9269662921348315,39.004215240478516,0.7407407407407408,0.6896551724137931,0.8,36.964969635009766,0.7461690301400378
+12,23,0.9747191011235955,0.7970025539398193,0.9032258064516129,0.9767441860465116,0.84,0.7970025539398193,0.9102056242312988,0.9803370786516854,104.02931213378906,0.9263157894736842,0.9777777777777777,0.88,104.02931213378906,0.9184098369182431,0.9803370786516854,4.761752128601074,0.9263157894736842,0.9777777777777777,0.88,4.761752128601074,0.9169216560714304,0.9269662921348315,43.83094024658203,0.7289719626168225,0.6842105263157895,0.78,42.47901916503906,0.7638110746994524
+12,-1,0.9747191011235955,0.7781329154968262,0.9052631578947369,0.9555555555555556,0.86,0.7687379121780396,0.9116145346305427,0.9775280898876404,105.80572509765625,0.9148936170212766,0.9772727272727273,0.86,105.80572509765625,0.9172019174843666,0.9775280898876404,4.882781505584717,0.9148936170212766,0.9772727272727273,0.86,4.882781505584717,0.9160458871422805,0.9438202247191011,39.73921203613281,0.8000000000000002,0.8,0.8,39.73921203613281,0.7813534993116035
+13,23,0.9691011235955056,0.7882981300354004,0.8865979381443299,0.9148936170212766,0.86,0.7618275880813599,0.9046568978936869,0.9775280898876404,108.10537719726562,0.9166666666666666,0.9565217391304348,0.88,108.10537719726562,0.9121038182187416,0.9775280898876404,4.953481674194336,0.9166666666666666,0.9565217391304348,0.88,4.953481674194336,0.9103933559010408,0.9269662921348315,38.739654541015625,0.7592592592592593,0.7068965517241379,0.82,38.739654541015625,0.7534890850035172
+13,-1,0.9719101123595506,0.7888282537460327,0.8958333333333334,0.9347826086956522,0.86,0.7794239521026611,0.9050664095399362,0.9775280898876404,106.4237060546875,0.9166666666666666,0.9565217391304348,0.88,106.4237060546875,0.9132408510887917,0.9775280898876404,4.882312774658203,0.9166666666666666,0.9565217391304348,0.88,4.882312774658203,0.9129982873762051,0.9325842696629213,42.002830505371094,0.7777777777777777,0.7241379310344828,0.84,41.52927017211914,0.7520835179182389
+14,23,0.9747191011235955,0.7946066856384277,0.9052631578947369,0.9555555555555556,0.86,0.7946066856384277,0.9070406442247817,0.9775280898876404,104.76173400878906,0.9166666666666666,0.9565217391304348,0.88,106.05206298828125,0.9122784730227579,0.9775280898876404,4.841429233551025,0.9166666666666666,0.9565217391304348,0.88,4.895715713500977,0.9123867656523225,0.9297752808988764,43.72325897216797,0.7826086956521738,0.6923076923076923,0.9,43.72325897216797,0.7496662108706375
+14,-1,0.9747191011235955,0.79632568359375,0.9052631578947369,0.9555555555555556,0.86,0.79632568359375,0.904249946650528,0.9775280898876404,105.94549560546875,0.9166666666666666,0.9565217391304348,0.88,105.94549560546875,0.9084665469078113,0.9719101123595506,4.869645118713379,0.8979591836734694,0.9166666666666666,0.88,4.978684902191162,0.9071520334025405,0.9269662921348315,45.64033508300781,0.7567567567567568,0.6885245901639344,0.84,43.902076721191406,0.7446525979550956
+15,23,0.9747191011235955,0.757116436958313,0.9052631578947369,0.9555555555555556,0.86,0.757116436958313,0.9029902558306236,0.9775280898876404,111.67085266113281,0.9166666666666666,0.9565217391304348,0.88,111.67085266113281,0.9082006723137132,0.9775280898876404,5.1614837646484375,0.9166666666666666,0.9565217391304348,0.88,5.1614837646484375,0.9087476596521028,0.9325842696629213,38.91661071777344,0.7663551401869159,0.7192982456140351,0.82,36.84577178955078,0.7662666681263876
+15,-1,0.9775280898876404,0.7731627225875854,0.9148936170212766,0.9772727272727273,0.86,0.7731627225875854,0.906781400252997,0.9775280898876404,107.91319274902344,0.9166666666666666,0.9565217391304348,0.88,107.91319274902344,0.912105720679669,0.9747191011235955,4.949427127838135,0.9072164948453608,0.9361702127659575,0.88,4.9965620040893555,0.9102785823579945,0.9410112359550562,38.2437744140625,0.8000000000000002,0.7636363636363637,0.84,37.98530960083008,0.789259865975788
+16,23,0.9775280898876404,0.7861059904098511,0.9148936170212766,0.9772727272727273,0.86,0.7861059904098511,0.9120837283944687,0.9775280898876404,105.0181884765625,0.9166666666666666,0.9565217391304348,0.88,105.0181884765625,0.9163208293635245,0.9775280898876404,4.821499347686768,0.9148936170212766,0.9772727272727273,0.86,4.821499347686768,0.914156622875107,0.9466292134831461,39.19158935546875,0.822429906542056,0.7719298245614035,0.88,39.094886779785156,0.8064825296704212
+16,-1,0.9747191011235955,0.7740309238433838,0.9052631578947369,0.9555555555555556,0.86,0.7740309238433838,0.90698702531905,0.9775280898876404,107.54243469238281,0.9166666666666666,0.9565217391304348,0.88,107.54243469238281,0.9148177473637294,0.9775280898876404,4.910013198852539,0.9166666666666666,0.9565217391304348,0.88,4.910013198852539,0.9129993070084564,0.9297752808988764,40.640464782714844,0.7272727272727272,0.7346938775510204,0.72,39.40990447998047,0.7539923058298452
+17,23,0.9719101123595506,0.754718542098999,0.8913043478260869,0.9761904761904762,0.82,0.754718542098999,0.9056354480783236,0.9775280898876404,110.16251373291016,0.9166666666666666,0.9565217391304348,0.88,110.16251373291016,0.9108866635883371,0.9775280898876404,5.053890228271484,0.9166666666666666,0.9565217391304348,0.88,5.053890228271484,0.9100394956941897,0.9353932584269663,36.54296112060547,0.7706422018348624,0.711864406779661,0.84,34.361900329589844,0.7611379120460091
+17,-1,0.9719101123595506,0.7743546962738037,0.8936170212765958,0.9545454545454546,0.84,0.7743546962738037,0.9109697633377534,0.9803370786516854,109.03651428222656,0.9263157894736842,0.9777777777777777,0.88,109.03651428222656,0.9188175539381039,0.9775280898876404,4.971738815307617,0.9166666666666666,0.9565217391304348,0.88,5.141246318817139,0.917652332008845,0.9269662921348315,44.812103271484375,0.6947368421052631,0.7333333333333333,0.66,42.01117706298828,0.7437167793759742
+18,23,0.9691011235955056,0.7767311334609985,0.8865979381443299,0.9148936170212766,0.86,0.7647597193717957,0.909612508829638,0.9775280898876404,111.6614761352539,0.9183673469387755,0.9375,0.9,112.78128051757812,0.9160636671576811,0.9775280898876404,5.169008255004883,0.9183673469387755,0.9375,0.9,5.169008255004883,0.9145299290449119,0.9185393258426966,42.87453842163086,0.6947368421052631,0.7333333333333333,0.66,41.97221374511719,0.7145702263619905
+18,-1,0.9691011235955056,0.674868643283844,0.8865979381443299,0.9148936170212766,0.86,0.674868643283844,0.8929955055433271,0.9747191011235955,117.67329406738281,0.9072164948453608,0.9361702127659575,0.88,118.14453887939453,0.9096905962124818,0.9775280898876404,5.440142631530762,0.9183673469387755,0.9375,0.9,5.440142631530762,0.9109213643726172,0.9185393258426966,33.14012908935547,0.6774193548387097,0.5675675675675675,0.84,27.827850341796875,0.6994926475798816
+19,23,0.9691011235955056,0.8085016012191772,0.888888888888889,0.8979591836734694,0.88,0.7923216819763184,0.9121304350787441,0.9747191011235955,105.7776870727539,0.9052631578947369,0.9555555555555556,0.86,105.7776870727539,0.9132313789145788,0.9747191011235955,4.890503406524658,0.9072164948453608,0.9361702127659575,0.88,5.013632774353027,0.9156334214635932,0.949438202247191,48.52427673339844,0.8235294117647058,0.8076923076923077,0.84,48.20258712768555,0.8114107242173342
+19,-1,0.9719101123595506,0.7987281084060669,0.8958333333333334,0.9347826086956522,0.86,0.7987281084060669,0.9151371560917831,0.9747191011235955,107.01678466796875,0.9052631578947369,0.9555555555555556,0.86,107.01678466796875,0.9190704661316182,0.9719101123595506,4.872873783111572,0.8958333333333334,0.9347826086956522,0.86,4.9853057861328125,0.9164692029052939,0.9382022471910112,48.30131530761719,0.7777777777777777,0.7241379310344828,0.84,46.090789794921875,0.8039797064519341
+20,23,0.9719101123595506,0.823287844657898,0.8936170212765958,0.9545454545454546,0.84,0.8203599452972412,0.9124273156763468,0.9719101123595506,105.75727844238281,0.9,0.9,0.9,112.07426452636719,0.9158118869667915,0.9719101123595506,4.8918962478637695,0.9,0.9,0.9,5.1326141357421875,0.9149155565645958,0.9297752808988764,55.905242919921875,0.7346938775510204,0.75,0.72,55.22452926635742,0.7568166130734612
+20,-1,0.9719101123595506,0.7956008911132812,0.8913043478260869,0.9761904761904762,0.82,0.7956008911132812,0.9050523868728308,0.9719101123595506,108.00279998779297,0.8913043478260869,0.9761904761904762,0.82,108.00279998779297,0.9075868120289698,0.9719101123595506,4.966668128967285,0.8913043478260869,0.9761904761904762,0.82,4.966668128967285,0.9083919668491782,0.9297752808988764,46.14461898803711,0.7524752475247525,0.7450980392156863,0.76,46.14461898803711,0.7659691048668138
+21,23,0.9747191011235955,0.7635411024093628,0.9032258064516129,0.9767441860465116,0.84,0.7635411024093628,0.9004684778420882,0.9747191011235955,112.10531616210938,0.9090909090909091,0.9183673469387755,0.9,112.10531616210938,0.9095824205736776,0.9747191011235955,5.083750247955322,0.9072164948453608,0.9361702127659575,0.88,5.083750247955322,0.906717488173826,0.9297752808988764,40.989994049072266,0.7326732673267328,0.7254901960784313,0.74,37.79800796508789,0.7409558677829522
+21,-1,0.9691011235955056,0.8064905405044556,0.888888888888889,0.8979591836734694,0.88,0.8025293350219727,0.9055735930855818,0.9691011235955056,107.59934997558594,0.8842105263157894,0.9333333333333333,0.84,108.37602233886719,0.9081878231615411,0.9719101123595506,4.953322410583496,0.8936170212765958,0.9545454545454546,0.84,4.953322410583496,0.906394138074706,0.9325842696629213,53.28094482421875,0.7818181818181817,0.7166666666666667,0.86,50.701171875,0.7892744951621148
+22,23,0.9634831460674157,0.8401658535003662,0.8599999999999999,0.86,0.86,0.8055012226104736,0.9027793545456066,0.9662921348314607,102.38534545898438,0.8686868686868686,0.8775510204081632,0.86,111.46479797363281,0.9017063065583569,0.9662921348314607,4.686491966247559,0.8686868686868686,0.8775510204081632,0.86,5.13873815536499,0.9012891033448521,0.9382022471910112,56.63432693481445,0.7708333333333333,0.8043478260869565,0.74,56.0113410949707,0.7937445725744379
+22,-1,0.9606741573033708,0.764726996421814,0.8599999999999999,0.86,0.86,0.764726996421814,0.8925162405927153,0.9662921348314607,107.48716735839844,0.8749999999999999,0.9130434782608695,0.84,107.48716735839844,0.8973321553062314,0.9662921348314607,4.636628150939941,0.8749999999999999,0.9130434782608695,0.84,4.914417266845703,0.8986961865229492,0.9325842696629213,39.604164123535156,0.7735849056603773,0.7321428571428571,0.82,39.604164123535156,0.7376290540021939
+23,23,0.9578651685393258,0.7800161838531494,0.8453608247422681,0.8723404255319149,0.82,0.7681931853294373,0.8901655921819371,0.9634831460674157,97.94947814941406,0.8659793814432989,0.8936170212765957,0.84,104.00948333740234,0.900430879417592,0.9662921348314607,4.685737133026123,0.8723404255319149,0.9318181818181818,0.82,4.685737133026123,0.9002693203488853,0.9213483146067416,37.30484390258789,0.7256637168141592,0.6507936507936508,0.82,34.770263671875,0.6970406581647776
+23,-1,0.9634831460674157,0.8007707595825195,0.8602150537634408,0.9302325581395349,0.8,0.7886914014816284,0.8916547322901897,0.9719101123595506,104.92393493652344,0.8958333333333334,0.9347826086956522,0.86,104.92393493652344,0.9034191809808072,0.9691011235955056,4.709863662719727,0.8842105263157894,0.9333333333333333,0.84,4.709863662719727,0.8998593238268282,0.9241573033707865,40.71446990966797,0.736842105263158,0.65625,0.84,38.09159851074219,0.7070230820743012
+24,23,0.9662921348314607,0.7919774055480957,0.8749999999999999,0.9130434782608695,0.84,0.7919774055480957,0.8954822084942367,0.9747191011235955,109.92268371582031,0.9072164948453608,0.9361702127659575,0.88,109.92268371582031,0.904516227561186,0.9747191011235955,5.035296440124512,0.9072164948453608,0.9361702127659575,0.88,5.035296440124512,0.9053701358140152,0.9353932584269663,47.75213623046875,0.7526881720430109,0.813953488372093,0.7,46.699703216552734,0.7838424957166424
+24,-1,0.9719101123595506,0.8171934485435486,0.8913043478260869,0.9761904761904762,0.82,0.8171934485435486,0.9071077472380257,0.9719101123595506,107.57754516601562,0.8979591836734694,0.9166666666666666,0.88,112.79017639160156,0.911802816967448,0.9719101123595506,4.905102252960205,0.8958333333333334,0.9347826086956522,0.86,5.143601894378662,0.9105371914358391,0.9410112359550562,51.941619873046875,0.792079207920792,0.7843137254901961,0.8,51.55586624145508,0.8109034843105584
+25,23,0.9691011235955056,0.8306732773780823,0.8791208791208791,0.975609756097561,0.8,0.8306732773780823,0.9097568347273967,0.9747191011235955,109.22808074951172,0.9052631578947369,0.9555555555555556,0.86,109.22808074951172,0.9110112046435387,0.9747191011235955,5.032811164855957,0.9052631578947369,0.9555555555555556,0.86,5.032811164855957,0.9135140880776013,0.9353932584269663,56.224464416503906,0.7857142857142856,0.7096774193548387,0.88,53.2777099609375,0.8091668770225988
+25,-1,0.9719101123595506,0.8341142535209656,0.8936170212765958,0.9545454545454546,0.84,0.8341142535209656,0.9026968985158847,0.9747191011235955,108.14537811279297,0.9052631578947369,0.9555555555555556,0.86,108.14537811279297,0.9054558387823018,0.9747191011235955,4.991763114929199,0.9052631578947369,0.9555555555555556,0.86,4.991763114929199,0.9047716919337345,0.9269662921348315,56.1902961730957,0.7291666666666667,0.7608695652173914,0.7,56.1902961730957,0.7431722165786608
+26,23,0.9719101123595506,0.8293887376785278,0.8979591836734694,0.9166666666666666,0.88,0.8218826651573181,0.9105506168943884,0.9775280898876404,104.31019592285156,0.9183673469387755,0.9375,0.9,105.4585189819336,0.9142801564780593,0.9775280898876404,4.792690277099609,0.9183673469387755,0.9375,0.9,4.843234062194824,0.9131303262091556,0.9353932584269663,53.6370735168457,0.7722772277227723,0.7647058823529411,0.78,53.6370735168457,0.7613492414284447
+26,-1,0.9691011235955056,0.8174388408660889,0.8842105263157894,0.9333333333333333,0.84,0.8174388408660889,0.9154412181719328,0.9775280898876404,105.65414428710938,0.9166666666666666,0.9565217391304348,0.88,105.65414428710938,0.9223135168799836,0.9747191011235955,4.846776962280273,0.9052631578947369,0.9555555555555556,0.86,4.846776962280273,0.920789827357473,0.9410112359550562,53.69249725341797,0.7924528301886793,0.75,0.84,50.78434753417969,0.8037078603890299
+27,23,0.9691011235955056,0.7670748829841614,0.8865979381443299,0.9148936170212766,0.86,0.7670748829841614,0.901147384948236,0.9747191011235955,107.53291320800781,0.9052631578947369,0.9555555555555556,0.86,107.53291320800781,0.9169106728756218,0.9747191011235955,5.009335517883301,0.9052631578947369,0.9555555555555556,0.86,5.009335517883301,0.9158314493045155,0.9410112359550562,40.34715270996094,0.8037383177570094,0.7543859649122807,0.86,40.34715270996094,0.7761040178934342
+27,-1,0.9691011235955056,0.7854174971580505,0.8842105263157894,0.9333333333333333,0.84,0.7854174971580505,0.8973759563883442,0.9775280898876404,103.55308532714844,0.9183673469387755,0.9375,0.9,103.55308532714844,0.9101116516474784,0.9775280898876404,4.7557854652404785,0.9183673469387755,0.9375,0.9,4.7557854652404785,0.9083740500178366,0.9241573033707865,40.11735153198242,0.7256637168141592,0.6507936507936508,0.82,37.420249938964844,0.7171167704461994
+28,23,0.9691011235955056,0.7878406047821045,0.8865979381443299,0.9148936170212766,0.86,0.7797698974609375,0.9169887974633828,0.9747191011235955,110.05758666992188,0.9090909090909091,0.9183673469387755,0.9,112.28560638427734,0.9184759593913688,0.9747191011235955,5.042779445648193,0.9090909090909091,0.9183673469387755,0.9,5.181422233581543,0.9179344968780414,0.9550561797752809,46.13587188720703,0.8316831683168315,0.8235294117647058,0.84,45.066551208496094,0.8259809690568878
+28,-1,0.9719101123595506,0.8004018068313599,0.8936170212765958,0.9545454545454546,0.84,0.8004018068313599,0.9191378840809217,0.9775280898876404,114.09004211425781,0.9183673469387755,0.9375,0.9,115.54694366455078,0.9200078896480781,0.9803370786516854,5.277122974395752,0.9278350515463918,0.9574468085106383,0.9,5.277122974395752,0.9198258817171239,0.949438202247191,50.04240417480469,0.8085106382978724,0.8636363636363636,0.76,49.77994155883789,0.8340606029144928
+29,23,0.9747191011235955,0.7224353551864624,0.9052631578947369,0.9555555555555556,0.86,0.7224353551864624,0.9104707693350436,0.9775280898876404,121.73721313476562,0.9166666666666666,0.9565217391304348,0.88,121.73721313476562,0.9108217116738783,0.9775280898876404,5.608170032501221,0.9166666666666666,0.9565217391304348,0.88,5.608170032501221,0.911392465341344,0.952247191011236,39.72545623779297,0.8316831683168315,0.8235294117647058,0.84,38.578704833984375,0.8637338658961354
+29,-1,0.9747191011235955,0.8046965599060059,0.9052631578947369,0.9555555555555556,0.86,0.8046965599060059,0.91608650641987,0.9775280898876404,110.248291015625,0.9166666666666666,0.9565217391304348,0.88,110.248291015625,0.9184548549009227,0.9747191011235955,4.992629051208496,0.9072164948453608,0.9361702127659575,0.88,5.152335166931152,0.9165339731796192,0.9410112359550562,55.04010772705078,0.780952380952381,0.7454545454545455,0.82,52.919532775878906,0.8443753466795544
+30,23,0.9747191011235955,0.7794127464294434,0.9052631578947369,0.9555555555555556,0.86,0.7794127464294434,0.9103000848619597,0.9747191011235955,112.59974670410156,0.9072164948453608,0.9361702127659575,0.88,117.51786804199219,0.9133412801325221,0.9747191011235955,5.169946670532227,0.9072164948453608,0.9361702127659575,0.88,5.3647050857543945,0.9106972407790476,0.949438202247191,47.883853912353516,0.816326530612245,0.8333333333333334,0.8,45.9074821472168,0.8423890083666495
+30,-1,0.9691011235955056,0.771475613117218,0.8842105263157894,0.9333333333333333,0.84,0.7666321992874146,0.8943102007152268,0.9719101123595506,114.31948852539062,0.8958333333333334,0.9347826086956522,0.86,117.26937866210938,0.9023545682572001,0.9719101123595506,5.272520065307617,0.8979591836734694,0.9166666666666666,0.88,5.698516845703125,0.9010073516444941,0.9438202247191011,45.77439880371094,0.7916666666666667,0.8260869565217391,0.76,44.493690490722656,0.8080733199276151
+31,23,0.9691011235955056,0.798578143119812,0.8817204301075269,0.9534883720930233,0.82,0.798578143119812,0.8967523450202168,0.9719101123595506,114.43120574951172,0.8958333333333334,0.9347826086956522,0.86,114.43120574951172,0.903047258325471,0.9719101123595506,5.232242107391357,0.8958333333333334,0.9347826086956522,0.86,5.262118339538574,0.9011113126251549,0.9466292134831461,52.36563491821289,0.7956989247311828,0.8604651162790697,0.74,50.673736572265625,0.8344457862936273
+31,-1,0.9691011235955056,0.80713951587677,0.8817204301075269,0.9534883720930233,0.82,0.80713951587677,0.8965633560159961,0.9719101123595506,108.87483215332031,0.8936170212765958,0.9545454545454546,0.84,108.87483215332031,0.9019055173309651,0.9719101123595506,5.025306701660156,0.8936170212765958,0.9545454545454546,0.84,5.025306701660156,0.9005207862038968,0.952247191011236,52.998291015625,0.8172043010752689,0.8837209302325582,0.76,51.89453887939453,0.8327237408143631
+32,23,0.9578651685393258,0.8509979248046875,0.8431372549019608,0.8269230769230769,0.86,0.7698633670806885,0.9133429487889059,0.9634831460674157,117.80952453613281,0.8659793814432989,0.8936170212765957,0.84,117.80952453613281,0.9191265441596094,0.9634831460674157,5.433820724487305,0.8659793814432989,0.8936170212765957,0.84,5.433820724487305,0.9168337179669709,0.9297752808988764,56.0783576965332,0.7368421052631577,0.7777777777777778,0.7,56.0783576965332,0.7657086161867439
+32,-1,0.9662921348314607,0.7151848077774048,0.8775510204081632,0.8958333333333334,0.86,0.7151848077774048,0.9147329630411819,0.9691011235955056,120.76054382324219,0.8817204301075269,0.9534883720930233,0.82,120.76054382324219,0.9149839374427727,0.9691011235955056,5.424457550048828,0.8791208791208791,0.975609756097561,0.8,5.424457550048828,0.916244036719307,0.9550561797752809,42.87438201904297,0.836734693877551,0.8541666666666666,0.82,42.87438201904297,0.8424976532809563
+33,23,0.9578651685393258,0.7741923332214355,0.8543689320388349,0.8301886792452831,0.88,0.7293167114257812,0.8863423225598874,0.9662921348314607,129.7685546875,0.88,0.88,0.88,129.7685546875,0.891920744881872,0.9662921348314607,5.853617191314697,0.88,0.88,0.88,5.993412971496582,0.8889431950468689,0.9466292134831461,50.367740631103516,0.8080808080808082,0.8163265306122449,0.8,49.23870086669922,0.8173298041360954
+33,-1,0.9606741573033708,0.7939156293869019,0.8541666666666666,0.8913043478260869,0.82,0.7939156293869019,0.8892969305394881,0.9691011235955056,119.04820251464844,0.8842105263157894,0.9333333333333333,0.84,119.04820251464844,0.8897290323821317,0.9662921348314607,5.328439712524414,0.8749999999999999,0.9130434782608695,0.84,5.55683708190918,0.8903811762941045,0.9353932584269663,57.072364807128906,0.7722772277227723,0.7647058823529411,0.78,57.072364807128906,0.7929285005405954
+34,23,0.9662921348314607,0.7351672649383545,0.8695652173913043,0.9523809523809523,0.8,0.7351672649383545,0.8957986957126207,0.9691011235955056,123.15740966796875,0.8791208791208791,0.975609756097561,0.8,123.15740966796875,0.8976823786582472,0.9662921348314607,5.701198577880859,0.8695652173913043,0.9523809523809523,0.8,5.701198577880859,0.8955820330759343,0.9466292134831461,47.028587341308594,0.8041237113402062,0.8297872340425532,0.78,44.33156204223633,0.8495262782133634
+34,-1,0.9662921348314607,0.7482127547264099,0.8695652173913043,0.9523809523809523,0.8,0.7482127547264099,0.9034720269405425,0.9691011235955056,126.19216918945312,0.8817204301075269,0.9534883720930233,0.82,126.19216918945312,0.9076320394579971,0.9662921348314607,6.0310540199279785,0.8749999999999999,0.9130434782608695,0.84,6.0310540199279785,0.906511379922384,0.952247191011236,50.217315673828125,0.8247422680412372,0.851063829787234,0.8,47.532814025878906,0.8608596408779989
+35,23,0.9662921348314607,0.7344657182693481,0.8723404255319149,0.9318181818181818,0.82,0.7344657182693481,0.8822676715960408,0.9691011235955056,130.579345703125,0.8865979381443299,0.9148936170212766,0.86,133.88400268554688,0.8895173578038754,0.9691011235955056,5.934551239013672,0.8842105263157894,0.9333333333333333,0.84,5.934551239013672,0.8897578233813269,0.9353932584269663,48.589874267578125,0.7850467289719626,0.7368421052631579,0.84,45.11005401611328,0.8129291329745233
+35,-1,0.9691011235955056,0.8289074301719666,0.8842105263157894,0.9333333333333333,0.84,0.8105036616325378,0.8927206192590444,0.9719101123595506,108.30807495117188,0.8936170212765958,0.9545454545454546,0.84,108.30807495117188,0.9042745271434024,0.9747191011235955,4.93955135345459,0.9032258064516129,0.9767441860465116,0.84,4.93955135345459,0.9049139232183842,0.9466292134831461,60.215789794921875,0.7912087912087912,0.8780487804878049,0.72,60.215789794921875,0.8187799280980084
+36,23,0.9691011235955056,0.7639759182929993,0.8817204301075269,0.9534883720930233,0.82,0.7639759182929993,0.9178021529710476,0.9691011235955056,122.774169921875,0.8865979381443299,0.9148936170212766,0.86,130.08770751953125,0.9234453663968449,0.9691011235955056,5.32758092880249,0.8865979381443299,0.9148936170212766,0.86,6.0031304359436035,0.9196582716142992,0.9550561797752809,47.586849212646484,0.836734693877551,0.8541666666666666,0.82,47.586849212646484,0.8435545730577011
+36,-1,0.9691011235955056,0.7178241014480591,0.8842105263157894,0.9333333333333333,0.84,0.7178241014480591,0.9251596235681904,0.9719101123595506,125.8282470703125,0.9,0.9,0.9,130.28555297851562,0.9292603289488298,0.9719101123595506,5.9163818359375,0.8958333333333334,0.9347826086956522,0.86,5.9163818359375,0.9264111511669142,0.952247191011236,40.58966827392578,0.8380952380952381,0.8,0.88,39.92177963256836,0.8741100874703153
+37,23,0.9747191011235955,0.8196407556533813,0.9052631578947369,0.9555555555555556,0.86,0.8196407556533813,0.9072550144806358,0.9775280898876404,109.03776550292969,0.9166666666666666,0.9565217391304348,0.88,109.03776550292969,0.9149130972066069,0.9775280898876404,4.939194679260254,0.9166666666666666,0.9565217391304348,0.88,4.939194679260254,0.9139667550759527,0.9410112359550562,55.91522216796875,0.7735849056603773,0.7321428571428571,0.82,52.99152374267578,0.80746997528082
+37,-1,0.9747191011235955,0.7910473346710205,0.9052631578947369,0.9555555555555556,0.86,0.7910473346710205,0.8978197111697521,0.9775280898876404,117.82610321044922,0.9183673469387755,0.9375,0.9,117.82610321044922,0.9117760610049824,0.9747191011235955,5.12831449508667,0.9052631578947369,0.9555555555555556,0.86,5.211759090423584,0.9055976412105502,0.9466292134831461,51.36717224121094,0.8155339805825242,0.7924528301886793,0.84,50.314693450927734,0.8183498359820306
+38,23,0.9719101123595506,0.6523736715316772,0.9,0.9,0.9,0.6523736715316772,0.907342700796678,0.9775280898876404,121.2411117553711,0.9148936170212766,0.9772727272727273,0.86,121.2411117553711,0.9086431812607745,0.9747191011235955,5.642573833465576,0.9052631578947369,0.9555555555555556,0.86,5.642573833465576,0.9084653043142874,0.9606741573033708,36.94999313354492,0.8510638297872342,0.9090909090909091,0.8,36.16839599609375,0.8456060266972244
+38,-1,0.9719101123595506,0.6727628707885742,0.8958333333333334,0.9347826086956522,0.86,0.6647723913192749,0.9022854990226995,0.9747191011235955,122.4629135131836,0.9052631578947369,0.9555555555555556,0.86,124.52556610107422,0.9096850339936473,0.9719101123595506,5.687566757202148,0.8958333333333334,0.9347826086956522,0.86,5.858804225921631,0.9082905373286047,0.9578651685393258,33.620243072509766,0.8387096774193548,0.9069767441860465,0.78,33.620243072509766,0.8517905734817721
+39,23,0.9634831460674157,0.8320704698562622,0.853932584269663,0.9743589743589743,0.76,0.8320704698562622,0.8928787824879894,0.9634831460674157,100.10120391845703,0.853932584269663,0.9743589743589743,0.76,100.10120391845703,0.9020965166167986,0.9634831460674157,4.608417510986328,0.8543689320388349,0.8301886792452831,0.88,5.410019874572754,0.9026226225892755,0.9185393258426966,54.3325080871582,0.7256637168141592,0.6507936507936508,0.82,48.481117248535156,0.7399466694003426
+39,-1,0.9634831460674157,0.8288320302963257,0.853932584269663,0.9743589743589743,0.76,0.8288320302963257,0.8855243245586559,0.9634831460674157,99.03549194335938,0.853932584269663,0.9743589743589743,0.76,99.03549194335938,0.8959497465862661,0.9634831460674157,4.59616756439209,0.853932584269663,0.9743589743589743,0.76,4.59616756439209,0.8934871333075214,0.9297752808988764,51.64586639404297,0.7428571428571428,0.7090909090909091,0.78,49.31752014160156,0.7786196733697578
+40,23,0.9691011235955056,0.8004737496376038,0.8791208791208791,0.975609756097561,0.8,0.8004737496376038,0.8997960636935498,0.9691011235955056,107.90061950683594,0.8791208791208791,0.975609756097561,0.8,107.90061950683594,0.9075117240000315,0.9662921348314607,4.899551868438721,0.8666666666666667,0.975,0.78,4.899551868438721,0.9063114679016798,0.9353932584269663,48.718650817871094,0.7499999999999999,0.782608695652174,0.72,46.709739685058594,0.8224648822545105
+40,-1,0.9662921348314607,0.822917640209198,0.8823529411764707,0.8653846153846154,0.9,0.822917640209198,0.9119510054149133,0.9691011235955056,101.92906188964844,0.8865979381443299,0.9148936170212766,0.86,105.349365234375,0.922022552027087,0.9691011235955056,5.073966979980469,0.8910891089108911,0.8823529411764706,0.9,5.073966979980469,0.9201822999844316,0.9241573033707865,59.95845031738281,0.7428571428571428,0.7090909090909091,0.78,59.70689392089844,0.7840972472543023
+41,23,0.9634831460674157,0.7924543619155884,0.8659793814432989,0.8936170212765957,0.84,0.7658122777938843,0.9162084500639573,0.9691011235955056,124.7623519897461,0.8865979381443299,0.9148936170212766,0.86,124.7623519897461,0.9259046613414521,0.9662921348314607,5.577917575836182,0.8823529411764707,0.8653846153846154,0.9,5.936610221862793,0.9234229169103003,0.952247191011236,54.104820251464844,0.8316831683168315,0.8235294117647058,0.84,53.382633209228516,0.8516515587791815
+41,-1,0.9662921348314607,0.7348983287811279,0.8695652173913043,0.9523809523809523,0.8,0.717664361000061,0.9071917872092818,0.9719101123595506,130.59132385253906,0.8958333333333334,0.9347826086956522,0.86,131.3431396484375,0.9117076211621491,0.9719101123595506,6.058423042297363,0.8958333333333334,0.9347826086956522,0.86,6.058423042297363,0.9083673652262505,0.952247191011236,41.118919372558594,0.8316831683168315,0.8235294117647058,0.84,41.118919372558594,0.8849470059427466
+42,23,0.9691011235955056,0.7808043360710144,0.8865979381443299,0.9148936170212766,0.86,0.770115315914154,0.9249489821750401,0.9691011235955056,122.7652359008789,0.8865979381443299,0.9148936170212766,0.86,122.7652359008789,0.921913546353083,0.9691011235955056,5.367074012756348,0.8817204301075269,0.9534883720930233,0.82,5.367074012756348,0.9227942109087149,0.9578651685393258,51.90674591064453,0.8453608247422681,0.8723404255319149,0.82,51.90674591064453,0.8837370337493913
+42,-1,0.9719101123595506,0.7989660501480103,0.8936170212765958,0.9545454545454546,0.84,0.7989660501480103,0.9005759025130828,0.9719101123595506,114.86650085449219,0.8936170212765958,0.9545454545454546,0.84,114.86650085449219,0.9115122786821067,0.9691011235955056,5.258968353271484,0.8842105263157894,0.9333333333333333,0.84,5.355371475219727,0.9103953704971383,0.952247191011236,54.558956146240234,0.8282828282828283,0.8367346938775511,0.82,54.131446838378906,0.8267393498419923
+43,23,0.9691011235955056,0.7800118923187256,0.8817204301075269,0.9534883720930233,0.82,0.7800118923187256,0.8718857286573347,0.9662921348314607,118.63558959960938,0.8723404255319149,0.9318181818181818,0.82,118.63558959960938,0.8967513687507479,0.9662921348314607,5.426270008087158,0.8723404255319149,0.9318181818181818,0.82,5.426270008087158,0.8888229962042999,0.9269662921348315,49.935630798339844,0.7326732673267328,0.7254901960784313,0.74,48.90339660644531,0.7158043054052314
+43,-1,0.9691011235955056,0.681570291519165,0.8791208791208791,0.975609756097561,0.8,0.681570291519165,0.9149153780156811,0.9691011235955056,131.72772216796875,0.8817204301075269,0.9534883720930233,0.82,140.0543212890625,0.9184245281269029,0.9691011235955056,6.189223289489746,0.8817204301075269,0.9534883720930233,0.82,6.607734203338623,0.9178360767519411,0.9606741573033708,42.98487854003906,0.8541666666666666,0.8913043478260869,0.82,42.98487854003906,0.8907375763244297
+44,23,0.9691011235955056,0.7478029727935791,0.8842105263157894,0.9333333333333333,0.84,0.7156367301940918,0.9140839477232533,0.9691011235955056,129.75376892089844,0.8791208791208791,0.975609756097561,0.8,129.75376892089844,0.9157566702531018,0.9691011235955056,6.124690055847168,0.8791208791208791,0.975609756097561,0.8,6.124690055847168,0.9156851054291166,0.949438202247191,53.42259979248047,0.816326530612245,0.8333333333333334,0.8,52.13511657714844,0.8828666915269352
+44,-1,0.9719101123595506,0.7951579093933105,0.8936170212765958,0.9545454545454546,0.84,0.7728476524353027,0.9168019333667904,0.9719101123595506,124.56387329101562,0.8913043478260869,0.9761904761904762,0.82,124.56387329101562,0.9155202539128492,0.9719101123595506,5.799117088317871,0.8913043478260869,0.9761904761904762,0.82,5.799117088317871,0.9214407875987923,0.9691011235955056,64.69192504882812,0.8791208791208791,0.975609756097561,0.8,64.69192504882812,0.8927821762497319
+45,23,0.9747191011235955,0.7052545547485352,0.9032258064516129,0.9767441860465116,0.84,0.7052545547485352,0.9111568969974821,0.9747191011235955,144.2850341796875,0.9032258064516129,0.9767441860465116,0.84,144.2850341796875,0.915950238424466,0.9747191011235955,6.6205854415893555,0.9032258064516129,0.9767441860465116,0.84,6.6205854415893555,0.9176101830509236,0.9606741573033708,50.29444885253906,0.8541666666666666,0.8913043478260869,0.82,50.29444885253906,0.896200102118035
+45,-1,0.9719101123595506,0.8474124073982239,0.8936170212765958,0.9545454545454546,0.84,0.8115786910057068,0.9212462718038612,0.9719101123595506,117.08747863769531,0.8958333333333334,0.9347826086956522,0.86,129.83761596679688,0.9233028000493921,0.9747191011235955,5.85803747177124,0.9052631578947369,0.9555555555555556,0.86,5.85803747177124,0.9242850187557865,0.9634831460674157,76.21965789794922,0.8631578947368421,0.9111111111111111,0.82,76.21965789794922,0.8925331710007733
+46,23,0.9691011235955056,0.7851330041885376,0.8817204301075269,0.9534883720930233,0.82,0.7851330041885376,0.9104385174311029,0.9719101123595506,133.11614990234375,0.8958333333333334,0.9347826086956522,0.86,137.63067626953125,0.9169616877700927,0.9719101123595506,6.097502708435059,0.8958333333333334,0.9347826086956522,0.86,6.216365814208984,0.9170840825278279,0.952247191011236,65.80685424804688,0.8282828282828283,0.8367346938775511,0.82,65.80685424804688,0.8770450268541403
+46,-1,0.9691011235955056,0.810309112071991,0.8791208791208791,0.975609756097561,0.8,0.810309112071991,0.8982288152725908,0.9691011235955056,123.30142211914062,0.8791208791208791,0.975609756097561,0.8,123.30142211914062,0.9021352418593955,0.9719101123595506,5.858206272125244,0.8913043478260869,0.9761904761904762,0.82,5.858206272125244,0.9051185417361892,0.9578651685393258,71.53939056396484,0.8351648351648352,0.926829268292683,0.76,71.53939056396484,0.8617387425162293
+47,23,0.9662921348314607,0.8454046249389648,0.8695652173913043,0.9523809523809523,0.8,0.8334178328514099,0.9022972788644086,0.9662921348314607,113.02739715576172,0.8695652173913043,0.9523809523809523,0.8,120.10932922363281,0.90425264324172,0.9691011235955056,5.5769453048706055,0.8817204301075269,0.9534883720930233,0.82,5.5769453048706055,0.9074118686023576,0.952247191011236,75.24490356445312,0.8247422680412372,0.851063829787234,0.8,75.24490356445312,0.8605373596045227
+47,-1,0.9634831460674157,0.7196446657180786,0.8631578947368421,0.9111111111111111,0.82,0.6817929744720459,0.8981536524178254,0.9691011235955056,136.25949096679688,0.8842105263157894,0.9333333333333333,0.84,144.79000854492188,0.9021342827944547,0.9719101123595506,6.406729698181152,0.8936170212765958,0.9545454545454546,0.84,6.599481105804443,0.9037432045153644,0.9438202247191011,45.38237380981445,0.8000000000000002,0.7636363636363637,0.84,41.15211486816406,0.835872729850138
+48,23,0.9691011235955056,0.8209483027458191,0.8791208791208791,0.975609756097561,0.8,0.8209483027458191,0.9054295662549844,0.9691011235955056,112.53204345703125,0.8791208791208791,0.975609756097561,0.8,112.53204345703125,0.9130014324356966,0.9691011235955056,5.328131675720215,0.8817204301075269,0.9534883720930233,0.82,5.645000457763672,0.9085193530691246,0.9606741573033708,61.8370361328125,0.8571428571428572,0.875,0.84,61.8370361328125,0.8494774391430407
+48,-1,0.9691011235955056,0.8320106267929077,0.8791208791208791,0.975609756097561,0.8,0.8320106267929077,0.90762682805952,0.9691011235955056,105.97569274902344,0.8791208791208791,0.975609756097561,0.8,105.97569274902344,0.9133165623155651,0.9719101123595506,5.245620250701904,0.8913043478260869,0.9761904761904762,0.82,5.245620250701904,0.9056677715735726,0.9550561797752809,63.566673278808594,0.8431372549019608,0.8269230769230769,0.86,61.936798095703125,0.8466213381467006
+49,23,0.9662921348314607,0.7874312400817871,0.8775510204081632,0.8958333333333334,0.86,0.7874312400817871,0.89301644520931,0.9634831460674157,112.55329895019531,0.8686868686868686,0.8775510204081632,0.86,122.33322143554688,0.8980789025230859,0.9691011235955056,5.429780960083008,0.8842105263157894,0.9333333333333333,0.84,5.518259048461914,0.8950327055527589,0.9466292134831461,56.67962646484375,0.819047619047619,0.7818181818181819,0.86,55.01148986816406,0.8010093241627646
+49,-1,0.9606741573033708,0.7766993045806885,0.8478260869565217,0.9285714285714286,0.78,0.7659547328948975,0.8749311669047674,0.9634831460674157,122.92623901367188,0.8631578947368421,0.9111111111111111,0.82,127.7076416015625,0.8901062873900732,0.9691011235955056,5.903319358825684,0.8817204301075269,0.9534883720930233,0.82,5.903319358825684,0.8859706481932335,0.9466292134831461,50.34870529174805,0.8041237113402062,0.8297872340425532,0.78,50.34870529174805,0.7896679557303196

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50c55d19867e26fc83db98c43d59b1d3af837a78e24a9d131a4953a572074379
+size 532299592

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 128,
+  "do_lower_case": false
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8b73a5e054936c920cf5b7d1ec21ce9c281977078269963beb821c6c86fbff7
+size 841889

special_tokens_map.json

@@ -0,0 +1,73 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<ent>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ent2>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ent>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ent2>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ent>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ent2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ent>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ent2>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

test_data.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71d2b470c96d3e8108f0a18ffe2707122bb0f30bf07bc11666de18b384714b0c
+size 206402

test_results_allmethods.csv

@@ -0,0 +1,2 @@
+edit_distance,edit_distance_threshold,SBERT,SBERT_threshold,LT ZS Wiki,LT ZS Wiki_threshold,LT,LT_threshold,OpenAI,OpenAI_threshold
+0.267639902676399,0.6411929806996904,0.6902654867256638,0.8265158985798167,0.7422680412371134,0.6450361028404435,0.918918918918919,0.7219039660115472,0.854368932038835,0.9252480658768194

tokenizer_config.json

@@ -0,0 +1,106 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32769": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32770": {
+      "content": "<ent>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32771": {
+      "content": "<ent2>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<ent>",
+    "<ent2>",
+    "<ent>",
+    "<ent2>",
+    "<ent>",
+    "<ent2>",
+    "<ent>",
+    "<ent2>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "entity_mask2_token": "[MASK2]",
+  "entity_mask_token": "[MASK]",
+  "entity_pad_token": "[PAD]",
+  "entity_token_1": {
+    "__type": "AddedToken",
+    "content": "<ent>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false,
+    "special": false
+  },
+  "entity_token_2": {
+    "__type": "AddedToken",
+    "content": "<ent2>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false,
+    "special": false
+  },
+  "entity_unk_token": "[UNK]",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "max_entity_length": 32,
+  "max_mention_length": 30,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "task": null,
+  "tokenizer_class": "MLukeTokenizer",
+  "unk_token": "<unk>"
+}

val_data.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e9a840c70377691526c516f49b5712585452cf7e7f6e0a30ce621fadb84b6f6
+size 211003