model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7843055555555556
+      value: None
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.4786096256684492
+      value: None
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.47774480712166173
+      value: None
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6842690383546415
+      value: None
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.824
+      value: None
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.4692982456140351
+      value: None
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.4675925925925926
+      value: None
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -91,10 +91,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9056802772336899
+      value: None
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8978336210952347
+      value: None
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -105,10 +105,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8429577464788731
+      value: None
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6640675224037798
+      value: None
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -119,10 +119,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6690140845070423
+      value: None
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6550072748489306
+      value: None
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -133,10 +133,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.960283786603603
+      value: None
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8858405261189973
+      value: None
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -147,10 +147,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8884362268881228
+      value: None
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8833158049359685
+      value: None
 
 ---
 # relbert/relbert-roberta-large-semeval2012-v6-mask-prompt-b-nce-0
@@ -160,20 +160,20 @@ RelBERT fine-tuned from [roberta-large](https://huggingface.co/roberta-large) on
 Fine-tuning is done via [RelBERT](https://github.com/asahi417/relbert) library (see the repository for more detail).
 It achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-large-semeval2012-v6-mask-prompt-b-nce-0/raw/main/analogy.json)):
-    - Accuracy on SAT (full): 0.4786096256684492 
-    - Accuracy on SAT: 0.47774480712166173
-    - Accuracy on BATS: 0.6842690383546415
-    - Accuracy on U2: 0.4692982456140351
-    - Accuracy on U4: 0.4675925925925926
-    - Accuracy on Google: 0.824
+    - Accuracy on SAT (full): None 
+    - Accuracy on SAT: None
+    - Accuracy on BATS: None
+    - Accuracy on U2: None
+    - Accuracy on U4: None
+    - Accuracy on Google: None
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-semeval2012-v6-mask-prompt-b-nce-0/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.9056802772336899
-    - Micro F1 score on CogALexV: 0.8429577464788731
-    - Micro F1 score on EVALution: 0.6690140845070423
-    - Micro F1 score on K&H+N: 0.960283786603603
-    - Micro F1 score on ROOT09: 0.8884362268881228
+    - Micro F1 score on BLESS: None
+    - Micro F1 score on CogALexV: None
+    - Micro F1 score on EVALution: None
+    - Micro F1 score on K&H+N: None
+    - Micro F1 score on ROOT09: None
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-large-semeval2012-v6-mask-prompt-b-nce-0/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.7843055555555556 
+    - Accuracy on Relation Mapping: None 
 
 
 ### Usage
@@ -202,7 +202,7 @@ The following hyperparameters were used during training:
  - classification_loss: False
  - temperature_nce_constant: 0.05
  - temperature_nce_rank: {'min': 0.01, 'max': 0.05, 'type': 'linear'}
- - epoch: 3
+ - epoch: 6
  - batch: 128
  - lr: 5e-06
  - lr_decay: False
@@ -210,7 +210,7 @@ The following hyperparameters were used during training:
  - weight_decay: 0
  - random_seed: 0
  - exclude_relation: None
- - n_sample: 320
+ - n_sample: 640
  - gradient_accumulation: 8
  - relation_level: None
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "relbert_output/models/semeval2012-v6.b.nce_logout.mask.roberta-large.0.000005.8.0.05.640.0/best_model",
+  "_name_or_path": "roberta-large",
   "architectures": [
     "RobertaModel"
   ],

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "relbert_output/models/semeval2012-v6.b.nce_logout.mask.roberta-large.0.000005.8.0.05.640.0/best_model",
+  "name_or_path": "roberta-large",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,

trainer_config.json

@@ -1 +1 @@
-{"model": "roberta-large", "max_length": 64, "mode": "mask", "data": "relbert/semeval2012_relational_similarity_v6", "split": "train", "split_eval": "validation", "template_mode": "manual", "template": "Today, I finally discovered the relation between <subj> and <obj> : <obj>  is <subj>'s <mask>", "loss_function": "nce_logout", "classification_loss": false, "temperature_nce_constant": 0.05, "temperature_nce_rank": {"min": 0.01, "max": 0.05, "type": "linear"}, "epoch": 3, "batch": 128, "lr": 5e-06, "lr_decay": false, "lr_warmup": 1, "weight_decay": 0, "random_seed": 0, "exclude_relation": null, "n_sample": 320, "gradient_accumulation": 8, "relation_level": null}
+{"model": "roberta-large", "max_length": 64, "mode": "mask", "data": "relbert/semeval2012_relational_similarity_v6", "split": "train", "split_eval": "validation", "template_mode": "manual", "template": "Today, I finally discovered the relation between <subj> and <obj> : <obj>  is <subj>'s <mask>", "loss_function": "nce_logout", "classification_loss": false, "temperature_nce_constant": 0.05, "temperature_nce_rank": {"min": 0.01, "max": 0.05, "type": "linear"}, "epoch": 6, "batch": 128, "lr": 5e-06, "lr_decay": false, "lr_warmup": 1, "weight_decay": 0, "random_seed": 0, "exclude_relation": null, "n_sample": 640, "gradient_accumulation": 8, "relation_level": null}

validation_loss.json

@@ -1 +1 @@
-{"split": "validation", "loss": 13.770311405474391, "data": "relbert/semeval2012_relational_similarity_v6", "exclude_relation": null, "relation_level": null}
+{"split": "validation", "loss": 13.479636465386397, "data": "relbert/semeval2012_relational_similarity_v6", "exclude_relation": null, "relation_level": null}