model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7419444444444444
+      value: 0.8049007936507937
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6497326203208557
+      value: 0.732620320855615
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6528189910979229
+      value: 0.7359050445103857
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.8265703168426903
+      value: 0.8093385214007782
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.934
+      value: 0.952
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6359649122807017
+      value: 0.6754385964912281
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6064814814814815
+      value: 0.6296296296296297
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.43288590604026844
+      value: 0.4748322147651007
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6120218579234973
+      value: 0.644808743169399
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -113,10 +113,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: None
+      value: 0.9199939731806539
     - name: F1 (macro)
       type: f1_macro
-      value: None
+      value: 0.9173175984713615
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -127,10 +127,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: None
+      value: 0.8497652582159625
     - name: F1 (macro)
       type: f1_macro
-      value: None
+      value: 0.6744248225015879
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -141,10 +141,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: None
+      value: 0.6836403033586133
     - name: F1 (macro)
       type: f1_macro
-      value: None
+      value: 0.6776792144071253
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -155,10 +155,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: None
+      value: 0.9563191208179731
     - name: F1 (macro)
       type: f1_macro
-      value: None
+      value: 0.8663013754934635
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -169,10 +169,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: None
+      value: 0.9041052961454089
     - name: F1 (macro)
       type: f1_macro
-      value: None
+      value: 0.9040831832304929
 
 ---
 # relbert/relbert-roberta-large-nce-d-semeval2012
@@ -180,22 +180,22 @@ model-index:
 RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.6497326203208557 
-    - Accuracy on SAT: 0.6528189910979229
-    - Accuracy on BATS: 0.8265703168426903
-    - Accuracy on U2: 0.6359649122807017
-    - Accuracy on U4: 0.6064814814814815
-    - Accuracy on Google: 0.934
-    - Accuracy on ConceptNet Analogy: 0.43288590604026844
-    - Accuracy on T-Rex Analogy: 0.6120218579234973
+    - Accuracy on SAT (full): 0.732620320855615 
+    - Accuracy on SAT: 0.7359050445103857
+    - Accuracy on BATS: 0.8093385214007782
+    - Accuracy on U2: 0.6754385964912281
+    - Accuracy on U4: 0.6296296296296297
+    - Accuracy on Google: 0.952
+    - Accuracy on ConceptNet Analogy: 0.4748322147651007
+    - Accuracy on T-Rex Analogy: 0.644808743169399
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/classification.json)):
-    - Micro F1 score on BLESS: None
-    - Micro F1 score on CogALexV: None
-    - Micro F1 score on EVALution: None
-    - Micro F1 score on K&H+N: None
-    - Micro F1 score on ROOT09: None
+    - Micro F1 score on BLESS: 0.9199939731806539
+    - Micro F1 score on CogALexV: 0.8497652582159625
+    - Micro F1 score on EVALution: 0.6836403033586133
+    - Micro F1 score on K&H+N: 0.9563191208179731
+    - Micro F1 score on ROOT09: 0.9041052961454089
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.7419444444444444 
+    - Accuracy on Relation Mapping: 0.8049007936507937 
 
 
 ### Usage
@@ -227,7 +227,7 @@ vector = model.get_embedding(['Tokyo', 'Japan'])  # shape of (n_dim, )
  - split_valid: validation
  - loss_function: nce
  - classification_loss: False
- - loss_function_config: {'temperature': 0.05, 'num_negative': 400, 'num_positive': 10}
+ - loss_function_config: {'temperature': 0.05, 'gradient_accumulation': 1, 'num_negative': 400, 'num_positive': 10}
  - augment_negative_by_positive: True
 
 See the full configuration at [config file](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/finetuning_config.json).

analogy.bidirection.json

@@ -1 +1 @@
-{"sat_full/test": 0.6818181818181818, "sat/test": 0.6884272997032641, "u2/test": 0.6447368421052632, "u4/test": 0.6597222222222222, "google/test": 0.936, "bats/test": 0.8132295719844358, "t_rex_relational_similarity/test": 0.644808743169399, "conceptnet_relational_similarity/test": 0.42449664429530204, "sat/validation": 0.6216216216216216, "u2/validation": 0.5833333333333334, "u4/validation": 0.625, "google/validation": 1.0, "bats/validation": 0.864321608040201, "semeval2012_relational_similarity/validation": 0.6582278481012658, "t_rex_relational_similarity/validation": 0.28024193548387094, "conceptnet_relational_similarity/validation": 0.3462230215827338}
+{"sat_full/test": 0.7272727272727273, "sat/test": 0.7299703264094956, "u2/test": 0.7149122807017544, "u4/test": 0.6875, "google/test": 0.962, "bats/test": 0.8354641467481935, "t_rex_relational_similarity/test": 0.644808743169399, "conceptnet_relational_similarity/test": 0.4672818791946309, "sat/validation": 0.7027027027027027, "u2/validation": 0.5833333333333334, "u4/validation": 0.5833333333333334, "google/validation": 1.0, "bats/validation": 0.8793969849246231, "semeval2012_relational_similarity/validation": 0.7341772151898734, "t_rex_relational_similarity/validation": 0.27419354838709675, "conceptnet_relational_similarity/validation": 0.38219424460431656}

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.7088607594936709, "sat_full/test": 0.6497326203208557, "sat/test": 0.6528189910979229, "u2/test": 0.6359649122807017, "u4/test": 0.6064814814814815, "google/test": 0.934, "bats/test": 0.8265703168426903, "t_rex_relational_similarity/test": 0.6120218579234973, "conceptnet_relational_similarity/test": 0.43288590604026844, "sat/validation": 0.6216216216216216, "u2/validation": 0.5833333333333334, "u4/validation": 0.625, "google/validation": 0.96, "bats/validation": 0.8542713567839196, "t_rex_relational_similarity/validation": 0.2762096774193548, "conceptnet_relational_similarity/validation": 0.36960431654676257}
+{"semeval2012_relational_similarity/validation": 0.7468354430379747, "sat_full/test": 0.732620320855615, "sat/test": 0.7359050445103857, "u2/test": 0.6754385964912281, "u4/test": 0.6296296296296297, "google/test": 0.952, "bats/test": 0.8093385214007782, "t_rex_relational_similarity/test": 0.644808743169399, "conceptnet_relational_similarity/test": 0.4748322147651007, "sat/validation": 0.7027027027027027, "u2/validation": 0.625, "u4/validation": 0.5625, "google/validation": 1.0, "bats/validation": 0.8542713567839196, "t_rex_relational_similarity/validation": 0.29435483870967744, "conceptnet_relational_similarity/validation": 0.37859712230215825}

analogy.reverse.json

@@ -1 +1 @@
-{"sat_full/test": 0.6363636363636364, "sat/test": 0.6468842729970327, "u2/test": 0.6491228070175439, "u4/test": 0.6597222222222222, "google/test": 0.934, "bats/test": 0.754863813229572, "t_rex_relational_similarity/test": 0.5409836065573771, "conceptnet_relational_similarity/test": 0.3347315436241611, "sat/validation": 0.5405405405405406, "u2/validation": 0.5833333333333334, "u4/validation": 0.625, "google/validation": 0.98, "bats/validation": 0.7989949748743719, "semeval2012_relational_similarity/validation": 0.6329113924050633, "t_rex_relational_similarity/validation": 0.2318548387096774, "conceptnet_relational_similarity/validation": 0.256294964028777}
+{"sat_full/test": 0.6524064171122995, "sat/test": 0.6468842729970327, "u2/test": 0.6885964912280702, "u4/test": 0.6597222222222222, "google/test": 0.944, "bats/test": 0.7976653696498055, "t_rex_relational_similarity/test": 0.5956284153005464, "conceptnet_relational_similarity/test": 0.40604026845637586, "sat/validation": 0.7027027027027027, "u2/validation": 0.7083333333333334, "u4/validation": 0.625, "google/validation": 0.98, "bats/validation": 0.8592964824120602, "semeval2012_relational_similarity/validation": 0.6708860759493671, "t_rex_relational_similarity/validation": 0.24596774193548387, "conceptnet_relational_similarity/validation": 0.3237410071942446}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "relbert_output/ckpt/nce_semeval2012/template-d/epoch_9",
+  "_name_or_path": "roberta-large",
   "architectures": [
     "RobertaModel"
   ],

finetuning_config.json

@@ -1,5 +1,5 @@
 {
-  "template": "Today, I finally discovered the relation between <subj> and <obj> : <mask>",
+  "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>",
   "model": "roberta-large",
   "max_length": 64,
   "epoch": 10,
@@ -17,6 +17,7 @@
   "classification_loss": false,
   "loss_function_config": {
     "temperature": 0.05,
+    "gradient_accumulation": 1,
     "num_negative": 400,
     "num_positive": 10
   },

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "relbert_output/ckpt/nce_semeval2012/template-d/epoch_9",
+  "name_or_path": "roberta-large",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,