model update

README.md

@@ -103,6 +103,17 @@ model-index:
     - name: Accuracy
       type: accuracy
       value: 0.5081967213114754
+  - task:
+      name: Analogy Questions (NELL-ONE Analogy)
+      type: multiple-choice-qa
+    dataset:
+      name: NELL-ONE Analogy
+      args: relbert/analogy_questions
+      type: analogy-questions
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.61
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -188,6 +199,7 @@ This model achieves the following results on the relation understanding tasks:
     - Accuracy on Google: 0.934
     - Accuracy on ConceptNet Analogy: 0.3674496644295302
     - Accuracy on T-Rex Analogy: 0.5081967213114754
+    - Accuracy on NELL-ONE Analogy: 0.61
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-e-semeval2012/raw/main/classification.json)):
     - Micro F1 score on BLESS: 0.9213500075335241
     - Micro F1 score on CogALexV: 0.8551643192488263

analogy.bidirection.json

@@ -1 +1 @@
-{"sat_full/test": 0.660427807486631, "sat/test": 0.6646884272997032, "u2/test": 0.6885964912280702, "u4/test": 0.6620370370370371, "google/test": 0.954, "bats/test": 0.8104502501389661, "t_rex_relational_similarity/test": 0.6010928961748634, "conceptnet_relational_similarity/test": 0.4312080536912752, "sat/validation": 0.6216216216216216, "u2/validation": 0.6666666666666666, "u4/validation": 0.625, "google/validation": 0.96, "bats/validation": 0.8592964824120602, "semeval2012_relational_similarity/validation": 0.6962025316455697, "t_rex_relational_similarity/validation": 0.26814516129032256, "conceptnet_relational_similarity/validation": 0.36241007194244607}
+{"sat_full/test": 0.660427807486631, "sat/test": 0.6646884272997032, "u2/test": 0.6885964912280702, "u4/test": 0.6620370370370371, "google/test": 0.954, "bats/test": 0.8104502501389661, "t_rex_relational_similarity/test": 0.6010928961748634, "conceptnet_relational_similarity/test": 0.4312080536912752, "sat/validation": 0.6216216216216216, "u2/validation": 0.6666666666666666, "u4/validation": 0.625, "google/validation": 0.96, "bats/validation": 0.8592964824120602, "semeval2012_relational_similarity/validation": 0.6962025316455697, "t_rex_relational_similarity/validation": 0.26814516129032256, "conceptnet_relational_similarity/validation": 0.36241007194244607, "nell_relational_similarity/test": 0.6483333333333333, "nell_relational_similarity/validation": 0.5175}

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.7341772151898734, "sat_full/test": 0.6203208556149733, "sat/test": 0.6261127596439169, "u2/test": 0.6403508771929824, "u4/test": 0.6157407407407407, "google/test": 0.934, "bats/test": 0.7787659811006115, "t_rex_relational_similarity/test": 0.5081967213114754, "conceptnet_relational_similarity/test": 0.3674496644295302, "sat/validation": 0.5675675675675675, "u2/validation": 0.625, "u4/validation": 0.5833333333333334, "google/validation": 1.0, "bats/validation": 0.8241206030150754, "t_rex_relational_similarity/validation": 0.2399193548387097, "conceptnet_relational_similarity/validation": 0.3183453237410072}
+{"semeval2012_relational_similarity/validation": 0.7341772151898734, "sat_full/test": 0.6203208556149733, "sat/test": 0.6261127596439169, "u2/test": 0.6403508771929824, "u4/test": 0.6157407407407407, "google/test": 0.934, "bats/test": 0.7787659811006115, "t_rex_relational_similarity/test": 0.5081967213114754, "conceptnet_relational_similarity/test": 0.3674496644295302, "sat/validation": 0.5675675675675675, "u2/validation": 0.625, "u4/validation": 0.5833333333333334, "google/validation": 1.0, "bats/validation": 0.8241206030150754, "t_rex_relational_similarity/validation": 0.2399193548387097, "conceptnet_relational_similarity/validation": 0.3183453237410072, "nell_relational_similarity/test": 0.61, "nell_relational_similarity/validation": 0.46}

analogy.reverse.json

@@ -1 +1 @@
-{"sat_full/test": 0.6550802139037433, "sat/test": 0.6646884272997032, "u2/test": 0.6754385964912281, "u4/test": 0.6527777777777778, "google/test": 0.946, "bats/test": 0.773763201778766, "t_rex_relational_similarity/test": 0.5956284153005464, "conceptnet_relational_similarity/test": 0.4186241610738255, "sat/validation": 0.5675675675675675, "u2/validation": 0.5833333333333334, "u4/validation": 0.5833333333333334, "google/validation": 0.92, "bats/validation": 0.8341708542713567, "semeval2012_relational_similarity/validation": 0.6582278481012658, "t_rex_relational_similarity/validation": 0.2661290322580645, "conceptnet_relational_similarity/validation": 0.32014388489208634}
+{"sat_full/test": 0.6550802139037433, "sat/test": 0.6646884272997032, "u2/test": 0.6754385964912281, "u4/test": 0.6527777777777778, "google/test": 0.946, "bats/test": 0.773763201778766, "t_rex_relational_similarity/test": 0.5956284153005464, "conceptnet_relational_similarity/test": 0.4186241610738255, "sat/validation": 0.5675675675675675, "u2/validation": 0.5833333333333334, "u4/validation": 0.5833333333333334, "google/validation": 0.92, "bats/validation": 0.8341708542713567, "semeval2012_relational_similarity/validation": 0.6582278481012658, "t_rex_relational_similarity/validation": 0.2661290322580645, "conceptnet_relational_similarity/validation": 0.32014388489208634, "nell_relational_similarity/test": 0.6483333333333333, "nell_relational_similarity/validation": 0.5375}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "relbert_output/ckpt/nce_semeval2012/template-e/epoch_9",
+  "_name_or_path": "roberta-large",
   "architectures": [
     "RobertaModel"
   ],

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "relbert_output/ckpt/nce_semeval2012/template-e/epoch_9",
+  "name_or_path": "roberta-large",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,