model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.8043055555555555
+      value: 0.788234126984127
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6577540106951871
+      value: 0.6684491978609626
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6468842729970327
+      value: 0.655786350148368
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7581989994441356
+      value: 0.7943301834352418
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.914
+      value: 0.948
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6228070175438597
+      value: 0.6140350877192983
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6064814814814815
+      value: 0.6134259259259259
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.4437919463087248
+      value: 0.4463087248322148
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.5956284153005464
+      value: 0.5901639344262295
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -113,10 +113,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9139671538345638
+      value: 0.9162272110893476
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9086682025661691
+      value: 0.9126691400768508
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -127,10 +127,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8495305164319249
+      value: 0.8532863849765259
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6825854190585644
+      value: 0.6881964823343752
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -141,10 +141,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6874322860238353
+      value: 0.6912242686890574
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6766038659960685
+      value: 0.6779372223928826
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -155,10 +155,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9489462335675036
+      value: 0.9558322320372817
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8595801390910651
+      value: 0.8723486583200801
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -169,10 +169,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9003447195236602
+      value: 0.898464431212786
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8976445914777993
+      value: 0.8978368087670114
 
 ---
 # relbert/relbert-roberta-large-nce-a-semeval2012
@@ -180,22 +180,22 @@ model-index:
 RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-a-semeval2012/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.6577540106951871 
-    - Accuracy on SAT: 0.6468842729970327
-    - Accuracy on BATS: 0.7581989994441356
-    - Accuracy on U2: 0.6228070175438597
-    - Accuracy on U4: 0.6064814814814815
-    - Accuracy on Google: 0.914
-    - Accuracy on ConceptNet Analogy: 0.4437919463087248
-    - Accuracy on T-Rex Analogy: 0.5956284153005464
+    - Accuracy on SAT (full): 0.6684491978609626 
+    - Accuracy on SAT: 0.655786350148368
+    - Accuracy on BATS: 0.7943301834352418
+    - Accuracy on U2: 0.6140350877192983
+    - Accuracy on U4: 0.6134259259259259
+    - Accuracy on Google: 0.948
+    - Accuracy on ConceptNet Analogy: 0.4463087248322148
+    - Accuracy on T-Rex Analogy: 0.5901639344262295
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-a-semeval2012/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.9139671538345638
-    - Micro F1 score on CogALexV: 0.8495305164319249
-    - Micro F1 score on EVALution: 0.6874322860238353
-    - Micro F1 score on K&H+N: 0.9489462335675036
-    - Micro F1 score on ROOT09: 0.9003447195236602
+    - Micro F1 score on BLESS: 0.9162272110893476
+    - Micro F1 score on CogALexV: 0.8532863849765259
+    - Micro F1 score on EVALution: 0.6912242686890574
+    - Micro F1 score on K&H+N: 0.9558322320372817
+    - Micro F1 score on ROOT09: 0.898464431212786
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-a-semeval2012/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.8043055555555555 
+    - Accuracy on Relation Mapping: 0.788234126984127 
 
 
 ### Usage

analogy.bidirection.json

@@ -1 +1 @@
-{"sat_full/test": 0.6711229946524064, "sat/test": 0.658753709198813, "u2/test": 0.6228070175438597, "u4/test": 0.6319444444444444, "google/test": 0.934, "bats/test": 0.7709838799332963, "t_rex_relational_similarity/test": 0.6284153005464481, "conceptnet_relational_similarity/test": 0.46308724832214765, "sat/validation": 0.7837837837837838, "u2/validation": 0.6666666666666666, "u4/validation": 0.6458333333333334, "google/validation": 0.98, "bats/validation": 0.8291457286432161, "semeval2012_relational_similarity/validation": 0.7088607594936709, "t_rex_relational_similarity/validation": 0.2560483870967742, "conceptnet_relational_similarity/validation": 0.368705035971223}
+{"sat_full/test": 0.6711229946524064, "sat/test": 0.6646884272997032, "u2/test": 0.6228070175438597, "u4/test": 0.6550925925925926, "google/test": 0.962, "bats/test": 0.7904391328515842, "t_rex_relational_similarity/test": 0.6338797814207651, "conceptnet_relational_similarity/test": 0.4446308724832215, "sat/validation": 0.7297297297297297, "u2/validation": 0.6666666666666666, "u4/validation": 0.5833333333333334, "google/validation": 1.0, "bats/validation": 0.8391959798994975, "semeval2012_relational_similarity/validation": 0.7341772151898734, "t_rex_relational_similarity/validation": 0.2661290322580645, "conceptnet_relational_similarity/validation": 0.37949640287769787}

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.759493670886076, "sat_full/test": 0.6577540106951871, "sat/test": 0.6468842729970327, "u2/test": 0.6228070175438597, "u4/test": 0.6064814814814815, "google/test": 0.914, "bats/test": 0.7581989994441356, "t_rex_relational_similarity/test": 0.5956284153005464, "conceptnet_relational_similarity/test": 0.4437919463087248, "sat/validation": 0.7567567567567568, "u2/validation": 0.5833333333333334, "u4/validation": 0.5625, "google/validation": 0.98, "bats/validation": 0.8391959798994975, "t_rex_relational_similarity/validation": 0.22580645161290322, "conceptnet_relational_similarity/validation": 0.35251798561151076}
+{"semeval2012_relational_similarity/validation": 0.7341772151898734, "sat_full/test": 0.6684491978609626, "sat/test": 0.655786350148368, "u2/test": 0.6140350877192983, "u4/test": 0.6134259259259259, "google/test": 0.948, "bats/test": 0.7943301834352418, "t_rex_relational_similarity/test": 0.5901639344262295, "conceptnet_relational_similarity/test": 0.4463087248322148, "sat/validation": 0.7837837837837838, "u2/validation": 0.5416666666666666, "u4/validation": 0.5416666666666666, "google/validation": 1.0, "bats/validation": 0.8492462311557789, "t_rex_relational_similarity/validation": 0.2620967741935484, "conceptnet_relational_similarity/validation": 0.39388489208633093}

analogy.reverse.json

@@ -1 +1 @@
-{"sat_full/test": 0.6176470588235294, "sat/test": 0.6112759643916914, "u2/test": 0.6096491228070176, "u4/test": 0.5972222222222222, "google/test": 0.922, "bats/test": 0.7287381878821567, "t_rex_relational_similarity/test": 0.5956284153005464, "conceptnet_relational_similarity/test": 0.4085570469798658, "sat/validation": 0.6756756756756757, "u2/validation": 0.6666666666666666, "u4/validation": 0.5833333333333334, "google/validation": 0.96, "bats/validation": 0.7839195979899497, "semeval2012_relational_similarity/validation": 0.6962025316455697, "t_rex_relational_similarity/validation": 0.24596774193548387, "conceptnet_relational_similarity/validation": 0.31384892086330934}
+{"sat_full/test": 0.6149732620320856, "sat/test": 0.6053412462908012, "u2/test": 0.6008771929824561, "u4/test": 0.6296296296296297, "google/test": 0.952, "bats/test": 0.7470817120622568, "t_rex_relational_similarity/test": 0.6229508196721312, "conceptnet_relational_similarity/test": 0.39932885906040266, "sat/validation": 0.7027027027027027, "u2/validation": 0.625, "u4/validation": 0.5833333333333334, "google/validation": 0.98, "bats/validation": 0.8140703517587939, "semeval2012_relational_similarity/validation": 0.6708860759493671, "t_rex_relational_similarity/validation": 0.25, "conceptnet_relational_similarity/validation": 0.3318345323741007}

classification.json

@@ -1 +1 @@
-{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9139671538345638, "test/f1_macro": 0.9086682025661691, "test/f1_micro": 0.9139671538345638, "test/p_macro": 0.9072703871807176, "test/p_micro": 0.9139671538345638, "test/r_macro": 0.9103957441016443, "test/r_micro": 0.9139671538345638}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8495305164319249, "test/f1_macro": 0.6825854190585644, "test/f1_micro": 0.8495305164319249, "test/p_macro": 0.713833404738869, "test/p_micro": 0.8495305164319249, "test/r_macro": 0.6582651578742065, "test/r_micro": 0.8495305164319249}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6874322860238353, "test/f1_macro": 0.6766038659960685, "test/f1_micro": 0.6874322860238353, "test/p_macro": 0.677756510883655, "test/p_micro": 0.6874322860238353, "test/r_macro": 0.6770380770682676, "test/r_micro": 0.6874322860238353}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9489462335675036, "test/f1_macro": 0.8595801390910651, "test/f1_micro": 0.9489462335675036, "test/p_macro": 0.8728913872732812, "test/p_micro": 0.9489462335675036, "test/r_macro": 0.8477758803228803, "test/r_micro": 0.9489462335675036}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9003447195236602, "test/f1_macro": 0.8976445914777993, "test/f1_micro": 0.9003447195236602, "test/p_macro": 0.9000751524652514, "test/p_micro": 0.9003447195236602, "test/r_macro": 0.8955420485369855, "test/r_micro": 0.9003447195236602}}
+{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9162272110893476, "test/f1_macro": 0.9126691400768508, "test/f1_micro": 0.9162272110893476, "test/p_macro": 0.9121763629267451, "test/p_micro": 0.9162272110893476, "test/r_macro": 0.9135529572876576, "test/r_micro": 0.9162272110893476}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8532863849765259, "test/f1_macro": 0.6881964823343752, "test/f1_micro": 0.8532863849765259, "test/p_macro": 0.7255568039986454, "test/p_micro": 0.8532863849765259, "test/r_macro": 0.6588623821958979, "test/r_micro": 0.8532863849765259}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6912242686890574, "test/f1_macro": 0.6779372223928826, "test/f1_micro": 0.6912242686890574, "test/p_macro": 0.6865626828233385, "test/p_micro": 0.6912242686890574, "test/r_macro": 0.6726722427005545, "test/r_micro": 0.6912242686890574}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9558322320372817, "test/f1_macro": 0.8723486583200801, "test/f1_micro": 0.9558322320372817, "test/p_macro": 0.9062745443993847, "test/p_micro": 0.9558322320372817, "test/r_macro": 0.846521674633046, "test/r_micro": 0.9558322320372817}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.898464431212786, "test/f1_macro": 0.8978368087670114, "test/f1_micro": 0.898464431212786, "test/p_macro": 0.895948162376852, "test/p_micro": 0.898464431212786, "test/r_macro": 0.8998170610834952, "test/r_micro": 0.898464431212786}}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "relbert_output/ckpt/nce_semeval2012/template-a/epoch_8",
+  "_name_or_path": "roberta-large",
   "architectures": [
     "RobertaModel"
   ],

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "relbert_output/ckpt/nce_semeval2012/template-a/epoch_8",
+  "name_or_path": "roberta-large",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,