model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.819484126984127
+      value: 0.8049007936507937
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6497326203208557
+      value: 0.732620320855615
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.658753709198813
+      value: 0.7359050445103857
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.735964424680378
+      value: 0.8093385214007782
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.896
+      value: 0.952
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.5570175438596491
+      value: 0.6754385964912281
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6203703703703703
+      value: 0.6296296296296297
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.41359060402684567
+      value: 0.4748322147651007
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6229508196721312
+      value: 0.644808743169399
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -113,10 +113,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.909899050775953
+      value: 0.9199939731806539
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9060888467687004
+      value: 0.9173175984713615
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -127,10 +127,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8617370892018781
+      value: 0.8497652582159625
     - name: F1 (macro)
       type: f1_macro
-      value: 0.704064732022559
+      value: 0.6744248225015879
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -141,10 +141,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6917659804983749
+      value: 0.6836403033586133
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6833995231298724
+      value: 0.6776792144071253
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -155,10 +155,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9581971204006399
+      value: 0.9563191208179731
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8741899049737119
+      value: 0.8663013754934635
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -169,10 +169,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9015982450642431
+      value: 0.9041052961454089
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9006321541459927
+      value: 0.9040831832304929
 
 ---
 # relbert/relbert-roberta-large-nce-d-semeval2012
@@ -180,22 +180,22 @@ model-index:
 RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.6497326203208557 
-    - Accuracy on SAT: 0.658753709198813
-    - Accuracy on BATS: 0.735964424680378
-    - Accuracy on U2: 0.5570175438596491
-    - Accuracy on U4: 0.6203703703703703
-    - Accuracy on Google: 0.896
-    - Accuracy on ConceptNet Analogy: 0.41359060402684567
-    - Accuracy on T-Rex Analogy: 0.6229508196721312
+    - Accuracy on SAT (full): 0.732620320855615 
+    - Accuracy on SAT: 0.7359050445103857
+    - Accuracy on BATS: 0.8093385214007782
+    - Accuracy on U2: 0.6754385964912281
+    - Accuracy on U4: 0.6296296296296297
+    - Accuracy on Google: 0.952
+    - Accuracy on ConceptNet Analogy: 0.4748322147651007
+    - Accuracy on T-Rex Analogy: 0.644808743169399
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.909899050775953
-    - Micro F1 score on CogALexV: 0.8617370892018781
-    - Micro F1 score on EVALution: 0.6917659804983749
-    - Micro F1 score on K&H+N: 0.9581971204006399
-    - Micro F1 score on ROOT09: 0.9015982450642431
+    - Micro F1 score on BLESS: 0.9199939731806539
+    - Micro F1 score on CogALexV: 0.8497652582159625
+    - Micro F1 score on EVALution: 0.6836403033586133
+    - Micro F1 score on K&H+N: 0.9563191208179731
+    - Micro F1 score on ROOT09: 0.9041052961454089
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.819484126984127 
+    - Accuracy on Relation Mapping: 0.8049007936507937 
 
 
 ### Usage
@@ -227,7 +227,7 @@ vector = model.get_embedding(['Tokyo', 'Japan'])  # shape of (n_dim, )
  - split_valid: validation
  - loss_function: nce
  - classification_loss: False
- - loss_function_config: {'temperature': 0.05, 'num_negative': 400, 'num_positive': 10}
+ - loss_function_config: {'temperature': 0.05, 'gradient_accumulation': 1, 'num_negative': 400, 'num_positive': 10}
  - augment_negative_by_positive: True
 
 See the full configuration at [config file](https://huggingface.co/relbert/relbert-roberta-large-nce-d-semeval2012/raw/main/finetuning_config.json).

analogy.bidirection.json

@@ -1 +1 @@
-{"sat_full/test": 0.6443850267379679, "sat/test": 0.6528189910979229, "u2/test": 0.5614035087719298, "u4/test": 0.6273148148148148, "google/test": 0.902, "bats/test": 0.7431906614785992, "t_rex_relational_similarity/test": 0.6885245901639344, "conceptnet_relational_similarity/test": 0.436241610738255, "sat/validation": 0.5675675675675675, "u2/validation": 0.7083333333333334, "u4/validation": 0.5208333333333334, "google/validation": 0.98, "bats/validation": 0.7587939698492462, "semeval2012_relational_similarity/validation": 0.7088607594936709, "t_rex_relational_similarity/validation": 0.2600806451612903, "conceptnet_relational_similarity/validation": 0.37050359712230213}
+{"sat_full/test": 0.7272727272727273, "sat/test": 0.7299703264094956, "u2/test": 0.7149122807017544, "u4/test": 0.6875, "google/test": 0.962, "bats/test": 0.8354641467481935, "t_rex_relational_similarity/test": 0.644808743169399, "conceptnet_relational_similarity/test": 0.4672818791946309, "sat/validation": 0.7027027027027027, "u2/validation": 0.5833333333333334, "u4/validation": 0.5833333333333334, "google/validation": 1.0, "bats/validation": 0.8793969849246231, "semeval2012_relational_similarity/validation": 0.7341772151898734, "t_rex_relational_similarity/validation": 0.27419354838709675, "conceptnet_relational_similarity/validation": 0.38219424460431656}

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.7088607594936709, "sat_full/test": 0.6497326203208557, "sat/test": 0.658753709198813, "u2/test": 0.5570175438596491, "u4/test": 0.6203703703703703, "google/test": 0.896, "bats/test": 0.735964424680378, "t_rex_relational_similarity/test": 0.6229508196721312, "conceptnet_relational_similarity/test": 0.41359060402684567, "sat/validation": 0.5675675675675675, "u2/validation": 0.625, "u4/validation": 0.5208333333333334, "google/validation": 0.98, "bats/validation": 0.7587939698492462, "t_rex_relational_similarity/validation": 0.2560483870967742, "conceptnet_relational_similarity/validation": 0.34802158273381295}
+{"semeval2012_relational_similarity/validation": 0.7468354430379747, "sat_full/test": 0.732620320855615, "sat/test": 0.7359050445103857, "u2/test": 0.6754385964912281, "u4/test": 0.6296296296296297, "google/test": 0.952, "bats/test": 0.8093385214007782, "t_rex_relational_similarity/test": 0.644808743169399, "conceptnet_relational_similarity/test": 0.4748322147651007, "sat/validation": 0.7027027027027027, "u2/validation": 0.625, "u4/validation": 0.5625, "google/validation": 1.0, "bats/validation": 0.8542713567839196, "t_rex_relational_similarity/validation": 0.29435483870967744, "conceptnet_relational_similarity/validation": 0.37859712230215825}

analogy.reverse.json

@@ -1 +1 @@
-{"sat_full/test": 0.6176470588235294, "sat/test": 0.6231454005934718, "u2/test": 0.5131578947368421, "u4/test": 0.5972222222222222, "google/test": 0.882, "bats/test": 0.708171206225681, "t_rex_relational_similarity/test": 0.6885245901639344, "conceptnet_relational_similarity/test": 0.39429530201342283, "sat/validation": 0.5675675675675675, "u2/validation": 0.6666666666666666, "u4/validation": 0.5833333333333334, "google/validation": 0.96, "bats/validation": 0.7638190954773869, "semeval2012_relational_similarity/validation": 0.5822784810126582, "t_rex_relational_similarity/validation": 0.25201612903225806, "conceptnet_relational_similarity/validation": 0.31654676258992803}
+{"sat_full/test": 0.6524064171122995, "sat/test": 0.6468842729970327, "u2/test": 0.6885964912280702, "u4/test": 0.6597222222222222, "google/test": 0.944, "bats/test": 0.7976653696498055, "t_rex_relational_similarity/test": 0.5956284153005464, "conceptnet_relational_similarity/test": 0.40604026845637586, "sat/validation": 0.7027027027027027, "u2/validation": 0.7083333333333334, "u4/validation": 0.625, "google/validation": 0.98, "bats/validation": 0.8592964824120602, "semeval2012_relational_similarity/validation": 0.6708860759493671, "t_rex_relational_similarity/validation": 0.24596774193548387, "conceptnet_relational_similarity/validation": 0.3237410071942446}

classification.json

@@ -1 +1 @@
-{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.909899050775953, "test/f1_macro": 0.9060888467687004, "test/f1_micro": 0.909899050775953, "test/p_macro": 0.8943391404664559, "test/p_micro": 0.909899050775953, "test/r_macro": 0.9193101184161824, "test/r_micro": 0.909899050775953}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.861737089201878, "test/f1_macro": 0.704064732022559, "test/f1_micro": 0.8617370892018781, "test/p_macro": 0.7373270203394411, "test/p_micro": 0.861737089201878, "test/r_macro": 0.6772908912463073, "test/r_micro": 0.861737089201878}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6917659804983749, "test/f1_macro": 0.6833995231298724, "test/f1_micro": 0.6917659804983749, "test/p_macro": 0.6820705742395324, "test/p_micro": 0.6917659804983749, "test/r_macro": 0.6863826843303636, "test/r_micro": 0.6917659804983749}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9581971204006399, "test/f1_macro": 0.8741899049737119, "test/f1_micro": 0.9581971204006399, "test/p_macro": 0.8670115001878431, "test/p_micro": 0.9581971204006399, "test/r_macro": 0.883168446080368, "test/r_micro": 0.9581971204006399}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9015982450642431, "test/f1_macro": 0.9006321541459927, "test/f1_micro": 0.9015982450642431, "test/p_macro": 0.9002161322813497, "test/p_micro": 0.9015982450642431, "test/r_macro": 0.9011948768823131, "test/r_micro": 0.9015982450642431}}
+{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9199939731806539, "test/f1_macro": 0.9173175984713615, "test/f1_micro": 0.9199939731806539, "test/p_macro": 0.9129525907822765, "test/p_micro": 0.9199939731806539, "test/r_macro": 0.9231397069650421, "test/r_micro": 0.9199939731806539}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8497652582159625, "test/f1_macro": 0.6744248225015879, "test/f1_micro": 0.8497652582159625, "test/p_macro": 0.7124501607954181, "test/p_micro": 0.8497652582159625, "test/r_macro": 0.6439012185599183, "test/r_micro": 0.8497652582159625}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6836403033586133, "test/f1_macro": 0.6776792144071253, "test/f1_micro": 0.6836403033586133, "test/p_macro": 0.6859955689335163, "test/p_micro": 0.6836403033586133, "test/r_macro": 0.6723026043339869, "test/r_micro": 0.6836403033586133}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9563191208179731, "test/f1_macro": 0.8663013754934635, "test/f1_micro": 0.9563191208179731, "test/p_macro": 0.8794831771565157, "test/p_micro": 0.9563191208179731, "test/r_macro": 0.8543598117081819, "test/r_micro": 0.9563191208179731}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9041052961454089, "test/f1_macro": 0.9040831832304929, "test/f1_micro": 0.9041052961454089, "test/p_macro": 0.9025403374928938, "test/p_micro": 0.9041052961454089, "test/r_macro": 0.9060703873054115, "test/r_micro": 0.9041052961454089}}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "relbert_output/ckpt/nce_semeval2012/template-d/epoch_9",
+  "_name_or_path": "roberta-large",
   "architectures": [
     "RobertaModel"
   ],

finetuning_config.json

@@ -1,5 +1,5 @@
 {
-  "template": "Today, I finally discovered the relation between <subj> and <obj> : <subj> is the <mask> of <obj>",
+  "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>",
   "model": "roberta-large",
   "max_length": 64,
   "epoch": 10,
@@ -17,6 +17,7 @@
   "classification_loss": false,
   "loss_function_config": {
     "temperature": 0.05,
+    "gradient_accumulation": 1,
     "num_negative": 400,
     "num_positive": 10
   },

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "relbert_output/ckpt/nce_semeval2012/template-d/epoch_9",
+  "name_or_path": "roberta-large",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,