asahi417 commited on
Commit
9c43eb6
1 Parent(s): 743f77d

model update

Browse files
README.md ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ datasets:
3
+ - relbert/semeval2012_relational_similarity
4
+ model-index:
5
+ - name: relbert/relbert-roberta-large-triplet-e-semeval2012
6
+ results:
7
+ - task:
8
+ name: Relation Mapping
9
+ type: sorting-task
10
+ dataset:
11
+ name: Relation Mapping
12
+ args: relbert/relation_mapping
13
+ type: relation-mapping
14
+ metrics:
15
+ - name: Accuracy
16
+ type: accuracy
17
+ value: 0.8003373015873015
18
+ - task:
19
+ name: Analogy Questions (SAT full)
20
+ type: multiple-choice-qa
21
+ dataset:
22
+ name: SAT full
23
+ args: relbert/analogy_questions
24
+ type: analogy-questions
25
+ metrics:
26
+ - name: Accuracy
27
+ type: accuracy
28
+ value: 0.56951871657754
29
+ - task:
30
+ name: Analogy Questions (SAT)
31
+ type: multiple-choice-qa
32
+ dataset:
33
+ name: SAT
34
+ args: relbert/analogy_questions
35
+ type: analogy-questions
36
+ metrics:
37
+ - name: Accuracy
38
+ type: accuracy
39
+ value: 0.5786350148367952
40
+ - task:
41
+ name: Analogy Questions (BATS)
42
+ type: multiple-choice-qa
43
+ dataset:
44
+ name: BATS
45
+ args: relbert/analogy_questions
46
+ type: analogy-questions
47
+ metrics:
48
+ - name: Accuracy
49
+ type: accuracy
50
+ value: 0.7476375764313508
51
+ - task:
52
+ name: Analogy Questions (Google)
53
+ type: multiple-choice-qa
54
+ dataset:
55
+ name: Google
56
+ args: relbert/analogy_questions
57
+ type: analogy-questions
58
+ metrics:
59
+ - name: Accuracy
60
+ type: accuracy
61
+ value: 0.902
62
+ - task:
63
+ name: Analogy Questions (U2)
64
+ type: multiple-choice-qa
65
+ dataset:
66
+ name: U2
67
+ args: relbert/analogy_questions
68
+ type: analogy-questions
69
+ metrics:
70
+ - name: Accuracy
71
+ type: accuracy
72
+ value: 0.618421052631579
73
+ - task:
74
+ name: Analogy Questions (U4)
75
+ type: multiple-choice-qa
76
+ dataset:
77
+ name: U4
78
+ args: relbert/analogy_questions
79
+ type: analogy-questions
80
+ metrics:
81
+ - name: Accuracy
82
+ type: accuracy
83
+ value: 0.5949074074074074
84
+ - task:
85
+ name: Analogy Questions (ConceptNet Analogy)
86
+ type: multiple-choice-qa
87
+ dataset:
88
+ name: ConceptNet Analogy
89
+ args: relbert/analogy_questions
90
+ type: analogy-questions
91
+ metrics:
92
+ - name: Accuracy
93
+ type: accuracy
94
+ value: 0.5531135531135531
95
+ - task:
96
+ name: Analogy Questions (TREX Analogy)
97
+ type: multiple-choice-qa
98
+ dataset:
99
+ name: TREX Analogy
100
+ args: relbert/analogy_questions
101
+ type: analogy-questions
102
+ metrics:
103
+ - name: Accuracy
104
+ type: accuracy
105
+ value: 0.8288690476190477
106
+ - task:
107
+ name: Lexical Relation Classification (BLESS)
108
+ type: classification
109
+ dataset:
110
+ name: BLESS
111
+ args: relbert/lexical_relation_classification
112
+ type: relation-classification
113
+ metrics:
114
+ - name: F1
115
+ type: f1
116
+ value: 0.9026668675606448
117
+ - name: F1 (macro)
118
+ type: f1_macro
119
+ value: 0.901099542864961
120
+ - task:
121
+ name: Lexical Relation Classification (CogALexV)
122
+ type: classification
123
+ dataset:
124
+ name: CogALexV
125
+ args: relbert/lexical_relation_classification
126
+ type: relation-classification
127
+ metrics:
128
+ - name: F1
129
+ type: f1
130
+ value: 0.8607981220657277
131
+ - name: F1 (macro)
132
+ type: f1_macro
133
+ value: 0.703607951808097
134
+ - task:
135
+ name: Lexical Relation Classification (EVALution)
136
+ type: classification
137
+ dataset:
138
+ name: BLESS
139
+ args: relbert/lexical_relation_classification
140
+ type: relation-classification
141
+ metrics:
142
+ - name: F1
143
+ type: f1
144
+ value: 0.6917659804983749
145
+ - name: F1 (macro)
146
+ type: f1_macro
147
+ value: 0.690966207619381
148
+ - task:
149
+ name: Lexical Relation Classification (K&H+N)
150
+ type: classification
151
+ dataset:
152
+ name: K&H+N
153
+ args: relbert/lexical_relation_classification
154
+ type: relation-classification
155
+ metrics:
156
+ - name: F1
157
+ type: f1
158
+ value: 0.9673088961535786
159
+ - name: F1 (macro)
160
+ type: f1_macro
161
+ value: 0.8870352978589001
162
+ - task:
163
+ name: Lexical Relation Classification (ROOT09)
164
+ type: classification
165
+ dataset:
166
+ name: ROOT09
167
+ args: relbert/lexical_relation_classification
168
+ type: relation-classification
169
+ metrics:
170
+ - name: F1
171
+ type: f1
172
+ value: 0.8975242870573488
173
+ - name: F1 (macro)
174
+ type: f1_macro
175
+ value: 0.8979403889476659
176
+
177
+ ---
178
+ # relbert/relbert-roberta-large-triplet-e-semeval2012
179
+
180
+ RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
181
+ This model achieves the following results on the relation understanding tasks:
182
+ - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-large-triplet-e-semeval2012/raw/main/analogy.forward.json)):
183
+ - Accuracy on SAT (full): 0.56951871657754
184
+ - Accuracy on SAT: 0.5786350148367952
185
+ - Accuracy on BATS: 0.7476375764313508
186
+ - Accuracy on U2: 0.618421052631579
187
+ - Accuracy on U4: 0.5949074074074074
188
+ - Accuracy on Google: 0.902
189
+ - Accuracy on ConceptNet Analogy: 0.5531135531135531
190
+ - Accuracy on T-Rex Analogy: 0.8288690476190477
191
+ - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-large-triplet-e-semeval2012/raw/main/classification.json)):
192
+ - Micro F1 score on BLESS: 0.9026668675606448
193
+ - Micro F1 score on CogALexV: 0.8607981220657277
194
+ - Micro F1 score on EVALution: 0.6917659804983749
195
+ - Micro F1 score on K&H+N: 0.9673088961535786
196
+ - Micro F1 score on ROOT09: 0.8975242870573488
197
+ - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-large-triplet-e-semeval2012/raw/main/relation_mapping.json)):
198
+ - Accuracy on Relation Mapping: 0.8003373015873015
199
+
200
+
201
+ ### Usage
202
+ This model can be used through the [relbert library](https://github.com/asahi417/relbert). Install the library via pip
203
+ ```shell
204
+ pip install relbert
205
+ ```
206
+ and activate model as below.
207
+ ```python
208
+ from relbert import RelBERT
209
+ model = RelBERT("relbert/relbert-roberta-large-triplet-e-semeval2012")
210
+ vector = model.get_embedding(['Tokyo', 'Japan']) # shape of (n_dim, )
211
+ ```
212
+
213
+ ### Training hyperparameters
214
+
215
+ - model: roberta-large
216
+ - max_length: 64
217
+ - epoch: 1
218
+ - batch: 79
219
+ - random_seed: 0
220
+ - lr: 2e-05
221
+ - lr_warmup: 10
222
+ - aggregation_mode: average_no_mask
223
+ - data: relbert/semeval2012_relational_similarity
224
+ - data_name: None
225
+ - exclude_relation: None
226
+ - split: train
227
+ - split_valid: validation
228
+ - loss_function: triplet
229
+ - classification_loss: False
230
+ - loss_function_config: {'mse_margin': 1}
231
+ - augment_negative_by_positive: False
232
+
233
+ See the full configuration at [config file](https://huggingface.co/relbert/relbert-roberta-large-triplet-e-semeval2012/raw/main/finetuning_config.json).
234
+
235
+ ### Reference
236
+ If you use any resource from RelBERT, please consider to cite our [paper](https://aclanthology.org/2021.emnlp-main.712/).
237
+
238
+ ```
239
+
240
+ @inproceedings{ushio-etal-2021-distilling,
241
+ title = "Distilling Relation Embeddings from Pretrained Language Models",
242
+ author = "Ushio, Asahi and
243
+ Camacho-Collados, Jose and
244
+ Schockaert, Steven",
245
+ booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
246
+ month = nov,
247
+ year = "2021",
248
+ address = "Online and Punta Cana, Dominican Republic",
249
+ publisher = "Association for Computational Linguistics",
250
+ url = "https://aclanthology.org/2021.emnlp-main.712",
251
+ doi = "10.18653/v1/2021.emnlp-main.712",
252
+ pages = "9044--9062",
253
+ abstract = "Pre-trained language models have been found to capture a surprisingly rich amount of lexical knowledge, ranging from commonsense properties of everyday concepts to detailed factual knowledge about named entities. Among others, this makes it possible to distill high-quality word vectors from pre-trained language models. However, it is currently unclear to what extent it is possible to distill relation embeddings, i.e. vectors that characterize the relationship between two words. Such relation embeddings are appealing because they can, in principle, encode relational knowledge in a more fine-grained way than is possible with knowledge graphs. To obtain relation embeddings from a pre-trained language model, we encode word pairs using a (manually or automatically generated) prompt, and we fine-tune the language model such that relationally similar word pairs yield similar output vectors. We find that the resulting relation embeddings are highly competitive on analogy (unsupervised) and relation classification (supervised) benchmarks, even without any task-specific fine-tuning. Source code to reproduce our experimental results and the model checkpoints are available in the following repository: https://github.com/asahi417/relbert",
254
+ }
255
+
256
+ ```
analogy.bidirection.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"sat_full/test": 0.606951871657754, "sat/test": 0.6142433234421365, "u2/test": 0.6491228070175439, "u4/test": 0.6620370370370371, "google/test": 0.946, "bats/test": 0.7782101167315175, "t_rex_relational_similarity/test": 0.8273809523809523, "conceptnet_relational_similarity/test": 0.6043956043956044, "sat/validation": 0.5405405405405406, "u2/validation": 0.5416666666666666, "u4/validation": 0.5208333333333334, "google/validation": 0.94, "bats/validation": 0.7537688442211056, "semeval2012_relational_similarity/validation": 0.6708860759493671, "t_rex_relational_similarity/validation": 0.6944444444444444, "conceptnet_relational_similarity/validation": 0.6228070175438597}
analogy.forward.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"sat_full/test": 0.56951871657754, "sat/test": 0.5786350148367952, "u2/test": 0.618421052631579, "u4/test": 0.5949074074074074, "google/test": 0.902, "bats/test": 0.7476375764313508, "t_rex_relational_similarity/test": 0.8288690476190477, "conceptnet_relational_similarity/test": 0.5531135531135531, "sat/validation": 0.4864864864864865, "u2/validation": 0.5, "u4/validation": 0.5, "google/validation": 0.9, "bats/validation": 0.7236180904522613, "semeval2012_relational_similarity/validation": 0.6708860759493671, "t_rex_relational_similarity/validation": 0.6773504273504274, "conceptnet_relational_similarity/validation": 0.5578947368421052}
analogy.reverse.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"sat_full/test": 0.5882352941176471, "sat/test": 0.5905044510385756, "u2/test": 0.6228070175438597, "u4/test": 0.6574074074074074, "google/test": 0.946, "bats/test": 0.7531962201222901, "t_rex_relational_similarity/test": 0.7827380952380952, "conceptnet_relational_similarity/test": 0.5824175824175825, "sat/validation": 0.5675675675675675, "u2/validation": 0.5833333333333334, "u4/validation": 0.5208333333333334, "google/validation": 0.94, "bats/validation": 0.7537688442211056, "semeval2012_relational_similarity/validation": 0.7088607594936709, "t_rex_relational_similarity/validation": 0.6730769230769231, "conceptnet_relational_similarity/validation": 0.6105263157894737}
classification.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9026668675606448, "test/f1_macro": 0.901099542864961, "test/f1_micro": 0.9026668675606448, "test/p_macro": 0.8955761960950932, "test/p_micro": 0.9026668675606448, "test/r_macro": 0.9113855846729316, "test/r_micro": 0.9026668675606448}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8607981220657277, "test/f1_macro": 0.703607951808097, "test/f1_micro": 0.8607981220657277, "test/p_macro": 0.7137224286878939, "test/p_micro": 0.8607981220657277, "test/r_macro": 0.7002041201040177, "test/r_micro": 0.8607981220657277}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6917659804983749, "test/f1_macro": 0.690966207619381, "test/f1_micro": 0.6917659804983749, "test/p_macro": 0.7011015572288086, "test/p_micro": 0.6917659804983749, "test/r_macro": 0.6873799321496953, "test/r_micro": 0.6917659804983749}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9673088961535786, "test/f1_macro": 0.8870352978589001, "test/f1_micro": 0.9673088961535786, "test/p_macro": 0.9225322196903244, "test/p_micro": 0.9673088961535786, "test/r_macro": 0.8652737963321708, "test/r_micro": 0.9673088961535786}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8975242870573488, "test/f1_macro": 0.8979403889476659, "test/f1_micro": 0.8975242870573488, "test/p_macro": 0.8914686926214072, "test/p_micro": 0.8975242870573488, "test/r_macro": 0.9065267508336042, "test/r_micro": 0.8975242870573488}}
config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "_name_or_path": "relbert_output/ckpt/triplet_semeval2012/template-e/model",
3
  "architectures": [
4
  "RobertaModel"
5
  ],
 
1
  {
2
+ "_name_or_path": "roberta-large",
3
  "architectures": [
4
  "RobertaModel"
5
  ],
finetuning_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <obj> is <subj>\u2019s <mask>",
3
+ "model": "roberta-large",
4
+ "max_length": 64,
5
+ "epoch": 1,
6
+ "batch": 79,
7
+ "random_seed": 0,
8
+ "lr": 2e-05,
9
+ "lr_warmup": 10,
10
+ "aggregation_mode": "average_no_mask",
11
+ "data": "relbert/semeval2012_relational_similarity",
12
+ "data_name": null,
13
+ "exclude_relation": null,
14
+ "split": "train",
15
+ "split_valid": "validation",
16
+ "loss_function": "triplet",
17
+ "classification_loss": false,
18
+ "loss_function_config": {
19
+ "mse_margin": 1
20
+ },
21
+ "augment_negative_by_positive": false
22
+ }
relation_mapping.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -6,7 +6,7 @@
6
  "errors": "replace",
7
  "mask_token": "<mask>",
8
  "model_max_length": 512,
9
- "name_or_path": "relbert_output/ckpt/triplet_semeval2012/template-e/model",
10
  "pad_token": "<pad>",
11
  "sep_token": "</s>",
12
  "special_tokens_map_file": null,
 
6
  "errors": "replace",
7
  "mask_token": "<mask>",
8
  "model_max_length": 512,
9
+ "name_or_path": "roberta-large",
10
  "pad_token": "<pad>",
11
  "sep_token": "</s>",
12
  "special_tokens_map_file": null,