mrm8488 commited on
Commit
0879610
1 Parent(s): 325032a

Initial commit from mrm8488

Browse files
README.md ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - en
4
+ license: mit
5
+ tags:
6
+ - generated_from_trainer
7
+ datasets:
8
+ - glue
9
+ metrics:
10
+ - accuracy
11
+ - f1
12
+ model-index:
13
+ - name: deberta-v3-small
14
+ results:
15
+ - task:
16
+ name: Text Classification
17
+ type: text-classification
18
+ dataset:
19
+ name: GLUE MRPC
20
+ type: glue
21
+ args: mrpc
22
+ metrics:
23
+ - name: Accuracy
24
+ type: accuracy
25
+ value: 0.8921568627450981
26
+ - name: F1
27
+ type: f1
28
+ value: 0.9233449477351917
29
+ ---
30
+
31
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
32
+ should probably proofread and complete it, then remove this comment. -->
33
+
34
+ # deberta-v3-small
35
+
36
+ This model is a fine-tuned version of [microsoft/deberta-v3-small](https://huggingface.co/microsoft/deberta-v3-small) on the GLUE MRPC dataset.
37
+ It achieves the following results on the evaluation set:
38
+ - Loss: 0.2787
39
+ - Accuracy: 0.8922
40
+ - F1: 0.9233
41
+ - Combined Score: 0.9078
42
+
43
+ ## Model description
44
+
45
+ More information needed
46
+
47
+ ## Intended uses & limitations
48
+
49
+ More information needed
50
+
51
+ ## Training and evaluation data
52
+
53
+ More information needed
54
+
55
+ ## Training procedure
56
+
57
+ ### Training hyperparameters
58
+
59
+ The following hyperparameters were used during training:
60
+ - learning_rate: 3e-05
61
+ - train_batch_size: 16
62
+ - eval_batch_size: 8
63
+ - seed: 42
64
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
65
+ - lr_scheduler_type: linear
66
+ - num_epochs: 10.0
67
+
68
+ ### Training results
69
+
70
+ | Training Loss | Epoch | Step | Validation Loss | Accuracy | F1 | Combined Score |
71
+ |:-------------:|:-----:|:----:|:---------------:|:--------:|:------:|:--------------:|
72
+ | No log | 1.0 | 230 | 0.2787 | 0.8922 | 0.9233 | 0.9078 |
73
+ | No log | 2.0 | 460 | 0.3651 | 0.875 | 0.9137 | 0.8944 |
74
+ | No log | 3.0 | 690 | 0.5238 | 0.8799 | 0.9179 | 0.8989 |
75
+ | No log | 4.0 | 920 | 0.4712 | 0.8946 | 0.9222 | 0.9084 |
76
+ | 0.2147 | 5.0 | 1150 | 0.5704 | 0.8946 | 0.9262 | 0.9104 |
77
+ | 0.2147 | 6.0 | 1380 | 0.5697 | 0.8995 | 0.9284 | 0.9140 |
78
+ | 0.2147 | 7.0 | 1610 | 0.6651 | 0.8922 | 0.9214 | 0.9068 |
79
+ | 0.2147 | 8.0 | 1840 | 0.6726 | 0.8946 | 0.9239 | 0.9093 |
80
+ | 0.0183 | 9.0 | 2070 | 0.7250 | 0.8848 | 0.9177 | 0.9012 |
81
+ | 0.0183 | 10.0 | 2300 | 0.7093 | 0.8922 | 0.9223 | 0.9072 |
82
+
83
+
84
+ ### Framework versions
85
+
86
+ - Transformers 4.13.0.dev0
87
+ - Pytorch 1.10.0+cu111
88
+ - Datasets 1.15.1
89
+ - Tokenizers 0.10.3
added_tokens.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"[MASK]": 128000}
all_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.8921568627450981,
4
+ "eval_combined_score": 0.9077509052401449,
5
+ "eval_f1": 0.9233449477351917,
6
+ "eval_loss": 0.27871406078338623,
7
+ "eval_runtime": 1.0505,
8
+ "eval_samples": 408,
9
+ "eval_samples_per_second": 388.37,
10
+ "eval_steps_per_second": 48.546,
11
+ "train_loss": 0.1021328606294549,
12
+ "train_runtime": 294.6784,
13
+ "train_samples": 3668,
14
+ "train_samples_per_second": 124.475,
15
+ "train_steps_per_second": 7.805
16
+ }
config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "microsoft/deberta-v3-small",
3
+ "architectures": [
4
+ "DebertaV2ForSequenceClassification"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "finetuning_task": "mrpc",
8
+ "hidden_act": "gelu",
9
+ "hidden_dropout_prob": 0.1,
10
+ "hidden_size": 768,
11
+ "id2label": {
12
+ "0": "not_equivalent",
13
+ "1": "equivalent"
14
+ },
15
+ "initializer_range": 0.02,
16
+ "intermediate_size": 3072,
17
+ "label2id": {
18
+ "equivalent": 1,
19
+ "not_equivalent": 0
20
+ },
21
+ "layer_norm_eps": 1e-07,
22
+ "max_position_embeddings": 512,
23
+ "max_relative_positions": -1,
24
+ "model_type": "deberta-v2",
25
+ "norm_rel_ebd": "layer_norm",
26
+ "num_attention_heads": 12,
27
+ "num_hidden_layers": 6,
28
+ "pad_token_id": 0,
29
+ "pooler_dropout": 0,
30
+ "pooler_hidden_act": "gelu",
31
+ "pooler_hidden_size": 768,
32
+ "pos_att_type": [
33
+ "p2c",
34
+ "c2p"
35
+ ],
36
+ "position_biased_input": false,
37
+ "position_buckets": 256,
38
+ "relative_attention": true,
39
+ "share_att_key": true,
40
+ "torch_dtype": "float32",
41
+ "transformers_version": "4.13.0.dev0",
42
+ "type_vocab_size": 0,
43
+ "vocab_size": 128100
44
+ }
eval_results.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "eval_accuracy": 0.8921568627450981,
4
+ "eval_combined_score": 0.9077509052401449,
5
+ "eval_f1": 0.9233449477351917,
6
+ "eval_loss": 0.27871406078338623,
7
+ "eval_runtime": 1.0505,
8
+ "eval_samples": 408,
9
+ "eval_samples_per_second": 388.37,
10
+ "eval_steps_per_second": 48.546
11
+ }
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50c960e91d9c4b5c3d6c12708cdd685b02f8a23b9a69022c7b2b4fc77ea52940
3
+ size 567632875
runs/events.out.tfevents.1637518922.93c238760a3d.594.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:281f868339067c38862f4d194922dd1b44d115bb533d43e8d1de9443a4d8a04b
3
+ size 8397
runs/events.out.tfevents.1637519220.93c238760a3d.594.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0eb18e4fc43fd35b9fc90042851873520221167a3848acb4adf13ec373d5ffe
3
+ size 467
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
spm.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
3
+ size 2464616
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"do_lower_case": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "split_by_punct": false, "sp_model_kwargs": {}, "vocab_type": "spm", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "microsoft/deberta-v3-small", "tokenizer_class": "DebertaV2Tokenizer"}
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.0,
3
+ "train_loss": 0.1021328606294549,
4
+ "train_runtime": 294.6784,
5
+ "train_samples": 3668,
6
+ "train_samples_per_second": 124.475,
7
+ "train_steps_per_second": 7.805
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.27871406078338623,
3
+ "best_model_checkpoint": "/content/deberta-v3-small/checkpoint-230",
4
+ "epoch": 10.0,
5
+ "global_step": 2300,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 1.0,
12
+ "eval_accuracy": 0.8921568627450981,
13
+ "eval_combined_score": 0.9077509052401449,
14
+ "eval_f1": 0.9233449477351917,
15
+ "eval_loss": 0.27871406078338623,
16
+ "eval_runtime": 0.9934,
17
+ "eval_samples_per_second": 410.721,
18
+ "eval_steps_per_second": 51.34,
19
+ "step": 230
20
+ },
21
+ {
22
+ "epoch": 2.0,
23
+ "eval_accuracy": 0.875,
24
+ "eval_combined_score": 0.8943527918781726,
25
+ "eval_f1": 0.9137055837563453,
26
+ "eval_loss": 0.3650500774383545,
27
+ "eval_runtime": 1.104,
28
+ "eval_samples_per_second": 369.555,
29
+ "eval_steps_per_second": 46.194,
30
+ "step": 460
31
+ },
32
+ {
33
+ "epoch": 3.0,
34
+ "eval_accuracy": 0.8799019607843137,
35
+ "eval_combined_score": 0.8989124544290077,
36
+ "eval_f1": 0.9179229480737018,
37
+ "eval_loss": 0.5237621068954468,
38
+ "eval_runtime": 1.014,
39
+ "eval_samples_per_second": 402.381,
40
+ "eval_steps_per_second": 50.298,
41
+ "step": 690
42
+ },
43
+ {
44
+ "epoch": 4.0,
45
+ "eval_accuracy": 0.8946078431372549,
46
+ "eval_combined_score": 0.9084250788923165,
47
+ "eval_f1": 0.9222423146473779,
48
+ "eval_loss": 0.471233606338501,
49
+ "eval_runtime": 1.021,
50
+ "eval_samples_per_second": 399.621,
51
+ "eval_steps_per_second": 49.953,
52
+ "step": 920
53
+ },
54
+ {
55
+ "epoch": 4.35,
56
+ "learning_rate": 1.6956521739130433e-05,
57
+ "loss": 0.2147,
58
+ "step": 1000
59
+ },
60
+ {
61
+ "epoch": 5.0,
62
+ "eval_accuracy": 0.8946078431372549,
63
+ "eval_combined_score": 0.9104257054451284,
64
+ "eval_f1": 0.9262435677530019,
65
+ "eval_loss": 0.570386528968811,
66
+ "eval_runtime": 0.9794,
67
+ "eval_samples_per_second": 416.598,
68
+ "eval_steps_per_second": 52.075,
69
+ "step": 1150
70
+ },
71
+ {
72
+ "epoch": 6.0,
73
+ "eval_accuracy": 0.8995098039215687,
74
+ "eval_combined_score": 0.9139782876501386,
75
+ "eval_f1": 0.9284467713787086,
76
+ "eval_loss": 0.5697466135025024,
77
+ "eval_runtime": 0.9891,
78
+ "eval_samples_per_second": 412.495,
79
+ "eval_steps_per_second": 51.562,
80
+ "step": 1380
81
+ },
82
+ {
83
+ "epoch": 7.0,
84
+ "eval_accuracy": 0.8921568627450981,
85
+ "eval_combined_score": 0.9067927170868348,
86
+ "eval_f1": 0.9214285714285715,
87
+ "eval_loss": 0.6650992631912231,
88
+ "eval_runtime": 1.008,
89
+ "eval_samples_per_second": 404.777,
90
+ "eval_steps_per_second": 50.597,
91
+ "step": 1610
92
+ },
93
+ {
94
+ "epoch": 8.0,
95
+ "eval_accuracy": 0.8946078431372549,
96
+ "eval_combined_score": 0.9092508242234947,
97
+ "eval_f1": 0.9238938053097344,
98
+ "eval_loss": 0.672588586807251,
99
+ "eval_runtime": 1.0378,
100
+ "eval_samples_per_second": 393.14,
101
+ "eval_steps_per_second": 49.143,
102
+ "step": 1840
103
+ },
104
+ {
105
+ "epoch": 8.7,
106
+ "learning_rate": 3.913043478260869e-06,
107
+ "loss": 0.0183,
108
+ "step": 2000
109
+ },
110
+ {
111
+ "epoch": 9.0,
112
+ "eval_accuracy": 0.8848039215686274,
113
+ "eval_combined_score": 0.9012460938841386,
114
+ "eval_f1": 0.9176882661996497,
115
+ "eval_loss": 0.7250252366065979,
116
+ "eval_runtime": 1.0544,
117
+ "eval_samples_per_second": 386.943,
118
+ "eval_steps_per_second": 48.368,
119
+ "step": 2070
120
+ },
121
+ {
122
+ "epoch": 10.0,
123
+ "eval_accuracy": 0.8921568627450981,
124
+ "eval_combined_score": 0.907209173422019,
125
+ "eval_f1": 0.9222614840989399,
126
+ "eval_loss": 0.7092880010604858,
127
+ "eval_runtime": 1.0253,
128
+ "eval_samples_per_second": 397.917,
129
+ "eval_steps_per_second": 49.74,
130
+ "step": 2300
131
+ },
132
+ {
133
+ "epoch": 10.0,
134
+ "step": 2300,
135
+ "total_flos": 1214769315164160.0,
136
+ "train_loss": 0.1021328606294549,
137
+ "train_runtime": 294.6784,
138
+ "train_samples_per_second": 124.475,
139
+ "train_steps_per_second": 7.805
140
+ }
141
+ ],
142
+ "max_steps": 2300,
143
+ "num_train_epochs": 10,
144
+ "total_flos": 1214769315164160.0,
145
+ "trial_name": null,
146
+ "trial_params": null
147
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb9917d9201ebaf524ca52aa6fa3ddc24a6abebea41d899bd804eb7e652df3cd
3
+ size 2863