Hugging Face's logo

AmelieSchreiber
/
esm2_t6_8m_qlora_binding_sites_v0

PEFT
English
ESM-2
QLoRA
Binding Sites
biology
Model card Files Community
AmelieSchreiber commited on
Commit
986143f
•
1 Parent(s): a049c3f

Upload 48 files

Browse files
Files changed (48) hide show
  1. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/README.md +21 -0
  2. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/adapter_config.json +26 -0
  3. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/adapter_model.bin +3 -0
  4. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/added_tokens.json +7 -0
  5. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/optimizer.pt +3 -0
  6. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/rng_state.pth +3 -0
  7. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/scheduler.pt +3 -0
  8. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/special_tokens_map.json +7 -0
  9. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/tokenizer_config.json +54 -0
  10. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/trainer_state.json +415 -0
  11. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/training_args.bin +3 -0
  12. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/vocab.txt +33 -0
  13. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/README.md +21 -0
  14. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/adapter_config.json +26 -0
  15. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/adapter_model.bin +3 -0
  16. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/added_tokens.json +7 -0
  17. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/optimizer.pt +3 -0
  18. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/rng_state.pth +3 -0
  19. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/scheduler.pt +3 -0
  20. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/special_tokens_map.json +7 -0
  21. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/tokenizer_config.json +54 -0
  22. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/trainer_state.json +543 -0
  23. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/training_args.bin +3 -0
  24. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/vocab.txt +33 -0
  25. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/README.md +21 -0
  26. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/adapter_config.json +26 -0
  27. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/adapter_model.bin +3 -0
  28. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/added_tokens.json +7 -0
  29. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/optimizer.pt +3 -0
  30. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/rng_state.pth +3 -0
  31. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/scheduler.pt +3 -0
  32. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/special_tokens_map.json +7 -0
  33. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/tokenizer_config.json +54 -0
  34. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/trainer_state.json +147 -0
  35. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/training_args.bin +3 -0
  36. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/vocab.txt +33 -0
  37. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/README.md +21 -0
  38. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/adapter_config.json +26 -0
  39. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/adapter_model.bin +3 -0
  40. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/added_tokens.json +7 -0
  41. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/optimizer.pt +3 -0
  42. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/rng_state.pth +3 -0
  43. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/scheduler.pt +3 -0
  44. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/special_tokens_map.json +7 -0
  45. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/tokenizer_config.json +54 -0
  46. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/trainer_state.json +281 -0
  47. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/training_args.bin +3 -0
  48. esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/vocab.txt +33 -0
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": [
14
+ "classifier"
15
+ ],
16
+ "peft_type": "LORA",
17
+ "r": 2,
18
+ "rank_pattern": {},
19
+ "revision": null,
20
+ "target_modules": [
21
+ "query",
22
+ "key",
23
+ "value"
24
+ ],
25
+ "task_type": "TOKEN_CLS"
26
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9aa2cfb408dccd4df17ebd460d1c53e78d1634a07d4fdbc9fa235a87559ba464
3
+ size 108323
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90d68ccc83326239b324895a33766d44e52a8f0e5db512b7ffc7f1460ab88f7e
3
+ size 211805
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77b0ab42bbe269c06df0d453790ac0c293c65348ede27908f98bd8f1ebd0fc13
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3676a61e287c72a94f887301dbd12aad47cd3033c71e16308c1190c5c28aecd
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/trainer_state.json ADDED
@@ -0,0 +1,415 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.20658952998649033,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878",
4
+ "epoch": 3.0,
5
+ "eval_steps": 500,
6
+ "global_step": 11817,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.4987,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3756,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3422,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3223,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3094,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.3001,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2916,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2847,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2787,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2725,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2699,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2668,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.2636,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2604,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2577,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2545,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2515,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2518,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195503301009428,
122
+ "loss": 0.2465,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8584993872301376,
128
+ "eval_auc": 0.8182032038033911,
129
+ "eval_f1": 0.1931125248049425,
130
+ "eval_loss": 0.4206424355506897,
131
+ "eval_mcc": 0.25789003640920655,
132
+ "eval_precision": 0.11027656492418843,
133
+ "eval_recall": 0.7760683791635847,
134
+ "eval_runtime": 1158.8552,
135
+ "eval_samples_per_second": 110.153,
136
+ "eval_steps_per_second": 1.722,
137
+ "step": 3939
138
+ },
139
+ {
140
+ "epoch": 1.02,
141
+ "learning_rate": 0.0003143736142501833,
142
+ "loss": 0.2489,
143
+ "step": 4000
144
+ },
145
+ {
146
+ "epoch": 1.07,
147
+ "learning_rate": 0.0003089913128054233,
148
+ "loss": 0.244,
149
+ "step": 4200
150
+ },
151
+ {
152
+ "epoch": 1.12,
153
+ "learning_rate": 0.0003034119838843777,
154
+ "loss": 0.2417,
155
+ "step": 4400
156
+ },
157
+ {
158
+ "epoch": 1.17,
159
+ "learning_rate": 0.00029764449888790783,
160
+ "loss": 0.243,
161
+ "step": 4600
162
+ },
163
+ {
164
+ "epoch": 1.22,
165
+ "learning_rate": 0.0002916980283940693,
166
+ "loss": 0.2387,
167
+ "step": 4800
168
+ },
169
+ {
170
+ "epoch": 1.27,
171
+ "learning_rate": 0.00028558202757645164,
172
+ "loss": 0.2384,
173
+ "step": 5000
174
+ },
175
+ {
176
+ "epoch": 1.32,
177
+ "learning_rate": 0.0002793062211699992,
178
+ "loss": 0.2369,
179
+ "step": 5200
180
+ },
181
+ {
182
+ "epoch": 1.37,
183
+ "learning_rate": 0.0002728805880082148,
184
+ "loss": 0.2378,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.42,
189
+ "learning_rate": 0.00026631534515633606,
190
+ "loss": 0.2344,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.47,
195
+ "learning_rate": 0.0002596209316657108,
196
+ "loss": 0.2355,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.52,
201
+ "learning_rate": 0.00025280799197520415,
202
+ "loss": 0.2336,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.57,
207
+ "learning_rate": 0.0002458873589860299,
208
+ "loss": 0.2308,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.62,
213
+ "learning_rate": 0.00023887003683691727,
214
+ "loss": 0.2336,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.68,
219
+ "learning_rate": 0.00023176718340700212,
220
+ "loss": 0.2323,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.73,
225
+ "learning_rate": 0.00022462614386106907,
226
+ "loss": 0.2313,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.78,
231
+ "learning_rate": 0.00021738651311196128,
232
+ "loss": 0.231,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.83,
237
+ "learning_rate": 0.00021009551091624989,
238
+ "loss": 0.2299,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.88,
243
+ "learning_rate": 0.0002027647303174849,
244
+ "loss": 0.2289,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.93,
249
+ "learning_rate": 0.00019540582760878698,
250
+ "loss": 0.229,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.98,
255
+ "learning_rate": 0.00018803050379878183,
256
+ "loss": 0.229,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 2.0,
261
+ "eval_accuracy": 0.8743961636075187,
262
+ "eval_auc": 0.8133290603995718,
263
+ "eval_f1": 0.20658952998649033,
264
+ "eval_loss": 0.44484642148017883,
265
+ "eval_mcc": 0.26666675881078966,
266
+ "eval_precision": 0.11980687472936644,
267
+ "eval_recall": 0.749475576865448,
268
+ "eval_runtime": 1151.9615,
269
+ "eval_samples_per_second": 110.812,
270
+ "eval_steps_per_second": 1.732,
271
+ "step": 7878
272
+ },
273
+ {
274
+ "epoch": 2.03,
275
+ "learning_rate": 0.00018065048600643456,
276
+ "loss": 0.2252,
277
+ "step": 8000
278
+ },
279
+ {
280
+ "epoch": 2.08,
281
+ "learning_rate": 0.0001732775088143677,
282
+ "loss": 0.2269,
283
+ "step": 8200
284
+ },
285
+ {
286
+ "epoch": 2.13,
287
+ "learning_rate": 0.0001659232956103113,
288
+ "loss": 0.2269,
289
+ "step": 8400
290
+ },
291
+ {
292
+ "epoch": 2.18,
293
+ "learning_rate": 0.00015859953994635335,
294
+ "loss": 0.227,
295
+ "step": 8600
296
+ },
297
+ {
298
+ "epoch": 2.23,
299
+ "learning_rate": 0.00015131788694563091,
300
+ "loss": 0.2238,
301
+ "step": 8800
302
+ },
303
+ {
304
+ "epoch": 2.28,
305
+ "learning_rate": 0.00014412590206441025,
306
+ "loss": 0.2258,
307
+ "step": 9000
308
+ },
309
+ {
310
+ "epoch": 2.34,
311
+ "learning_rate": 0.00013696274927583722,
312
+ "loss": 0.2231,
313
+ "step": 9200
314
+ },
315
+ {
316
+ "epoch": 2.39,
317
+ "learning_rate": 0.00012987610268650092,
318
+ "loss": 0.2245,
319
+ "step": 9400
320
+ },
321
+ {
322
+ "epoch": 2.44,
323
+ "learning_rate": 0.00012287723040472557,
324
+ "loss": 0.2241,
325
+ "step": 9600
326
+ },
327
+ {
328
+ "epoch": 2.49,
329
+ "learning_rate": 0.00011597726097347204,
330
+ "loss": 0.225,
331
+ "step": 9800
332
+ },
333
+ {
334
+ "epoch": 2.54,
335
+ "learning_rate": 0.00010918716567542043,
336
+ "loss": 0.2228,
337
+ "step": 10000
338
+ },
339
+ {
340
+ "epoch": 2.59,
341
+ "learning_rate": 0.0001025177410881037,
342
+ "loss": 0.2212,
343
+ "step": 10200
344
+ },
345
+ {
346
+ "epoch": 2.64,
347
+ "learning_rate": 9.597959191683088e-05,
348
+ "loss": 0.2212,
349
+ "step": 10400
350
+ },
351
+ {
352
+ "epoch": 2.69,
353
+ "learning_rate": 8.958311413269684e-05,
354
+ "loss": 0.2211,
355
+ "step": 10600
356
+ },
357
+ {
358
+ "epoch": 2.74,
359
+ "learning_rate": 8.333847844248891e-05,
360
+ "loss": 0.2208,
361
+ "step": 10800
362
+ },
363
+ {
364
+ "epoch": 2.79,
365
+ "learning_rate": 7.725561411677462e-05,
366
+ "loss": 0.2208,
367
+ "step": 11000
368
+ },
369
+ {
370
+ "epoch": 2.84,
371
+ "learning_rate": 7.137330817547789e-05,
372
+ "loss": 0.2236,
373
+ "step": 11200
374
+ },
375
+ {
376
+ "epoch": 2.89,
377
+ "learning_rate": 6.564180298866504e-05,
378
+ "loss": 0.2252,
379
+ "step": 11400
380
+ },
381
+ {
382
+ "epoch": 2.94,
383
+ "learning_rate": 6.010020772997792e-05,
384
+ "loss": 0.2217,
385
+ "step": 11600
386
+ },
387
+ {
388
+ "epoch": 3.0,
389
+ "learning_rate": 5.475733380194338e-05,
390
+ "loss": 0.2171,
391
+ "step": 11800
392
+ },
393
+ {
394
+ "epoch": 3.0,
395
+ "eval_accuracy": 0.871436177611881,
396
+ "eval_auc": 0.8150915702010527,
397
+ "eval_f1": 0.2042404682874535,
398
+ "eval_loss": 0.4439041316509247,
399
+ "eval_mcc": 0.2655282757252214,
400
+ "eval_precision": 0.11806465629899295,
401
+ "eval_recall": 0.7561760613174451,
402
+ "eval_runtime": 1159.3364,
403
+ "eval_samples_per_second": 110.107,
404
+ "eval_steps_per_second": 1.721,
405
+ "step": 11817
406
+ }
407
+ ],
408
+ "logging_steps": 200,
409
+ "max_steps": 15756,
410
+ "num_train_epochs": 4,
411
+ "save_steps": 500,
412
+ "total_flos": 6.076133341593283e+16,
413
+ "trial_name": null,
414
+ "trial_params": null
415
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21e29ca655570a4c93caaae2152a14df90626498f407241f0d253a3640c7a8ce
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-11817/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": [
14
+ "classifier"
15
+ ],
16
+ "peft_type": "LORA",
17
+ "r": 2,
18
+ "rank_pattern": {},
19
+ "revision": null,
20
+ "target_modules": [
21
+ "query",
22
+ "key",
23
+ "value"
24
+ ],
25
+ "task_type": "TOKEN_CLS"
26
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ea428d19fc1c256e6d448e032b63cbcb1c0fe35c1cad4960c7a6ed03c5b92ba
3
+ size 108323
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f16afa5d73496c13e042eb7e137c69517f95ec7f0ab3b4c76fd442bf9edf2e0
3
+ size 211805
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:291bc0bdc85d782988ee26dcae148038cc1f15508622f285332bbe7b99270d1b
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86ab8ca8d941f7a9dc0e1bfb3c414eb3afc0b3e1513ea8a7ce0db94586cd403d
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/trainer_state.json ADDED
@@ -0,0 +1,543 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.20658952998649033,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878",
4
+ "epoch": 4.0,
5
+ "eval_steps": 500,
6
+ "global_step": 15756,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.4987,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3756,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3422,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3223,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3094,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.3001,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2916,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2847,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2787,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2725,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2699,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2668,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.2636,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2604,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2577,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2545,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2515,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2518,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195503301009428,
122
+ "loss": 0.2465,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8584993872301376,
128
+ "eval_auc": 0.8182032038033911,
129
+ "eval_f1": 0.1931125248049425,
130
+ "eval_loss": 0.4206424355506897,
131
+ "eval_mcc": 0.25789003640920655,
132
+ "eval_precision": 0.11027656492418843,
133
+ "eval_recall": 0.7760683791635847,
134
+ "eval_runtime": 1158.8552,
135
+ "eval_samples_per_second": 110.153,
136
+ "eval_steps_per_second": 1.722,
137
+ "step": 3939
138
+ },
139
+ {
140
+ "epoch": 1.02,
141
+ "learning_rate": 0.0003143736142501833,
142
+ "loss": 0.2489,
143
+ "step": 4000
144
+ },
145
+ {
146
+ "epoch": 1.07,
147
+ "learning_rate": 0.0003089913128054233,
148
+ "loss": 0.244,
149
+ "step": 4200
150
+ },
151
+ {
152
+ "epoch": 1.12,
153
+ "learning_rate": 0.0003034119838843777,
154
+ "loss": 0.2417,
155
+ "step": 4400
156
+ },
157
+ {
158
+ "epoch": 1.17,
159
+ "learning_rate": 0.00029764449888790783,
160
+ "loss": 0.243,
161
+ "step": 4600
162
+ },
163
+ {
164
+ "epoch": 1.22,
165
+ "learning_rate": 0.0002916980283940693,
166
+ "loss": 0.2387,
167
+ "step": 4800
168
+ },
169
+ {
170
+ "epoch": 1.27,
171
+ "learning_rate": 0.00028558202757645164,
172
+ "loss": 0.2384,
173
+ "step": 5000
174
+ },
175
+ {
176
+ "epoch": 1.32,
177
+ "learning_rate": 0.0002793062211699992,
178
+ "loss": 0.2369,
179
+ "step": 5200
180
+ },
181
+ {
182
+ "epoch": 1.37,
183
+ "learning_rate": 0.0002728805880082148,
184
+ "loss": 0.2378,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.42,
189
+ "learning_rate": 0.00026631534515633606,
190
+ "loss": 0.2344,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.47,
195
+ "learning_rate": 0.0002596209316657108,
196
+ "loss": 0.2355,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.52,
201
+ "learning_rate": 0.00025280799197520415,
202
+ "loss": 0.2336,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.57,
207
+ "learning_rate": 0.0002458873589860299,
208
+ "loss": 0.2308,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.62,
213
+ "learning_rate": 0.00023887003683691727,
214
+ "loss": 0.2336,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.68,
219
+ "learning_rate": 0.00023176718340700212,
220
+ "loss": 0.2323,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.73,
225
+ "learning_rate": 0.00022462614386106907,
226
+ "loss": 0.2313,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.78,
231
+ "learning_rate": 0.00021738651311196128,
232
+ "loss": 0.231,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.83,
237
+ "learning_rate": 0.00021009551091624989,
238
+ "loss": 0.2299,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.88,
243
+ "learning_rate": 0.0002027647303174849,
244
+ "loss": 0.2289,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.93,
249
+ "learning_rate": 0.00019540582760878698,
250
+ "loss": 0.229,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.98,
255
+ "learning_rate": 0.00018803050379878183,
256
+ "loss": 0.229,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 2.0,
261
+ "eval_accuracy": 0.8743961636075187,
262
+ "eval_auc": 0.8133290603995718,
263
+ "eval_f1": 0.20658952998649033,
264
+ "eval_loss": 0.44484642148017883,
265
+ "eval_mcc": 0.26666675881078966,
266
+ "eval_precision": 0.11980687472936644,
267
+ "eval_recall": 0.749475576865448,
268
+ "eval_runtime": 1151.9615,
269
+ "eval_samples_per_second": 110.812,
270
+ "eval_steps_per_second": 1.732,
271
+ "step": 7878
272
+ },
273
+ {
274
+ "epoch": 2.03,
275
+ "learning_rate": 0.00018065048600643456,
276
+ "loss": 0.2252,
277
+ "step": 8000
278
+ },
279
+ {
280
+ "epoch": 2.08,
281
+ "learning_rate": 0.0001732775088143677,
282
+ "loss": 0.2269,
283
+ "step": 8200
284
+ },
285
+ {
286
+ "epoch": 2.13,
287
+ "learning_rate": 0.0001659232956103113,
288
+ "loss": 0.2269,
289
+ "step": 8400
290
+ },
291
+ {
292
+ "epoch": 2.18,
293
+ "learning_rate": 0.00015859953994635335,
294
+ "loss": 0.227,
295
+ "step": 8600
296
+ },
297
+ {
298
+ "epoch": 2.23,
299
+ "learning_rate": 0.00015131788694563091,
300
+ "loss": 0.2238,
301
+ "step": 8800
302
+ },
303
+ {
304
+ "epoch": 2.28,
305
+ "learning_rate": 0.00014412590206441025,
306
+ "loss": 0.2258,
307
+ "step": 9000
308
+ },
309
+ {
310
+ "epoch": 2.34,
311
+ "learning_rate": 0.00013696274927583722,
312
+ "loss": 0.2231,
313
+ "step": 9200
314
+ },
315
+ {
316
+ "epoch": 2.39,
317
+ "learning_rate": 0.00012987610268650092,
318
+ "loss": 0.2245,
319
+ "step": 9400
320
+ },
321
+ {
322
+ "epoch": 2.44,
323
+ "learning_rate": 0.00012287723040472557,
324
+ "loss": 0.2241,
325
+ "step": 9600
326
+ },
327
+ {
328
+ "epoch": 2.49,
329
+ "learning_rate": 0.00011597726097347204,
330
+ "loss": 0.225,
331
+ "step": 9800
332
+ },
333
+ {
334
+ "epoch": 2.54,
335
+ "learning_rate": 0.00010918716567542043,
336
+ "loss": 0.2228,
337
+ "step": 10000
338
+ },
339
+ {
340
+ "epoch": 2.59,
341
+ "learning_rate": 0.0001025177410881037,
342
+ "loss": 0.2212,
343
+ "step": 10200
344
+ },
345
+ {
346
+ "epoch": 2.64,
347
+ "learning_rate": 9.597959191683088e-05,
348
+ "loss": 0.2212,
349
+ "step": 10400
350
+ },
351
+ {
352
+ "epoch": 2.69,
353
+ "learning_rate": 8.958311413269684e-05,
354
+ "loss": 0.2211,
355
+ "step": 10600
356
+ },
357
+ {
358
+ "epoch": 2.74,
359
+ "learning_rate": 8.333847844248891e-05,
360
+ "loss": 0.2208,
361
+ "step": 10800
362
+ },
363
+ {
364
+ "epoch": 2.79,
365
+ "learning_rate": 7.725561411677462e-05,
366
+ "loss": 0.2208,
367
+ "step": 11000
368
+ },
369
+ {
370
+ "epoch": 2.84,
371
+ "learning_rate": 7.137330817547789e-05,
372
+ "loss": 0.2236,
373
+ "step": 11200
374
+ },
375
+ {
376
+ "epoch": 2.89,
377
+ "learning_rate": 6.564180298866504e-05,
378
+ "loss": 0.2252,
379
+ "step": 11400
380
+ },
381
+ {
382
+ "epoch": 2.94,
383
+ "learning_rate": 6.010020772997792e-05,
384
+ "loss": 0.2217,
385
+ "step": 11600
386
+ },
387
+ {
388
+ "epoch": 3.0,
389
+ "learning_rate": 5.475733380194338e-05,
390
+ "loss": 0.2171,
391
+ "step": 11800
392
+ },
393
+ {
394
+ "epoch": 3.0,
395
+ "eval_accuracy": 0.871436177611881,
396
+ "eval_auc": 0.8150915702010527,
397
+ "eval_f1": 0.2042404682874535,
398
+ "eval_loss": 0.4439041316509247,
399
+ "eval_mcc": 0.2655282757252214,
400
+ "eval_precision": 0.11806465629899295,
401
+ "eval_recall": 0.7561760613174451,
402
+ "eval_runtime": 1159.3364,
403
+ "eval_samples_per_second": 110.107,
404
+ "eval_steps_per_second": 1.721,
405
+ "step": 11817
406
+ },
407
+ {
408
+ "epoch": 3.05,
409
+ "learning_rate": 4.962167663063128e-05,
410
+ "loss": 0.2222,
411
+ "step": 12000
412
+ },
413
+ {
414
+ "epoch": 3.1,
415
+ "learning_rate": 4.4701402157519476e-05,
416
+ "loss": 0.2192,
417
+ "step": 12200
418
+ },
419
+ {
420
+ "epoch": 3.15,
421
+ "learning_rate": 4.000433385525458e-05,
422
+ "loss": 0.2214,
423
+ "step": 12400
424
+ },
425
+ {
426
+ "epoch": 3.2,
427
+ "learning_rate": 3.553794028795492e-05,
428
+ "loss": 0.2231,
429
+ "step": 12600
430
+ },
431
+ {
432
+ "epoch": 3.25,
433
+ "learning_rate": 3.130932323583492e-05,
434
+ "loss": 0.2182,
435
+ "step": 12800
436
+ },
437
+ {
438
+ "epoch": 3.3,
439
+ "learning_rate": 2.7325206403033605e-05,
440
+ "loss": 0.2206,
441
+ "step": 13000
442
+ },
443
+ {
444
+ "epoch": 3.35,
445
+ "learning_rate": 2.359192472660235e-05,
446
+ "loss": 0.2211,
447
+ "step": 13200
448
+ },
449
+ {
450
+ "epoch": 3.4,
451
+ "learning_rate": 2.0132148821294626e-05,
452
+ "loss": 0.22,
453
+ "step": 13400
454
+ },
455
+ {
456
+ "epoch": 3.45,
457
+ "learning_rate": 1.691661291092208e-05,
458
+ "loss": 0.2191,
459
+ "step": 13600
460
+ },
461
+ {
462
+ "epoch": 3.5,
463
+ "learning_rate": 1.3968462320313953e-05,
464
+ "loss": 0.2178,
465
+ "step": 13800
466
+ },
467
+ {
468
+ "epoch": 3.55,
469
+ "learning_rate": 1.1292384750457188e-05,
470
+ "loss": 0.2212,
471
+ "step": 14000
472
+ },
473
+ {
474
+ "epoch": 3.6,
475
+ "learning_rate": 8.892635293172929e-06,
476
+ "loss": 0.2194,
477
+ "step": 14200
478
+ },
479
+ {
480
+ "epoch": 3.66,
481
+ "learning_rate": 6.773029665315992e-06,
482
+ "loss": 0.2195,
483
+ "step": 14400
484
+ },
485
+ {
486
+ "epoch": 3.71,
487
+ "learning_rate": 4.936938141602164e-06,
488
+ "loss": 0.2153,
489
+ "step": 14600
490
+ },
491
+ {
492
+ "epoch": 3.76,
493
+ "learning_rate": 3.387280195709328e-06,
494
+ "loss": 0.2199,
495
+ "step": 14800
496
+ },
497
+ {
498
+ "epoch": 3.81,
499
+ "learning_rate": 2.126519858174352e-06,
500
+ "loss": 0.2191,
501
+ "step": 15000
502
+ },
503
+ {
504
+ "epoch": 3.86,
505
+ "learning_rate": 1.1566617984660576e-06,
506
+ "loss": 0.2208,
507
+ "step": 15200
508
+ },
509
+ {
510
+ "epoch": 3.91,
511
+ "learning_rate": 4.792481374646427e-07,
512
+ "loss": 0.2189,
513
+ "step": 15400
514
+ },
515
+ {
516
+ "epoch": 3.96,
517
+ "learning_rate": 9.535599541537485e-08,
518
+ "loss": 0.2214,
519
+ "step": 15600
520
+ },
521
+ {
522
+ "epoch": 4.0,
523
+ "eval_accuracy": 0.8728236161082246,
524
+ "eval_auc": 0.8142150086545907,
525
+ "eval_f1": 0.2053073108400613,
526
+ "eval_loss": 0.4466347098350525,
527
+ "eval_mcc": 0.26601389088241206,
528
+ "eval_precision": 0.11885866956163944,
529
+ "eval_recall": 0.7529321975434273,
530
+ "eval_runtime": 1151.0693,
531
+ "eval_samples_per_second": 110.898,
532
+ "eval_steps_per_second": 1.733,
533
+ "step": 15756
534
+ }
535
+ ],
536
+ "logging_steps": 200,
537
+ "max_steps": 15756,
538
+ "num_train_epochs": 4,
539
+ "save_steps": 500,
540
+ "total_flos": 8.101511122124378e+16,
541
+ "trial_name": null,
542
+ "trial_params": null
543
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21e29ca655570a4c93caaae2152a14df90626498f407241f0d253a3640c7a8ce
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-15756/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": [
14
+ "classifier"
15
+ ],
16
+ "peft_type": "LORA",
17
+ "r": 2,
18
+ "rank_pattern": {},
19
+ "revision": null,
20
+ "target_modules": [
21
+ "query",
22
+ "key",
23
+ "value"
24
+ ],
25
+ "task_type": "TOKEN_CLS"
26
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58f0e1345bfe41c5efffdc521d99e708f2403a2bda6bc0dc7f54366b7e83c583
3
+ size 108323
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5667c0e66802b9aeb1d3905ba5b9d7744b3b18a6fa4588c3ee4a32c87d175357
3
+ size 211805
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dda48e7f13921baa764c07484621328651515c0c8ccfdfb93f93ae7e61fae19f
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ca35a022c60477fa8e7993efcd1e5fb3018c98742658e53d75e6554387a88dd
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/trainer_state.json ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.1931125248049425,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939",
4
+ "epoch": 1.0,
5
+ "eval_steps": 500,
6
+ "global_step": 3939,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.4987,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3756,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3422,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3223,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3094,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.3001,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2916,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2847,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2787,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2725,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2699,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2668,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.2636,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2604,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2577,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2545,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2515,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2518,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195503301009428,
122
+ "loss": 0.2465,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8584993872301376,
128
+ "eval_auc": 0.8182032038033911,
129
+ "eval_f1": 0.1931125248049425,
130
+ "eval_loss": 0.4206424355506897,
131
+ "eval_mcc": 0.25789003640920655,
132
+ "eval_precision": 0.11027656492418843,
133
+ "eval_recall": 0.7760683791635847,
134
+ "eval_runtime": 1158.8552,
135
+ "eval_samples_per_second": 110.153,
136
+ "eval_steps_per_second": 1.722,
137
+ "step": 3939
138
+ }
139
+ ],
140
+ "logging_steps": 200,
141
+ "max_steps": 15756,
142
+ "num_train_epochs": 4,
143
+ "save_steps": 500,
144
+ "total_flos": 2.025377780531094e+16,
145
+ "trial_name": null,
146
+ "trial_params": null
147
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21e29ca655570a4c93caaae2152a14df90626498f407241f0d253a3640c7a8ce
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-3939/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/adapter_config.json ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": [
14
+ "classifier"
15
+ ],
16
+ "peft_type": "LORA",
17
+ "r": 2,
18
+ "rank_pattern": {},
19
+ "revision": null,
20
+ "target_modules": [
21
+ "query",
22
+ "key",
23
+ "value"
24
+ ],
25
+ "task_type": "TOKEN_CLS"
26
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46cf5fda038695acb6e2984cf4f53d4538084b803b547514f0d1116818a239d4
3
+ size 108323
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e84b07be31e0a5c5b01c1af139af70b47871b2b3307d6b701dbceafd3d65b2f
3
+ size 211805
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4749862068cb738e8ae2866b01be18899de3c0c66608f4b02e33b522b0c2ac7
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ee80568c48d9c580fd7750f89963a598159ebb4c182b6e39f1bf7f626966ded
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/trainer_state.json ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.20658952998649033,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878",
4
+ "epoch": 2.0,
5
+ "eval_steps": 500,
6
+ "global_step": 7878,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.4987,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3756,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3422,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3223,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3094,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.3001,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2916,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2847,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2787,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2725,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2699,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2668,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.2636,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2604,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2577,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2545,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2515,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2518,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195503301009428,
122
+ "loss": 0.2465,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8584993872301376,
128
+ "eval_auc": 0.8182032038033911,
129
+ "eval_f1": 0.1931125248049425,
130
+ "eval_loss": 0.4206424355506897,
131
+ "eval_mcc": 0.25789003640920655,
132
+ "eval_precision": 0.11027656492418843,
133
+ "eval_recall": 0.7760683791635847,
134
+ "eval_runtime": 1158.8552,
135
+ "eval_samples_per_second": 110.153,
136
+ "eval_steps_per_second": 1.722,
137
+ "step": 3939
138
+ },
139
+ {
140
+ "epoch": 1.02,
141
+ "learning_rate": 0.0003143736142501833,
142
+ "loss": 0.2489,
143
+ "step": 4000
144
+ },
145
+ {
146
+ "epoch": 1.07,
147
+ "learning_rate": 0.0003089913128054233,
148
+ "loss": 0.244,
149
+ "step": 4200
150
+ },
151
+ {
152
+ "epoch": 1.12,
153
+ "learning_rate": 0.0003034119838843777,
154
+ "loss": 0.2417,
155
+ "step": 4400
156
+ },
157
+ {
158
+ "epoch": 1.17,
159
+ "learning_rate": 0.00029764449888790783,
160
+ "loss": 0.243,
161
+ "step": 4600
162
+ },
163
+ {
164
+ "epoch": 1.22,
165
+ "learning_rate": 0.0002916980283940693,
166
+ "loss": 0.2387,
167
+ "step": 4800
168
+ },
169
+ {
170
+ "epoch": 1.27,
171
+ "learning_rate": 0.00028558202757645164,
172
+ "loss": 0.2384,
173
+ "step": 5000
174
+ },
175
+ {
176
+ "epoch": 1.32,
177
+ "learning_rate": 0.0002793062211699992,
178
+ "loss": 0.2369,
179
+ "step": 5200
180
+ },
181
+ {
182
+ "epoch": 1.37,
183
+ "learning_rate": 0.0002728805880082148,
184
+ "loss": 0.2378,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.42,
189
+ "learning_rate": 0.00026631534515633606,
190
+ "loss": 0.2344,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.47,
195
+ "learning_rate": 0.0002596209316657108,
196
+ "loss": 0.2355,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.52,
201
+ "learning_rate": 0.00025280799197520415,
202
+ "loss": 0.2336,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.57,
207
+ "learning_rate": 0.0002458873589860299,
208
+ "loss": 0.2308,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.62,
213
+ "learning_rate": 0.00023887003683691727,
214
+ "loss": 0.2336,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.68,
219
+ "learning_rate": 0.00023176718340700212,
220
+ "loss": 0.2323,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.73,
225
+ "learning_rate": 0.00022462614386106907,
226
+ "loss": 0.2313,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.78,
231
+ "learning_rate": 0.00021738651311196128,
232
+ "loss": 0.231,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.83,
237
+ "learning_rate": 0.00021009551091624989,
238
+ "loss": 0.2299,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.88,
243
+ "learning_rate": 0.0002027647303174849,
244
+ "loss": 0.2289,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.93,
249
+ "learning_rate": 0.00019540582760878698,
250
+ "loss": 0.229,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.98,
255
+ "learning_rate": 0.00018803050379878183,
256
+ "loss": 0.229,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 2.0,
261
+ "eval_accuracy": 0.8743961636075187,
262
+ "eval_auc": 0.8133290603995718,
263
+ "eval_f1": 0.20658952998649033,
264
+ "eval_loss": 0.44484642148017883,
265
+ "eval_mcc": 0.26666675881078966,
266
+ "eval_precision": 0.11980687472936644,
267
+ "eval_recall": 0.749475576865448,
268
+ "eval_runtime": 1151.9615,
269
+ "eval_samples_per_second": 110.812,
270
+ "eval_steps_per_second": 1.732,
271
+ "step": 7878
272
+ }
273
+ ],
274
+ "logging_steps": 200,
275
+ "max_steps": 15756,
276
+ "num_train_epochs": 4,
277
+ "save_steps": 500,
278
+ "total_flos": 4.050755561062189e+16,
279
+ "trial_name": null,
280
+ "trial_params": null
281
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21e29ca655570a4c93caaae2152a14df90626498f407241f0d253a3640c7a8ce
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-28_00-05-45/checkpoint-7878/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>