Hugging Face's logo

AmelieSchreiber
/
esm2_t12_35M_qlora_binding_2600K_cp1

Model card Files Community
AmelieSchreiber commited on
Commit
85c9414
•
1 Parent(s): 28bad80

Upload 36 files

Browse files
Files changed (36) hide show
  1. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/README.md +21 -0
  2. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/adapter_config.json +29 -0
  3. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/adapter_model.bin +3 -0
  4. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/added_tokens.json +7 -0
  5. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/optimizer.pt +3 -0
  6. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/rng_state.pth +3 -0
  7. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/scheduler.pt +3 -0
  8. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/special_tokens_map.json +7 -0
  9. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/tokenizer_config.json +54 -0
  10. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/trainer_state.json +359 -0
  11. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/training_args.bin +3 -0
  12. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/vocab.txt +33 -0
  13. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/README.md +21 -0
  14. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/adapter_config.json +29 -0
  15. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/adapter_model.bin +3 -0
  16. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/added_tokens.json +7 -0
  17. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/optimizer.pt +3 -0
  18. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/rng_state.pth +3 -0
  19. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/scheduler.pt +3 -0
  20. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/special_tokens_map.json +7 -0
  21. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/tokenizer_config.json +54 -0
  22. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/trainer_state.json +535 -0
  23. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/training_args.bin +3 -0
  24. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/vocab.txt +33 -0
  25. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/README.md +21 -0
  26. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/adapter_config.json +29 -0
  27. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/adapter_model.bin +3 -0
  28. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/added_tokens.json +7 -0
  29. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/optimizer.pt +3 -0
  30. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/rng_state.pth +3 -0
  31. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/scheduler.pt +3 -0
  32. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/special_tokens_map.json +7 -0
  33. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/tokenizer_config.json +54 -0
  34. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/trainer_state.json +189 -0
  35. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/training_args.bin +3 -0
  36. esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/vocab.txt +33 -0
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t12_35M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 2,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "query",
20
+ "key",
21
+ "value",
22
+ "EsmSelfOutput.dense",
23
+ "EsmIntermediate.dense",
24
+ "EsmOutput.dense",
25
+ "EsmContactPredictionHead.regression",
26
+ "classifier"
27
+ ],
28
+ "task_type": "TOKEN_CLS"
29
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61603f9daa3bc5426da6a9e63b9cd87357286d668b7e1bea44d3fbadd044ffa1
3
+ size 316315
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66dd295f17f17a981b2542eb96066ea1cb61b5570e9d4878e71c30f67f72db63
3
+ size 613061
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3825cc7bd56dd7de04ff391db0d3fba19f0e08a47cdd6ec62f6ae3d21bfe4519
3
+ size 14575
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8a7e486b2eaac07274ab77c32387d286c9a5d11eab8cd4b9575bcc5d00c2535
3
+ size 627
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/trainer_state.json ADDED
@@ -0,0 +1,359 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.3534505903118732,
3
+ "best_model_checkpoint": "esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578",
4
+ "epoch": 2.0,
5
+ "eval_steps": 500,
6
+ "global_step": 10578,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.04,
13
+ "learning_rate": 0.0003700117150420716,
14
+ "loss": 0.4265,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.08,
19
+ "learning_rate": 0.0003695766709151845,
20
+ "loss": 0.2446,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.11,
25
+ "learning_rate": 0.00036885235529645174,
26
+ "loss": 0.1964,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.15,
31
+ "learning_rate": 0.00036783990382720894,
32
+ "loss": 0.1714,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.19,
37
+ "learning_rate": 0.000366540903911771,
38
+ "loss": 0.1556,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.23,
43
+ "learning_rate": 0.00036495739222857005,
44
+ "loss": 0.1454,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.26,
49
+ "learning_rate": 0.00036309185153688393,
50
+ "loss": 0.1387,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.3,
55
+ "learning_rate": 0.0003609472067841633,
56
+ "loss": 0.133,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.34,
61
+ "learning_rate": 0.0003585268205200591,
62
+ "loss": 0.1289,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.38,
67
+ "learning_rate": 0.0003558344876243421,
68
+ "loss": 0.1228,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.42,
73
+ "learning_rate": 0.0003528744293569798,
74
+ "loss": 0.1208,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.45,
79
+ "learning_rate": 0.00034965128673969907,
80
+ "loss": 0.1167,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.49,
85
+ "learning_rate": 0.0003461701132794125,
86
+ "loss": 0.1131,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.53,
91
+ "learning_rate": 0.0003424363670449164,
92
+ "loss": 0.1113,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.57,
97
+ "learning_rate": 0.00033845590210928363,
98
+ "loss": 0.1088,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.61,
103
+ "learning_rate": 0.00033423495937136847,
104
+ "loss": 0.1081,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.64,
109
+ "learning_rate": 0.00032978015677081454,
110
+ "loss": 0.1054,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.68,
115
+ "learning_rate": 0.00032509847891190683,
116
+ "loss": 0.1029,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.72,
121
+ "learning_rate": 0.0003201972661125376,
122
+ "loss": 0.102,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 0.76,
127
+ "learning_rate": 0.0003150842028954544,
128
+ "loss": 0.0999,
129
+ "step": 4000
130
+ },
131
+ {
132
+ "epoch": 0.79,
133
+ "learning_rate": 0.0003097673059398367,
134
+ "loss": 0.098,
135
+ "step": 4200
136
+ },
137
+ {
138
+ "epoch": 0.83,
139
+ "learning_rate": 0.00030425491151208934,
140
+ "loss": 0.0972,
141
+ "step": 4400
142
+ },
143
+ {
144
+ "epoch": 0.87,
145
+ "learning_rate": 0.00029855566239556173,
146
+ "loss": 0.0968,
147
+ "step": 4600
148
+ },
149
+ {
150
+ "epoch": 0.91,
151
+ "learning_rate": 0.0002926784943396845,
152
+ "loss": 0.0947,
153
+ "step": 4800
154
+ },
155
+ {
156
+ "epoch": 0.95,
157
+ "learning_rate": 0.00028663262204976965,
158
+ "loss": 0.0933,
159
+ "step": 5000
160
+ },
161
+ {
162
+ "epoch": 0.98,
163
+ "learning_rate": 0.00028042752473944106,
164
+ "loss": 0.094,
165
+ "step": 5200
166
+ },
167
+ {
168
+ "epoch": 1.0,
169
+ "eval_accuracy": 0.9297529150299436,
170
+ "eval_auc": 0.8167222019799886,
171
+ "eval_f1": 0.3440490710592986,
172
+ "eval_loss": 0.6011912822723389,
173
+ "eval_mcc": 0.3730152153022164,
174
+ "eval_precision": 0.22835223718675476,
175
+ "eval_recall": 0.697386656717114,
176
+ "eval_runtime": 3756.1015,
177
+ "eval_samples_per_second": 144.201,
178
+ "eval_steps_per_second": 0.687,
179
+ "step": 5289
180
+ },
181
+ {
182
+ "epoch": 1.02,
183
+ "learning_rate": 0.00027407293126834736,
184
+ "loss": 0.0924,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.06,
189
+ "learning_rate": 0.00026757880488845914,
190
+ "loss": 0.0912,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.1,
195
+ "learning_rate": 0.00026095532762286633,
196
+ "loss": 0.0902,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.13,
201
+ "learning_rate": 0.0002542128843015694,
202
+ "loss": 0.09,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.17,
207
+ "learning_rate": 0.00024739655239955165,
208
+ "loss": 0.0901,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.21,
213
+ "learning_rate": 0.00024048348750604134,
214
+ "loss": 0.0889,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.25,
219
+ "learning_rate": 0.00023344905030678293,
220
+ "loss": 0.0872,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.29,
225
+ "learning_rate": 0.00022633877364566408,
226
+ "loss": 0.0873,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.32,
231
+ "learning_rate": 0.0002191638055966375,
232
+ "loss": 0.0866,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.36,
237
+ "learning_rate": 0.00021193539566211078,
238
+ "loss": 0.0864,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.4,
243
+ "learning_rate": 0.00020466487713505563,
244
+ "loss": 0.085,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.44,
249
+ "learning_rate": 0.000197363649329743,
250
+ "loss": 0.086,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.47,
255
+ "learning_rate": 0.00019004315970896487,
256
+ "loss": 0.0847,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 1.51,
261
+ "learning_rate": 0.00018271488593576474,
262
+ "loss": 0.0836,
263
+ "step": 8000
264
+ },
265
+ {
266
+ "epoch": 1.55,
267
+ "learning_rate": 0.0001753903178778174,
268
+ "loss": 0.0844,
269
+ "step": 8200
270
+ },
271
+ {
272
+ "epoch": 1.59,
273
+ "learning_rate": 0.00016808093959267418,
274
+ "loss": 0.0828,
275
+ "step": 8400
276
+ },
277
+ {
278
+ "epoch": 1.63,
279
+ "learning_rate": 0.00016079821132211663,
280
+ "loss": 0.0848,
281
+ "step": 8600
282
+ },
283
+ {
284
+ "epoch": 1.66,
285
+ "learning_rate": 0.00015355355152385142,
286
+ "loss": 0.0826,
287
+ "step": 8800
288
+ },
289
+ {
290
+ "epoch": 1.7,
291
+ "learning_rate": 0.00014635831896871702,
292
+ "loss": 0.0826,
293
+ "step": 9000
294
+ },
295
+ {
296
+ "epoch": 1.74,
297
+ "learning_rate": 0.00013922379493147316,
298
+ "loss": 0.0833,
299
+ "step": 9200
300
+ },
301
+ {
302
+ "epoch": 1.78,
303
+ "learning_rate": 0.0001321611655030941,
304
+ "loss": 0.0816,
305
+ "step": 9400
306
+ },
307
+ {
308
+ "epoch": 1.82,
309
+ "learning_rate": 0.00012518150405229972,
310
+ "loss": 0.0817,
311
+ "step": 9600
312
+ },
313
+ {
314
+ "epoch": 1.85,
315
+ "learning_rate": 0.0001182957538638206,
316
+ "loss": 0.0827,
317
+ "step": 9800
318
+ },
319
+ {
320
+ "epoch": 1.89,
321
+ "learning_rate": 0.00011151471098062111,
322
+ "loss": 0.0807,
323
+ "step": 10000
324
+ },
325
+ {
326
+ "epoch": 1.93,
327
+ "learning_rate": 0.00010484900727697857,
328
+ "loss": 0.0818,
329
+ "step": 10200
330
+ },
331
+ {
332
+ "epoch": 1.97,
333
+ "learning_rate": 9.830909378896018e-05,
334
+ "loss": 0.0811,
335
+ "step": 10400
336
+ },
337
+ {
338
+ "epoch": 2.0,
339
+ "eval_accuracy": 0.9345128329975654,
340
+ "eval_auc": 0.8095464647248594,
341
+ "eval_f1": 0.3534505903118732,
342
+ "eval_loss": 0.646632730960846,
343
+ "eval_mcc": 0.37724366807619925,
344
+ "eval_precision": 0.23907868925380343,
345
+ "eval_recall": 0.6776094956043187,
346
+ "eval_runtime": 3741.5855,
347
+ "eval_samples_per_second": 144.761,
348
+ "eval_steps_per_second": 0.69,
349
+ "step": 10578
350
+ }
351
+ ],
352
+ "logging_steps": 200,
353
+ "max_steps": 15867,
354
+ "num_train_epochs": 3,
355
+ "save_steps": 500,
356
+ "total_flos": 8.013035409672134e+17,
357
+ "trial_name": null,
358
+ "trial_params": null
359
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b915c4deb5ba56cacb3593b78264a7c9bd0e44e5e5af7a361d94ad20b8ca1c89
3
+ size 4155
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-10578/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t12_35M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 2,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "query",
20
+ "key",
21
+ "value",
22
+ "EsmSelfOutput.dense",
23
+ "EsmIntermediate.dense",
24
+ "EsmOutput.dense",
25
+ "EsmContactPredictionHead.regression",
26
+ "classifier"
27
+ ],
28
+ "task_type": "TOKEN_CLS"
29
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ba4edd057f3a0bc2a442f8477de0e0ea8035db2595747ba855e70bd963307e3b
3
+ size 316315
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac484a31bf2f2adfd47adeae9e731c1edd93bc3a1a02bc0f0c4da3c2e52addee
3
+ size 613061
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e15e4908e305ead99c37b0c73cda396d47059d09d3389e311282bf02b9b21f8d
3
+ size 14575
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55939009af75a35b0d684fd3fd80e9b6bc79f81fc341707729eb4dbfe06fb08c
3
+ size 627
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/trainer_state.json ADDED
@@ -0,0 +1,535 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.35447033515145,
3
+ "best_model_checkpoint": "esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867",
4
+ "epoch": 3.0,
5
+ "eval_steps": 500,
6
+ "global_step": 15867,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.04,
13
+ "learning_rate": 0.0003700117150420716,
14
+ "loss": 0.4265,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.08,
19
+ "learning_rate": 0.0003695766709151845,
20
+ "loss": 0.2446,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.11,
25
+ "learning_rate": 0.00036885235529645174,
26
+ "loss": 0.1964,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.15,
31
+ "learning_rate": 0.00036783990382720894,
32
+ "loss": 0.1714,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.19,
37
+ "learning_rate": 0.000366540903911771,
38
+ "loss": 0.1556,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.23,
43
+ "learning_rate": 0.00036495739222857005,
44
+ "loss": 0.1454,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.26,
49
+ "learning_rate": 0.00036309185153688393,
50
+ "loss": 0.1387,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.3,
55
+ "learning_rate": 0.0003609472067841633,
56
+ "loss": 0.133,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.34,
61
+ "learning_rate": 0.0003585268205200591,
62
+ "loss": 0.1289,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.38,
67
+ "learning_rate": 0.0003558344876243421,
68
+ "loss": 0.1228,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.42,
73
+ "learning_rate": 0.0003528744293569798,
74
+ "loss": 0.1208,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.45,
79
+ "learning_rate": 0.00034965128673969907,
80
+ "loss": 0.1167,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.49,
85
+ "learning_rate": 0.0003461701132794125,
86
+ "loss": 0.1131,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.53,
91
+ "learning_rate": 0.0003424363670449164,
92
+ "loss": 0.1113,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.57,
97
+ "learning_rate": 0.00033845590210928363,
98
+ "loss": 0.1088,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.61,
103
+ "learning_rate": 0.00033423495937136847,
104
+ "loss": 0.1081,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.64,
109
+ "learning_rate": 0.00032978015677081454,
110
+ "loss": 0.1054,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.68,
115
+ "learning_rate": 0.00032509847891190683,
116
+ "loss": 0.1029,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.72,
121
+ "learning_rate": 0.0003201972661125376,
122
+ "loss": 0.102,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 0.76,
127
+ "learning_rate": 0.0003150842028954544,
128
+ "loss": 0.0999,
129
+ "step": 4000
130
+ },
131
+ {
132
+ "epoch": 0.79,
133
+ "learning_rate": 0.0003097673059398367,
134
+ "loss": 0.098,
135
+ "step": 4200
136
+ },
137
+ {
138
+ "epoch": 0.83,
139
+ "learning_rate": 0.00030425491151208934,
140
+ "loss": 0.0972,
141
+ "step": 4400
142
+ },
143
+ {
144
+ "epoch": 0.87,
145
+ "learning_rate": 0.00029855566239556173,
146
+ "loss": 0.0968,
147
+ "step": 4600
148
+ },
149
+ {
150
+ "epoch": 0.91,
151
+ "learning_rate": 0.0002926784943396845,
152
+ "loss": 0.0947,
153
+ "step": 4800
154
+ },
155
+ {
156
+ "epoch": 0.95,
157
+ "learning_rate": 0.00028663262204976965,
158
+ "loss": 0.0933,
159
+ "step": 5000
160
+ },
161
+ {
162
+ "epoch": 0.98,
163
+ "learning_rate": 0.00028042752473944106,
164
+ "loss": 0.094,
165
+ "step": 5200
166
+ },
167
+ {
168
+ "epoch": 1.0,
169
+ "eval_accuracy": 0.9297529150299436,
170
+ "eval_auc": 0.8167222019799886,
171
+ "eval_f1": 0.3440490710592986,
172
+ "eval_loss": 0.6011912822723389,
173
+ "eval_mcc": 0.3730152153022164,
174
+ "eval_precision": 0.22835223718675476,
175
+ "eval_recall": 0.697386656717114,
176
+ "eval_runtime": 3756.1015,
177
+ "eval_samples_per_second": 144.201,
178
+ "eval_steps_per_second": 0.687,
179
+ "step": 5289
180
+ },
181
+ {
182
+ "epoch": 1.02,
183
+ "learning_rate": 0.00027407293126834736,
184
+ "loss": 0.0924,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.06,
189
+ "learning_rate": 0.00026757880488845914,
190
+ "loss": 0.0912,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.1,
195
+ "learning_rate": 0.00026095532762286633,
196
+ "loss": 0.0902,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.13,
201
+ "learning_rate": 0.0002542128843015694,
202
+ "loss": 0.09,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.17,
207
+ "learning_rate": 0.00024739655239955165,
208
+ "loss": 0.0901,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.21,
213
+ "learning_rate": 0.00024048348750604134,
214
+ "loss": 0.0889,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.25,
219
+ "learning_rate": 0.00023344905030678293,
220
+ "loss": 0.0872,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.29,
225
+ "learning_rate": 0.00022633877364566408,
226
+ "loss": 0.0873,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.32,
231
+ "learning_rate": 0.0002191638055966375,
232
+ "loss": 0.0866,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.36,
237
+ "learning_rate": 0.00021193539566211078,
238
+ "loss": 0.0864,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.4,
243
+ "learning_rate": 0.00020466487713505563,
244
+ "loss": 0.085,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.44,
249
+ "learning_rate": 0.000197363649329743,
250
+ "loss": 0.086,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.47,
255
+ "learning_rate": 0.00019004315970896487,
256
+ "loss": 0.0847,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 1.51,
261
+ "learning_rate": 0.00018271488593576474,
262
+ "loss": 0.0836,
263
+ "step": 8000
264
+ },
265
+ {
266
+ "epoch": 1.55,
267
+ "learning_rate": 0.0001753903178778174,
268
+ "loss": 0.0844,
269
+ "step": 8200
270
+ },
271
+ {
272
+ "epoch": 1.59,
273
+ "learning_rate": 0.00016808093959267418,
274
+ "loss": 0.0828,
275
+ "step": 8400
276
+ },
277
+ {
278
+ "epoch": 1.63,
279
+ "learning_rate": 0.00016079821132211663,
280
+ "loss": 0.0848,
281
+ "step": 8600
282
+ },
283
+ {
284
+ "epoch": 1.66,
285
+ "learning_rate": 0.00015355355152385142,
286
+ "loss": 0.0826,
287
+ "step": 8800
288
+ },
289
+ {
290
+ "epoch": 1.7,
291
+ "learning_rate": 0.00014635831896871702,
292
+ "loss": 0.0826,
293
+ "step": 9000
294
+ },
295
+ {
296
+ "epoch": 1.74,
297
+ "learning_rate": 0.00013922379493147316,
298
+ "loss": 0.0833,
299
+ "step": 9200
300
+ },
301
+ {
302
+ "epoch": 1.78,
303
+ "learning_rate": 0.0001321611655030941,
304
+ "loss": 0.0816,
305
+ "step": 9400
306
+ },
307
+ {
308
+ "epoch": 1.82,
309
+ "learning_rate": 0.00012518150405229972,
310
+ "loss": 0.0817,
311
+ "step": 9600
312
+ },
313
+ {
314
+ "epoch": 1.85,
315
+ "learning_rate": 0.0001182957538638206,
316
+ "loss": 0.0827,
317
+ "step": 9800
318
+ },
319
+ {
320
+ "epoch": 1.89,
321
+ "learning_rate": 0.00011151471098062111,
322
+ "loss": 0.0807,
323
+ "step": 10000
324
+ },
325
+ {
326
+ "epoch": 1.93,
327
+ "learning_rate": 0.00010484900727697857,
328
+ "loss": 0.0818,
329
+ "step": 10200
330
+ },
331
+ {
332
+ "epoch": 1.97,
333
+ "learning_rate": 9.830909378896018e-05,
334
+ "loss": 0.0811,
335
+ "step": 10400
336
+ },
337
+ {
338
+ "epoch": 2.0,
339
+ "eval_accuracy": 0.9345128329975654,
340
+ "eval_auc": 0.8095464647248594,
341
+ "eval_f1": 0.3534505903118732,
342
+ "eval_loss": 0.646632730960846,
343
+ "eval_mcc": 0.37724366807619925,
344
+ "eval_precision": 0.23907868925380343,
345
+ "eval_recall": 0.6776094956043187,
346
+ "eval_runtime": 3741.5855,
347
+ "eval_samples_per_second": 144.761,
348
+ "eval_steps_per_second": 0.69,
349
+ "step": 10578
350
+ },
351
+ {
352
+ "epoch": 2.0,
353
+ "learning_rate": 9.19368885647311e-05,
354
+ "loss": 0.0811,
355
+ "step": 10600
356
+ },
357
+ {
358
+ "epoch": 2.04,
359
+ "learning_rate": 8.567834861656006e-05,
360
+ "loss": 0.0807,
361
+ "step": 10800
362
+ },
363
+ {
364
+ "epoch": 2.08,
365
+ "learning_rate": 7.960576607056005e-05,
366
+ "loss": 0.0804,
367
+ "step": 11000
368
+ },
369
+ {
370
+ "epoch": 2.12,
371
+ "learning_rate": 7.3667639053756e-05,
372
+ "loss": 0.0814,
373
+ "step": 11200
374
+ },
375
+ {
376
+ "epoch": 2.16,
377
+ "learning_rate": 6.790419095486784e-05,
378
+ "loss": 0.0806,
379
+ "step": 11400
380
+ },
381
+ {
382
+ "epoch": 2.19,
383
+ "learning_rate": 6.232445817974756e-05,
384
+ "loss": 0.0801,
385
+ "step": 11600
386
+ },
387
+ {
388
+ "epoch": 2.23,
389
+ "learning_rate": 5.6937189090319155e-05,
390
+ "loss": 0.0791,
391
+ "step": 11800
392
+ },
393
+ {
394
+ "epoch": 2.27,
395
+ "learning_rate": 5.1750830288181386e-05,
396
+ "loss": 0.0803,
397
+ "step": 12000
398
+ },
399
+ {
400
+ "epoch": 2.31,
401
+ "learning_rate": 4.677351337133321e-05,
402
+ "loss": 0.0802,
403
+ "step": 12200
404
+ },
405
+ {
406
+ "epoch": 2.34,
407
+ "learning_rate": 4.2013042184789433e-05,
408
+ "loss": 0.0786,
409
+ "step": 12400
410
+ },
411
+ {
412
+ "epoch": 2.38,
413
+ "learning_rate": 3.7476880585072944e-05,
414
+ "loss": 0.0792,
415
+ "step": 12600
416
+ },
417
+ {
418
+ "epoch": 2.42,
419
+ "learning_rate": 3.317214073776957e-05,
420
+ "loss": 0.0798,
421
+ "step": 12800
422
+ },
423
+ {
424
+ "epoch": 2.46,
425
+ "learning_rate": 2.9105571966492357e-05,
426
+ "loss": 0.0784,
427
+ "step": 13000
428
+ },
429
+ {
430
+ "epoch": 2.5,
431
+ "learning_rate": 2.5302041894688252e-05,
432
+ "loss": 0.079,
433
+ "step": 13200
434
+ },
435
+ {
436
+ "epoch": 2.53,
437
+ "learning_rate": 2.172929259566275e-05,
438
+ "loss": 0.0787,
439
+ "step": 13400
440
+ },
441
+ {
442
+ "epoch": 2.57,
443
+ "learning_rate": 1.8412655406943377e-05,
444
+ "loss": 0.0791,
445
+ "step": 13600
446
+ },
447
+ {
448
+ "epoch": 2.61,
449
+ "learning_rate": 1.5357330423887333e-05,
450
+ "loss": 0.0795,
451
+ "step": 13800
452
+ },
453
+ {
454
+ "epoch": 2.65,
455
+ "learning_rate": 1.2568108035179009e-05,
456
+ "loss": 0.0785,
457
+ "step": 14000
458
+ },
459
+ {
460
+ "epoch": 2.68,
461
+ "learning_rate": 1.0049361412066457e-05,
462
+ "loss": 0.0777,
463
+ "step": 14200
464
+ },
465
+ {
466
+ "epoch": 2.72,
467
+ "learning_rate": 7.805039651745196e-06,
468
+ "loss": 0.0781,
469
+ "step": 14400
470
+ },
471
+ {
472
+ "epoch": 2.76,
473
+ "learning_rate": 5.838661585640596e-06,
474
+ "loss": 0.0801,
475
+ "step": 14600
476
+ },
477
+ {
478
+ "epoch": 2.8,
479
+ "learning_rate": 4.153310262295508e-06,
480
+ "loss": 0.0782,
481
+ "step": 14800
482
+ },
483
+ {
484
+ "epoch": 2.84,
485
+ "learning_rate": 2.7516281135145465e-06,
486
+ "loss": 0.0782,
487
+ "step": 15000
488
+ },
489
+ {
490
+ "epoch": 2.87,
491
+ "learning_rate": 1.6358128113430393e-06,
492
+ "loss": 0.0775,
493
+ "step": 15200
494
+ },
495
+ {
496
+ "epoch": 2.91,
497
+ "learning_rate": 8.076138223768694e-07,
498
+ "loss": 0.0786,
499
+ "step": 15400
500
+ },
501
+ {
502
+ "epoch": 2.95,
503
+ "learning_rate": 2.6832966480547197e-07,
504
+ "loss": 0.0776,
505
+ "step": 15600
506
+ },
507
+ {
508
+ "epoch": 2.99,
509
+ "learning_rate": 1.8805872488762563e-08,
510
+ "loss": 0.0785,
511
+ "step": 15800
512
+ },
513
+ {
514
+ "epoch": 3.0,
515
+ "eval_accuracy": 0.9350567264686818,
516
+ "eval_auc": 0.8085491733369476,
517
+ "eval_f1": 0.35447033515145,
518
+ "eval_loss": 0.6618000268936157,
519
+ "eval_mcc": 0.3776148132284474,
520
+ "eval_precision": 0.24034378012664542,
521
+ "eval_recall": 0.6749850523517478,
522
+ "eval_runtime": 3760.204,
523
+ "eval_samples_per_second": 144.044,
524
+ "eval_steps_per_second": 0.686,
525
+ "step": 15867
526
+ }
527
+ ],
528
+ "logging_steps": 200,
529
+ "max_steps": 15867,
530
+ "num_train_epochs": 3,
531
+ "save_steps": 500,
532
+ "total_flos": 1.20195531145082e+18,
533
+ "trial_name": null,
534
+ "trial_params": null
535
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b915c4deb5ba56cacb3593b78264a7c9bd0e44e5e5af7a361d94ad20b8ca1c89
3
+ size 4155
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-15867/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t12_35M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 2,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "query",
20
+ "key",
21
+ "value",
22
+ "EsmSelfOutput.dense",
23
+ "EsmIntermediate.dense",
24
+ "EsmOutput.dense",
25
+ "EsmContactPredictionHead.regression",
26
+ "classifier"
27
+ ],
28
+ "task_type": "TOKEN_CLS"
29
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43b452dd060767dd70f47cedeaf44542b697374130a372bf310a1f45bd6c156f
3
+ size 316315
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0246b16ad2a4ff5346cc7cd436d9865710a83f218ce64578c913184989a1485a
3
+ size 613061
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd9c5833a930482059798aab998f33fa3d0167a6fcf4b32dba0ae0b19178c0c2
3
+ size 14575
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:081e2d3bd93b47a1a918240d8077826ebb0b43206a9f65cc14d171889238fc62
3
+ size 627
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/trainer_state.json ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.3440490710592986,
3
+ "best_model_checkpoint": "esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289",
4
+ "epoch": 1.0,
5
+ "eval_steps": 500,
6
+ "global_step": 5289,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.04,
13
+ "learning_rate": 0.0003700117150420716,
14
+ "loss": 0.4265,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.08,
19
+ "learning_rate": 0.0003695766709151845,
20
+ "loss": 0.2446,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.11,
25
+ "learning_rate": 0.00036885235529645174,
26
+ "loss": 0.1964,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.15,
31
+ "learning_rate": 0.00036783990382720894,
32
+ "loss": 0.1714,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.19,
37
+ "learning_rate": 0.000366540903911771,
38
+ "loss": 0.1556,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.23,
43
+ "learning_rate": 0.00036495739222857005,
44
+ "loss": 0.1454,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.26,
49
+ "learning_rate": 0.00036309185153688393,
50
+ "loss": 0.1387,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.3,
55
+ "learning_rate": 0.0003609472067841633,
56
+ "loss": 0.133,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.34,
61
+ "learning_rate": 0.0003585268205200591,
62
+ "loss": 0.1289,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.38,
67
+ "learning_rate": 0.0003558344876243421,
68
+ "loss": 0.1228,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.42,
73
+ "learning_rate": 0.0003528744293569798,
74
+ "loss": 0.1208,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.45,
79
+ "learning_rate": 0.00034965128673969907,
80
+ "loss": 0.1167,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.49,
85
+ "learning_rate": 0.0003461701132794125,
86
+ "loss": 0.1131,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.53,
91
+ "learning_rate": 0.0003424363670449164,
92
+ "loss": 0.1113,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.57,
97
+ "learning_rate": 0.00033845590210928363,
98
+ "loss": 0.1088,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.61,
103
+ "learning_rate": 0.00033423495937136847,
104
+ "loss": 0.1081,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.64,
109
+ "learning_rate": 0.00032978015677081454,
110
+ "loss": 0.1054,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.68,
115
+ "learning_rate": 0.00032509847891190683,
116
+ "loss": 0.1029,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.72,
121
+ "learning_rate": 0.0003201972661125376,
122
+ "loss": 0.102,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 0.76,
127
+ "learning_rate": 0.0003150842028954544,
128
+ "loss": 0.0999,
129
+ "step": 4000
130
+ },
131
+ {
132
+ "epoch": 0.79,
133
+ "learning_rate": 0.0003097673059398367,
134
+ "loss": 0.098,
135
+ "step": 4200
136
+ },
137
+ {
138
+ "epoch": 0.83,
139
+ "learning_rate": 0.00030425491151208934,
140
+ "loss": 0.0972,
141
+ "step": 4400
142
+ },
143
+ {
144
+ "epoch": 0.87,
145
+ "learning_rate": 0.00029855566239556173,
146
+ "loss": 0.0968,
147
+ "step": 4600
148
+ },
149
+ {
150
+ "epoch": 0.91,
151
+ "learning_rate": 0.0002926784943396845,
152
+ "loss": 0.0947,
153
+ "step": 4800
154
+ },
155
+ {
156
+ "epoch": 0.95,
157
+ "learning_rate": 0.00028663262204976965,
158
+ "loss": 0.0933,
159
+ "step": 5000
160
+ },
161
+ {
162
+ "epoch": 0.98,
163
+ "learning_rate": 0.00028042752473944106,
164
+ "loss": 0.094,
165
+ "step": 5200
166
+ },
167
+ {
168
+ "epoch": 1.0,
169
+ "eval_accuracy": 0.9297529150299436,
170
+ "eval_auc": 0.8167222019799886,
171
+ "eval_f1": 0.3440490710592986,
172
+ "eval_loss": 0.6011912822723389,
173
+ "eval_mcc": 0.3730152153022164,
174
+ "eval_precision": 0.22835223718675476,
175
+ "eval_recall": 0.697386656717114,
176
+ "eval_runtime": 3756.1015,
177
+ "eval_samples_per_second": 144.201,
178
+ "eval_steps_per_second": 0.687,
179
+ "step": 5289
180
+ }
181
+ ],
182
+ "logging_steps": 200,
183
+ "max_steps": 15867,
184
+ "num_train_epochs": 3,
185
+ "save_steps": 500,
186
+ "total_flos": 4.006517704836067e+17,
187
+ "trial_name": null,
188
+ "trial_params": null
189
+ }
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b915c4deb5ba56cacb3593b78264a7c9bd0e44e5e5af7a361d94ad20b8ca1c89
3
+ size 4155
esm2_t12_35M_qlora_binding_sites_2023-09-30_06-27-21/checkpoint-5289/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>