Hugging Face's logo

AmelieSchreiber
/
esm2_t6_8m_qlora_binding_sites_v1

Model card Files Community
AmelieSchreiber commited on
Commit
692f69c
·
1 Parent(s): 32ec1db

Upload 48 files

Browse files
Files changed (48) hide show
  1. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/README.md +21 -0
  2. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/adapter_config.json +29 -0
  3. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/adapter_model.bin +3 -0
  4. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/added_tokens.json +7 -0
  5. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/optimizer.pt +3 -0
  6. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/rng_state.pth +3 -0
  7. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/scheduler.pt +3 -0
  8. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/special_tokens_map.json +7 -0
  9. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/tokenizer_config.json +54 -0
  10. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/trainer_state.json +415 -0
  11. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/training_args.bin +3 -0
  12. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/vocab.txt +33 -0
  13. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/README.md +21 -0
  14. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/adapter_config.json +29 -0
  15. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/adapter_model.bin +3 -0
  16. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/added_tokens.json +7 -0
  17. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/optimizer.pt +3 -0
  18. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/rng_state.pth +3 -0
  19. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/scheduler.pt +3 -0
  20. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/special_tokens_map.json +7 -0
  21. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/tokenizer_config.json +54 -0
  22. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/trainer_state.json +543 -0
  23. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/training_args.bin +3 -0
  24. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/vocab.txt +33 -0
  25. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/README.md +21 -0
  26. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/adapter_config.json +29 -0
  27. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/adapter_model.bin +3 -0
  28. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/added_tokens.json +7 -0
  29. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/optimizer.pt +3 -0
  30. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/rng_state.pth +3 -0
  31. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/scheduler.pt +3 -0
  32. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/special_tokens_map.json +7 -0
  33. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/tokenizer_config.json +54 -0
  34. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/trainer_state.json +147 -0
  35. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/training_args.bin +3 -0
  36. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/vocab.txt +33 -0
  37. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/README.md +21 -0
  38. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/adapter_config.json +29 -0
  39. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/adapter_model.bin +3 -0
  40. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/added_tokens.json +7 -0
  41. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/optimizer.pt +3 -0
  42. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/rng_state.pth +3 -0
  43. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/scheduler.pt +3 -0
  44. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/special_tokens_map.json +7 -0
  45. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/tokenizer_config.json +54 -0
  46. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/trainer_state.json +281 -0
  47. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/training_args.bin +3 -0
  48. esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/vocab.txt +33 -0
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 2,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "query",
20
+ "key",
21
+ "value",
22
+ "EsmSelfOutput.dense",
23
+ "EsmIntermediate.dense",
24
+ "EsmOutput.dense",
25
+ "EsmContactPredictionHead.regression",
26
+ "classifier"
27
+ ],
28
+ "task_type": "TOKEN_CLS"
29
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3e861c1a99b016ce76e69794f42346cccbdc6bf95658b3c3bc4d3cf0b8b999c
3
+ size 114991
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84927c18255420a7ed70e70a5f7bf68e48b1a0b356f64c733677a7eeb9ad36c5
3
+ size 218073
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:baa211bb85c9638f95b353de5eefb72e14175247f5be85ccfed47b88c3f91a13
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3676a61e287c72a94f887301dbd12aad47cd3033c71e16308c1190c5c28aecd
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/trainer_state.json ADDED
@@ -0,0 +1,415 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.21386215966220054,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878",
4
+ "epoch": 3.0,
5
+ "eval_steps": 500,
6
+ "global_step": 11817,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.5009,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3725,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3363,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3171,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3041,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.2954,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2872,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2804,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2754,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2696,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2675,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2642,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.262,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2592,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2565,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2542,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2509,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2509,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195249716710097,
122
+ "loss": 0.2464,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8625347674451358,
128
+ "eval_auc": 0.8222331548742136,
129
+ "eval_f1": 0.19848246486644372,
130
+ "eval_loss": 0.41490185260772705,
131
+ "eval_mcc": 0.2639007297474409,
132
+ "eval_precision": 0.11370668247419904,
133
+ "eval_recall": 0.7800926533683039,
134
+ "eval_runtime": 1163.712,
135
+ "eval_samples_per_second": 109.693,
136
+ "eval_steps_per_second": 1.714,
137
+ "step": 3939
138
+ },
139
+ {
140
+ "epoch": 1.02,
141
+ "learning_rate": 0.00031434720713917336,
142
+ "loss": 0.2484,
143
+ "step": 4000
144
+ },
145
+ {
146
+ "epoch": 1.07,
147
+ "learning_rate": 0.0003089913128054233,
148
+ "loss": 0.2437,
149
+ "step": 4200
150
+ },
151
+ {
152
+ "epoch": 1.12,
153
+ "learning_rate": 0.0003034119838843777,
154
+ "loss": 0.242,
155
+ "step": 4400
156
+ },
157
+ {
158
+ "epoch": 1.17,
159
+ "learning_rate": 0.00029764449888790783,
160
+ "loss": 0.2423,
161
+ "step": 4600
162
+ },
163
+ {
164
+ "epoch": 1.22,
165
+ "learning_rate": 0.0002916980283940693,
166
+ "loss": 0.2387,
167
+ "step": 4800
168
+ },
169
+ {
170
+ "epoch": 1.27,
171
+ "learning_rate": 0.00028558202757645164,
172
+ "loss": 0.239,
173
+ "step": 5000
174
+ },
175
+ {
176
+ "epoch": 1.32,
177
+ "learning_rate": 0.0002793062211699992,
178
+ "loss": 0.2376,
179
+ "step": 5200
180
+ },
181
+ {
182
+ "epoch": 1.37,
183
+ "learning_rate": 0.0002728805880082148,
184
+ "loss": 0.2384,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.42,
189
+ "learning_rate": 0.00026631534515633606,
190
+ "loss": 0.2355,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.47,
195
+ "learning_rate": 0.0002596209316657108,
196
+ "loss": 0.2352,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.52,
201
+ "learning_rate": 0.00025280799197520415,
202
+ "loss": 0.2343,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.57,
207
+ "learning_rate": 0.0002458873589860299,
208
+ "loss": 0.231,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.62,
213
+ "learning_rate": 0.00023890534556422294,
214
+ "loss": 0.2339,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.68,
219
+ "learning_rate": 0.00023180289180308998,
220
+ "loss": 0.2332,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.73,
225
+ "learning_rate": 0.00022462614386106907,
226
+ "loss": 0.232,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.78,
231
+ "learning_rate": 0.00021738651311196128,
232
+ "loss": 0.231,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.83,
237
+ "learning_rate": 0.00021013207456600314,
238
+ "loss": 0.2305,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.88,
243
+ "learning_rate": 0.00020280146390924225,
244
+ "loss": 0.2298,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.93,
249
+ "learning_rate": 0.00019544267273437384,
250
+ "loss": 0.2302,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.98,
255
+ "learning_rate": 0.0001880674018726795,
256
+ "loss": 0.2295,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 2.0,
261
+ "eval_accuracy": 0.8789903474277315,
262
+ "eval_auc": 0.8180827968848764,
263
+ "eval_f1": 0.21386215966220054,
264
+ "eval_loss": 0.4392629563808441,
265
+ "eval_mcc": 0.27446788968929925,
266
+ "eval_precision": 0.12459111498869992,
267
+ "eval_recall": 0.7543961461123881,
268
+ "eval_runtime": 1167.9434,
269
+ "eval_samples_per_second": 109.296,
270
+ "eval_steps_per_second": 1.708,
271
+ "step": 7878
272
+ },
273
+ {
274
+ "epoch": 2.03,
275
+ "learning_rate": 0.00018068737835893393,
276
+ "loss": 0.2261,
277
+ "step": 8000
278
+ },
279
+ {
280
+ "epoch": 2.08,
281
+ "learning_rate": 0.00017331433678485712,
282
+ "loss": 0.2277,
283
+ "step": 8200
284
+ },
285
+ {
286
+ "epoch": 2.13,
287
+ "learning_rate": 0.00016596000064054934,
288
+ "loss": 0.2275,
289
+ "step": 8400
290
+ },
291
+ {
292
+ "epoch": 2.18,
293
+ "learning_rate": 0.0001586360636735797,
294
+ "loss": 0.2267,
295
+ "step": 8600
296
+ },
297
+ {
298
+ "epoch": 2.23,
299
+ "learning_rate": 0.0001513541712953658,
300
+ "loss": 0.2243,
301
+ "step": 8800
302
+ },
303
+ {
304
+ "epoch": 2.28,
305
+ "learning_rate": 0.00014412590206441025,
306
+ "loss": 0.2266,
307
+ "step": 9000
308
+ },
309
+ {
310
+ "epoch": 2.34,
311
+ "learning_rate": 0.00013696274927583722,
312
+ "loss": 0.2243,
313
+ "step": 9200
314
+ },
315
+ {
316
+ "epoch": 2.39,
317
+ "learning_rate": 0.00012987610268650092,
318
+ "loss": 0.2253,
319
+ "step": 9400
320
+ },
321
+ {
322
+ "epoch": 2.44,
323
+ "learning_rate": 0.00012287723040472557,
324
+ "loss": 0.2255,
325
+ "step": 9600
326
+ },
327
+ {
328
+ "epoch": 2.49,
329
+ "learning_rate": 0.00011597726097347204,
330
+ "loss": 0.2254,
331
+ "step": 9800
332
+ },
333
+ {
334
+ "epoch": 2.54,
335
+ "learning_rate": 0.00010918716567542043,
336
+ "loss": 0.2241,
337
+ "step": 10000
338
+ },
339
+ {
340
+ "epoch": 2.59,
341
+ "learning_rate": 0.0001025177410881037,
342
+ "loss": 0.223,
343
+ "step": 10200
344
+ },
345
+ {
346
+ "epoch": 2.64,
347
+ "learning_rate": 9.597959191683088e-05,
348
+ "loss": 0.2217,
349
+ "step": 10400
350
+ },
351
+ {
352
+ "epoch": 2.69,
353
+ "learning_rate": 8.958311413269684e-05,
354
+ "loss": 0.222,
355
+ "step": 10600
356
+ },
357
+ {
358
+ "epoch": 2.74,
359
+ "learning_rate": 8.333847844248891e-05,
360
+ "loss": 0.2219,
361
+ "step": 10800
362
+ },
363
+ {
364
+ "epoch": 2.79,
365
+ "learning_rate": 7.725561411677462e-05,
366
+ "loss": 0.2218,
367
+ "step": 11000
368
+ },
369
+ {
370
+ "epoch": 2.84,
371
+ "learning_rate": 7.134419320188491e-05,
372
+ "loss": 0.2233,
373
+ "step": 11200
374
+ },
375
+ {
376
+ "epoch": 2.89,
377
+ "learning_rate": 6.564180298866504e-05,
378
+ "loss": 0.2261,
379
+ "step": 11400
380
+ },
381
+ {
382
+ "epoch": 2.94,
383
+ "learning_rate": 6.010020772997792e-05,
384
+ "loss": 0.2225,
385
+ "step": 11600
386
+ },
387
+ {
388
+ "epoch": 3.0,
389
+ "learning_rate": 5.475733380194338e-05,
390
+ "loss": 0.2176,
391
+ "step": 11800
392
+ },
393
+ {
394
+ "epoch": 3.0,
395
+ "eval_accuracy": 0.8735072413749101,
396
+ "eval_auc": 0.8195797310899098,
397
+ "eval_f1": 0.20841207680022153,
398
+ "eval_loss": 0.4386225640773773,
399
+ "eval_mcc": 0.2707498152735772,
400
+ "eval_precision": 0.12068424210520438,
401
+ "eval_recall": 0.7631916070570107,
402
+ "eval_runtime": 1165.7733,
403
+ "eval_samples_per_second": 109.499,
404
+ "eval_steps_per_second": 1.711,
405
+ "step": 11817
406
+ }
407
+ ],
408
+ "logging_steps": 200,
409
+ "max_steps": 15756,
410
+ "num_train_epochs": 4,
411
+ "save_steps": 500,
412
+ "total_flos": 6.077187662375136e+16,
413
+ "trial_name": null,
414
+ "trial_params": null
415
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19560f827f37de980ef9907eb4a996feaadf6e4f6a723cacbef99fc7a0ba1b25
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-11817/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 2,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "query",
20
+ "key",
21
+ "value",
22
+ "EsmSelfOutput.dense",
23
+ "EsmIntermediate.dense",
24
+ "EsmOutput.dense",
25
+ "EsmContactPredictionHead.regression",
26
+ "classifier"
27
+ ],
28
+ "task_type": "TOKEN_CLS"
29
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3075ce6140607858becdd4528bfbba621d0fe5981ed48db8ecdfdd6ed1679e3
3
+ size 114991
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a02b58d7f2080fec37a419183d7fefebe47ae919ad6368dd61a35b4e860a40c0
3
+ size 218073
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8a9a8ddf2bef1b580758f72457a9be5be71f4dc25e234839bea7418a3d9bc67
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86ab8ca8d941f7a9dc0e1bfb3c414eb3afc0b3e1513ea8a7ce0db94586cd403d
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/trainer_state.json ADDED
@@ -0,0 +1,543 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.21386215966220054,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878",
4
+ "epoch": 4.0,
5
+ "eval_steps": 500,
6
+ "global_step": 15756,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.5009,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3725,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3363,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3171,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3041,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.2954,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2872,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2804,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2754,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2696,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2675,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2642,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.262,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2592,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2565,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2542,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2509,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2509,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195249716710097,
122
+ "loss": 0.2464,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8625347674451358,
128
+ "eval_auc": 0.8222331548742136,
129
+ "eval_f1": 0.19848246486644372,
130
+ "eval_loss": 0.41490185260772705,
131
+ "eval_mcc": 0.2639007297474409,
132
+ "eval_precision": 0.11370668247419904,
133
+ "eval_recall": 0.7800926533683039,
134
+ "eval_runtime": 1163.712,
135
+ "eval_samples_per_second": 109.693,
136
+ "eval_steps_per_second": 1.714,
137
+ "step": 3939
138
+ },
139
+ {
140
+ "epoch": 1.02,
141
+ "learning_rate": 0.00031434720713917336,
142
+ "loss": 0.2484,
143
+ "step": 4000
144
+ },
145
+ {
146
+ "epoch": 1.07,
147
+ "learning_rate": 0.0003089913128054233,
148
+ "loss": 0.2437,
149
+ "step": 4200
150
+ },
151
+ {
152
+ "epoch": 1.12,
153
+ "learning_rate": 0.0003034119838843777,
154
+ "loss": 0.242,
155
+ "step": 4400
156
+ },
157
+ {
158
+ "epoch": 1.17,
159
+ "learning_rate": 0.00029764449888790783,
160
+ "loss": 0.2423,
161
+ "step": 4600
162
+ },
163
+ {
164
+ "epoch": 1.22,
165
+ "learning_rate": 0.0002916980283940693,
166
+ "loss": 0.2387,
167
+ "step": 4800
168
+ },
169
+ {
170
+ "epoch": 1.27,
171
+ "learning_rate": 0.00028558202757645164,
172
+ "loss": 0.239,
173
+ "step": 5000
174
+ },
175
+ {
176
+ "epoch": 1.32,
177
+ "learning_rate": 0.0002793062211699992,
178
+ "loss": 0.2376,
179
+ "step": 5200
180
+ },
181
+ {
182
+ "epoch": 1.37,
183
+ "learning_rate": 0.0002728805880082148,
184
+ "loss": 0.2384,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.42,
189
+ "learning_rate": 0.00026631534515633606,
190
+ "loss": 0.2355,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.47,
195
+ "learning_rate": 0.0002596209316657108,
196
+ "loss": 0.2352,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.52,
201
+ "learning_rate": 0.00025280799197520415,
202
+ "loss": 0.2343,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.57,
207
+ "learning_rate": 0.0002458873589860299,
208
+ "loss": 0.231,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.62,
213
+ "learning_rate": 0.00023890534556422294,
214
+ "loss": 0.2339,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.68,
219
+ "learning_rate": 0.00023180289180308998,
220
+ "loss": 0.2332,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.73,
225
+ "learning_rate": 0.00022462614386106907,
226
+ "loss": 0.232,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.78,
231
+ "learning_rate": 0.00021738651311196128,
232
+ "loss": 0.231,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.83,
237
+ "learning_rate": 0.00021013207456600314,
238
+ "loss": 0.2305,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.88,
243
+ "learning_rate": 0.00020280146390924225,
244
+ "loss": 0.2298,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.93,
249
+ "learning_rate": 0.00019544267273437384,
250
+ "loss": 0.2302,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.98,
255
+ "learning_rate": 0.0001880674018726795,
256
+ "loss": 0.2295,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 2.0,
261
+ "eval_accuracy": 0.8789903474277315,
262
+ "eval_auc": 0.8180827968848764,
263
+ "eval_f1": 0.21386215966220054,
264
+ "eval_loss": 0.4392629563808441,
265
+ "eval_mcc": 0.27446788968929925,
266
+ "eval_precision": 0.12459111498869992,
267
+ "eval_recall": 0.7543961461123881,
268
+ "eval_runtime": 1167.9434,
269
+ "eval_samples_per_second": 109.296,
270
+ "eval_steps_per_second": 1.708,
271
+ "step": 7878
272
+ },
273
+ {
274
+ "epoch": 2.03,
275
+ "learning_rate": 0.00018068737835893393,
276
+ "loss": 0.2261,
277
+ "step": 8000
278
+ },
279
+ {
280
+ "epoch": 2.08,
281
+ "learning_rate": 0.00017331433678485712,
282
+ "loss": 0.2277,
283
+ "step": 8200
284
+ },
285
+ {
286
+ "epoch": 2.13,
287
+ "learning_rate": 0.00016596000064054934,
288
+ "loss": 0.2275,
289
+ "step": 8400
290
+ },
291
+ {
292
+ "epoch": 2.18,
293
+ "learning_rate": 0.0001586360636735797,
294
+ "loss": 0.2267,
295
+ "step": 8600
296
+ },
297
+ {
298
+ "epoch": 2.23,
299
+ "learning_rate": 0.0001513541712953658,
300
+ "loss": 0.2243,
301
+ "step": 8800
302
+ },
303
+ {
304
+ "epoch": 2.28,
305
+ "learning_rate": 0.00014412590206441025,
306
+ "loss": 0.2266,
307
+ "step": 9000
308
+ },
309
+ {
310
+ "epoch": 2.34,
311
+ "learning_rate": 0.00013696274927583722,
312
+ "loss": 0.2243,
313
+ "step": 9200
314
+ },
315
+ {
316
+ "epoch": 2.39,
317
+ "learning_rate": 0.00012987610268650092,
318
+ "loss": 0.2253,
319
+ "step": 9400
320
+ },
321
+ {
322
+ "epoch": 2.44,
323
+ "learning_rate": 0.00012287723040472557,
324
+ "loss": 0.2255,
325
+ "step": 9600
326
+ },
327
+ {
328
+ "epoch": 2.49,
329
+ "learning_rate": 0.00011597726097347204,
330
+ "loss": 0.2254,
331
+ "step": 9800
332
+ },
333
+ {
334
+ "epoch": 2.54,
335
+ "learning_rate": 0.00010918716567542043,
336
+ "loss": 0.2241,
337
+ "step": 10000
338
+ },
339
+ {
340
+ "epoch": 2.59,
341
+ "learning_rate": 0.0001025177410881037,
342
+ "loss": 0.223,
343
+ "step": 10200
344
+ },
345
+ {
346
+ "epoch": 2.64,
347
+ "learning_rate": 9.597959191683088e-05,
348
+ "loss": 0.2217,
349
+ "step": 10400
350
+ },
351
+ {
352
+ "epoch": 2.69,
353
+ "learning_rate": 8.958311413269684e-05,
354
+ "loss": 0.222,
355
+ "step": 10600
356
+ },
357
+ {
358
+ "epoch": 2.74,
359
+ "learning_rate": 8.333847844248891e-05,
360
+ "loss": 0.2219,
361
+ "step": 10800
362
+ },
363
+ {
364
+ "epoch": 2.79,
365
+ "learning_rate": 7.725561411677462e-05,
366
+ "loss": 0.2218,
367
+ "step": 11000
368
+ },
369
+ {
370
+ "epoch": 2.84,
371
+ "learning_rate": 7.134419320188491e-05,
372
+ "loss": 0.2233,
373
+ "step": 11200
374
+ },
375
+ {
376
+ "epoch": 2.89,
377
+ "learning_rate": 6.564180298866504e-05,
378
+ "loss": 0.2261,
379
+ "step": 11400
380
+ },
381
+ {
382
+ "epoch": 2.94,
383
+ "learning_rate": 6.010020772997792e-05,
384
+ "loss": 0.2225,
385
+ "step": 11600
386
+ },
387
+ {
388
+ "epoch": 3.0,
389
+ "learning_rate": 5.475733380194338e-05,
390
+ "loss": 0.2176,
391
+ "step": 11800
392
+ },
393
+ {
394
+ "epoch": 3.0,
395
+ "eval_accuracy": 0.8735072413749101,
396
+ "eval_auc": 0.8195797310899098,
397
+ "eval_f1": 0.20841207680022153,
398
+ "eval_loss": 0.4386225640773773,
399
+ "eval_mcc": 0.2707498152735772,
400
+ "eval_precision": 0.12068424210520438,
401
+ "eval_recall": 0.7631916070570107,
402
+ "eval_runtime": 1165.7733,
403
+ "eval_samples_per_second": 109.499,
404
+ "eval_steps_per_second": 1.711,
405
+ "step": 11817
406
+ },
407
+ {
408
+ "epoch": 3.05,
409
+ "learning_rate": 4.962167663063128e-05,
410
+ "loss": 0.2223,
411
+ "step": 12000
412
+ },
413
+ {
414
+ "epoch": 3.1,
415
+ "learning_rate": 4.4701402157519476e-05,
416
+ "loss": 0.2204,
417
+ "step": 12200
418
+ },
419
+ {
420
+ "epoch": 3.15,
421
+ "learning_rate": 4.000433385525458e-05,
422
+ "loss": 0.2224,
423
+ "step": 12400
424
+ },
425
+ {
426
+ "epoch": 3.2,
427
+ "learning_rate": 3.553794028795492e-05,
428
+ "loss": 0.2239,
429
+ "step": 12600
430
+ },
431
+ {
432
+ "epoch": 3.25,
433
+ "learning_rate": 3.130932323583492e-05,
434
+ "loss": 0.2188,
435
+ "step": 12800
436
+ },
437
+ {
438
+ "epoch": 3.3,
439
+ "learning_rate": 2.7325206403033605e-05,
440
+ "loss": 0.2214,
441
+ "step": 13000
442
+ },
443
+ {
444
+ "epoch": 3.35,
445
+ "learning_rate": 2.359192472660235e-05,
446
+ "loss": 0.2216,
447
+ "step": 13200
448
+ },
449
+ {
450
+ "epoch": 3.4,
451
+ "learning_rate": 2.0132148821294626e-05,
452
+ "loss": 0.2207,
453
+ "step": 13400
454
+ },
455
+ {
456
+ "epoch": 3.45,
457
+ "learning_rate": 1.691661291092208e-05,
458
+ "loss": 0.2199,
459
+ "step": 13600
460
+ },
461
+ {
462
+ "epoch": 3.5,
463
+ "learning_rate": 1.3968462320313953e-05,
464
+ "loss": 0.219,
465
+ "step": 13800
466
+ },
467
+ {
468
+ "epoch": 3.55,
469
+ "learning_rate": 1.1292384750457188e-05,
470
+ "loss": 0.2219,
471
+ "step": 14000
472
+ },
473
+ {
474
+ "epoch": 3.6,
475
+ "learning_rate": 8.892635293172929e-06,
476
+ "loss": 0.2202,
477
+ "step": 14200
478
+ },
479
+ {
480
+ "epoch": 3.66,
481
+ "learning_rate": 6.773029665315992e-06,
482
+ "loss": 0.2202,
483
+ "step": 14400
484
+ },
485
+ {
486
+ "epoch": 3.71,
487
+ "learning_rate": 4.936938141602164e-06,
488
+ "loss": 0.2168,
489
+ "step": 14600
490
+ },
491
+ {
492
+ "epoch": 3.76,
493
+ "learning_rate": 3.387280195709328e-06,
494
+ "loss": 0.221,
495
+ "step": 14800
496
+ },
497
+ {
498
+ "epoch": 3.81,
499
+ "learning_rate": 2.1321015191903147e-06,
500
+ "loss": 0.2195,
501
+ "step": 15000
502
+ },
503
+ {
504
+ "epoch": 3.86,
505
+ "learning_rate": 1.1607847254776198e-06,
506
+ "loss": 0.2214,
507
+ "step": 15200
508
+ },
509
+ {
510
+ "epoch": 3.91,
511
+ "learning_rate": 4.819057748199279e-07,
512
+ "loss": 0.2202,
513
+ "step": 15400
514
+ },
515
+ {
516
+ "epoch": 3.96,
517
+ "learning_rate": 9.654411734337413e-08,
518
+ "loss": 0.2222,
519
+ "step": 15600
520
+ },
521
+ {
522
+ "epoch": 4.0,
523
+ "eval_accuracy": 0.8741850393920096,
524
+ "eval_auc": 0.819488037818028,
525
+ "eval_f1": 0.20910544751589633,
526
+ "eval_loss": 0.43909892439842224,
527
+ "eval_mcc": 0.27126529185336096,
528
+ "eval_precision": 0.12117210076100346,
529
+ "eval_recall": 0.7622953120147897,
530
+ "eval_runtime": 1162.7299,
531
+ "eval_samples_per_second": 109.786,
532
+ "eval_steps_per_second": 1.716,
533
+ "step": 15756
534
+ }
535
+ ],
536
+ "logging_steps": 200,
537
+ "max_steps": 15756,
538
+ "num_train_epochs": 4,
539
+ "save_steps": 500,
540
+ "total_flos": 8.102916883166848e+16,
541
+ "trial_name": null,
542
+ "trial_params": null
543
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19560f827f37de980ef9907eb4a996feaadf6e4f6a723cacbef99fc7a0ba1b25
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-15756/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 2,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "query",
20
+ "key",
21
+ "value",
22
+ "EsmSelfOutput.dense",
23
+ "EsmIntermediate.dense",
24
+ "EsmOutput.dense",
25
+ "EsmContactPredictionHead.regression",
26
+ "classifier"
27
+ ],
28
+ "task_type": "TOKEN_CLS"
29
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44501128da965af2da76593044b359220b9cbd2c5b399bc9de7aac4197d83273
3
+ size 114991
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:550477f400ea5edc292478c534a1dae4e96fbbd3c25f393b7ce864fbce100b69
3
+ size 218073
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4867cc190d63a633b96f2672fb906a1c6c53f31c353db36e79d6433f4038473
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4bc87ecdf29595d0187df2e3c0721fbf08cc9aa799fab4c51aafa53363300bd1
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/trainer_state.json ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.19848246486644372,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939",
4
+ "epoch": 1.0,
5
+ "eval_steps": 500,
6
+ "global_step": 3939,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.5009,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3725,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3363,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3171,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3041,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.2954,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2872,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2804,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2754,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2696,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2675,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2642,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.262,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2592,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2565,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2542,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2509,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2509,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195249716710097,
122
+ "loss": 0.2464,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8625347674451358,
128
+ "eval_auc": 0.8222331548742136,
129
+ "eval_f1": 0.19848246486644372,
130
+ "eval_loss": 0.41490185260772705,
131
+ "eval_mcc": 0.2639007297474409,
132
+ "eval_precision": 0.11370668247419904,
133
+ "eval_recall": 0.7800926533683039,
134
+ "eval_runtime": 1163.712,
135
+ "eval_samples_per_second": 109.693,
136
+ "eval_steps_per_second": 1.714,
137
+ "step": 3939
138
+ }
139
+ ],
140
+ "logging_steps": 200,
141
+ "max_steps": 15756,
142
+ "num_train_epochs": 4,
143
+ "save_steps": 500,
144
+ "total_flos": 2.0257292207917116e+16,
145
+ "trial_name": null,
146
+ "trial_params": null
147
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19560f827f37de980ef9907eb4a996feaadf6e4f6a723cacbef99fc7a0ba1b25
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-3939/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/README.md ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - quant_method: bitsandbytes
9
+ - load_in_8bit: False
10
+ - load_in_4bit: True
11
+ - llm_int8_threshold: 6.0
12
+ - llm_int8_skip_modules: None
13
+ - llm_int8_enable_fp32_cpu_offload: False
14
+ - llm_int8_has_fp16_weight: False
15
+ - bnb_4bit_quant_type: nf4
16
+ - bnb_4bit_use_double_quant: True
17
+ - bnb_4bit_compute_dtype: bfloat16
18
+ ### Framework versions
19
+
20
+
21
+ - PEFT 0.6.0.dev0
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/adapter_config.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "facebook/esm2_t6_8M_UR50D",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "lora_alpha": 1,
12
+ "lora_dropout": 0.5,
13
+ "modules_to_save": null,
14
+ "peft_type": "LORA",
15
+ "r": 2,
16
+ "rank_pattern": {},
17
+ "revision": null,
18
+ "target_modules": [
19
+ "query",
20
+ "key",
21
+ "value",
22
+ "EsmSelfOutput.dense",
23
+ "EsmIntermediate.dense",
24
+ "EsmOutput.dense",
25
+ "EsmContactPredictionHead.regression",
26
+ "classifier"
27
+ ],
28
+ "task_type": "TOKEN_CLS"
29
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d6bae6aa0fe5623cd84eb369c0e9e8027fbbc24839cc196e742de63964ef886
3
+ size 114991
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/added_tokens.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "<cls>": 0,
3
+ "<eos>": 2,
4
+ "<mask>": 32,
5
+ "<pad>": 1,
6
+ "<unk>": 3
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b24037eda5de7e73c7d7b3ffec29293e2178a5511cb51b0d27fca0ff899fabe
3
+ size 218073
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:370252d5d09ed129e985231b3e21fc550a6f7a7c660b7c74edd054a4e7ae94ed
3
+ size 14575
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b402c76a39b9f4986855645efcbcac8cfd51de5b448f17a8868056e810e6bdf0
3
+ size 627
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "<cls>",
3
+ "eos_token": "<eos>",
4
+ "mask_token": "<mask>",
5
+ "pad_token": "<pad>",
6
+ "unk_token": "<unk>"
7
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/tokenizer_config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [],
45
+ "clean_up_tokenization_spaces": true,
46
+ "cls_token": "<cls>",
47
+ "eos_token": "<eos>",
48
+ "mask_token": "<mask>",
49
+ "model_max_length": 1024,
50
+ "pad_token": "<pad>",
51
+ "tokenizer_class": "EsmTokenizer",
52
+ "tokenizer_file": null,
53
+ "unk_token": "<unk>"
54
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/trainer_state.json ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.21386215966220054,
3
+ "best_model_checkpoint": "esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878",
4
+ "epoch": 2.0,
5
+ "eval_steps": 500,
6
+ "global_step": 7878,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "learning_rate": 0.0003700096638060586,
14
+ "loss": 0.5009,
15
+ "step": 200
16
+ },
17
+ {
18
+ "epoch": 0.1,
19
+ "learning_rate": 0.0003695684724487918,
20
+ "loss": 0.3725,
21
+ "step": 400
22
+ },
23
+ {
24
+ "epoch": 0.15,
25
+ "learning_rate": 0.00036883393302294926,
26
+ "loss": 0.3363,
27
+ "step": 600
28
+ },
29
+ {
30
+ "epoch": 0.2,
31
+ "learning_rate": 0.0003678072134814851,
32
+ "loss": 0.3171,
33
+ "step": 800
34
+ },
35
+ {
36
+ "epoch": 0.25,
37
+ "learning_rate": 0.0003664899463577655,
38
+ "loss": 0.3041,
39
+ "step": 1000
40
+ },
41
+ {
42
+ "epoch": 0.3,
43
+ "learning_rate": 0.00036488422616976233,
44
+ "loss": 0.2954,
45
+ "step": 1200
46
+ },
47
+ {
48
+ "epoch": 0.36,
49
+ "learning_rate": 0.0003629926060896693,
50
+ "loss": 0.2872,
51
+ "step": 1400
52
+ },
53
+ {
54
+ "epoch": 0.41,
55
+ "learning_rate": 0.00036081809388423475,
56
+ "loss": 0.2804,
57
+ "step": 1600
58
+ },
59
+ {
60
+ "epoch": 0.46,
61
+ "learning_rate": 0.0003583641471322691,
62
+ "loss": 0.2754,
63
+ "step": 1800
64
+ },
65
+ {
66
+ "epoch": 0.51,
67
+ "learning_rate": 0.00035563466772692797,
68
+ "loss": 0.2696,
69
+ "step": 2000
70
+ },
71
+ {
72
+ "epoch": 0.56,
73
+ "learning_rate": 0.0003526339956715147,
74
+ "loss": 0.2675,
75
+ "step": 2200
76
+ },
77
+ {
78
+ "epoch": 0.61,
79
+ "learning_rate": 0.00034936690217866705,
80
+ "loss": 0.2642,
81
+ "step": 2400
82
+ },
83
+ {
84
+ "epoch": 0.66,
85
+ "learning_rate": 0.00034583858208389975,
86
+ "loss": 0.262,
87
+ "step": 2600
88
+ },
89
+ {
90
+ "epoch": 0.71,
91
+ "learning_rate": 0.0003420546455855667,
92
+ "loss": 0.2592,
93
+ "step": 2800
94
+ },
95
+ {
96
+ "epoch": 0.76,
97
+ "learning_rate": 0.00033802110932437625,
98
+ "loss": 0.2565,
99
+ "step": 3000
100
+ },
101
+ {
102
+ "epoch": 0.81,
103
+ "learning_rate": 0.0003337443868166437,
104
+ "loss": 0.2542,
105
+ "step": 3200
106
+ },
107
+ {
108
+ "epoch": 0.86,
109
+ "learning_rate": 0.00032923127825649205,
110
+ "loss": 0.2509,
111
+ "step": 3400
112
+ },
113
+ {
114
+ "epoch": 0.91,
115
+ "learning_rate": 0.00032448895970321745,
116
+ "loss": 0.2509,
117
+ "step": 3600
118
+ },
119
+ {
120
+ "epoch": 0.96,
121
+ "learning_rate": 0.0003195249716710097,
122
+ "loss": 0.2464,
123
+ "step": 3800
124
+ },
125
+ {
126
+ "epoch": 1.0,
127
+ "eval_accuracy": 0.8625347674451358,
128
+ "eval_auc": 0.8222331548742136,
129
+ "eval_f1": 0.19848246486644372,
130
+ "eval_loss": 0.41490185260772705,
131
+ "eval_mcc": 0.2639007297474409,
132
+ "eval_precision": 0.11370668247419904,
133
+ "eval_recall": 0.7800926533683039,
134
+ "eval_runtime": 1163.712,
135
+ "eval_samples_per_second": 109.693,
136
+ "eval_steps_per_second": 1.714,
137
+ "step": 3939
138
+ },
139
+ {
140
+ "epoch": 1.02,
141
+ "learning_rate": 0.00031434720713917336,
142
+ "loss": 0.2484,
143
+ "step": 4000
144
+ },
145
+ {
146
+ "epoch": 1.07,
147
+ "learning_rate": 0.0003089913128054233,
148
+ "loss": 0.2437,
149
+ "step": 4200
150
+ },
151
+ {
152
+ "epoch": 1.12,
153
+ "learning_rate": 0.0003034119838843777,
154
+ "loss": 0.242,
155
+ "step": 4400
156
+ },
157
+ {
158
+ "epoch": 1.17,
159
+ "learning_rate": 0.00029764449888790783,
160
+ "loss": 0.2423,
161
+ "step": 4600
162
+ },
163
+ {
164
+ "epoch": 1.22,
165
+ "learning_rate": 0.0002916980283940693,
166
+ "loss": 0.2387,
167
+ "step": 4800
168
+ },
169
+ {
170
+ "epoch": 1.27,
171
+ "learning_rate": 0.00028558202757645164,
172
+ "loss": 0.239,
173
+ "step": 5000
174
+ },
175
+ {
176
+ "epoch": 1.32,
177
+ "learning_rate": 0.0002793062211699992,
178
+ "loss": 0.2376,
179
+ "step": 5200
180
+ },
181
+ {
182
+ "epoch": 1.37,
183
+ "learning_rate": 0.0002728805880082148,
184
+ "loss": 0.2384,
185
+ "step": 5400
186
+ },
187
+ {
188
+ "epoch": 1.42,
189
+ "learning_rate": 0.00026631534515633606,
190
+ "loss": 0.2355,
191
+ "step": 5600
192
+ },
193
+ {
194
+ "epoch": 1.47,
195
+ "learning_rate": 0.0002596209316657108,
196
+ "loss": 0.2352,
197
+ "step": 5800
198
+ },
199
+ {
200
+ "epoch": 1.52,
201
+ "learning_rate": 0.00025280799197520415,
202
+ "loss": 0.2343,
203
+ "step": 6000
204
+ },
205
+ {
206
+ "epoch": 1.57,
207
+ "learning_rate": 0.0002458873589860299,
208
+ "loss": 0.231,
209
+ "step": 6200
210
+ },
211
+ {
212
+ "epoch": 1.62,
213
+ "learning_rate": 0.00023890534556422294,
214
+ "loss": 0.2339,
215
+ "step": 6400
216
+ },
217
+ {
218
+ "epoch": 1.68,
219
+ "learning_rate": 0.00023180289180308998,
220
+ "loss": 0.2332,
221
+ "step": 6600
222
+ },
223
+ {
224
+ "epoch": 1.73,
225
+ "learning_rate": 0.00022462614386106907,
226
+ "loss": 0.232,
227
+ "step": 6800
228
+ },
229
+ {
230
+ "epoch": 1.78,
231
+ "learning_rate": 0.00021738651311196128,
232
+ "loss": 0.231,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 1.83,
237
+ "learning_rate": 0.00021013207456600314,
238
+ "loss": 0.2305,
239
+ "step": 7200
240
+ },
241
+ {
242
+ "epoch": 1.88,
243
+ "learning_rate": 0.00020280146390924225,
244
+ "loss": 0.2298,
245
+ "step": 7400
246
+ },
247
+ {
248
+ "epoch": 1.93,
249
+ "learning_rate": 0.00019544267273437384,
250
+ "loss": 0.2302,
251
+ "step": 7600
252
+ },
253
+ {
254
+ "epoch": 1.98,
255
+ "learning_rate": 0.0001880674018726795,
256
+ "loss": 0.2295,
257
+ "step": 7800
258
+ },
259
+ {
260
+ "epoch": 2.0,
261
+ "eval_accuracy": 0.8789903474277315,
262
+ "eval_auc": 0.8180827968848764,
263
+ "eval_f1": 0.21386215966220054,
264
+ "eval_loss": 0.4392629563808441,
265
+ "eval_mcc": 0.27446788968929925,
266
+ "eval_precision": 0.12459111498869992,
267
+ "eval_recall": 0.7543961461123881,
268
+ "eval_runtime": 1167.9434,
269
+ "eval_samples_per_second": 109.296,
270
+ "eval_steps_per_second": 1.708,
271
+ "step": 7878
272
+ }
273
+ ],
274
+ "logging_steps": 200,
275
+ "max_steps": 15756,
276
+ "num_train_epochs": 4,
277
+ "save_steps": 500,
278
+ "total_flos": 4.051458441583424e+16,
279
+ "trial_name": null,
280
+ "trial_params": null
281
+ }
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19560f827f37de980ef9907eb4a996feaadf6e4f6a723cacbef99fc7a0ba1b25
3
+ size 4155
esm2_t6_8M_qlora_binding_sites_2023-09-29_03-34-40/checkpoint-7878/vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>