Training in progress, step 500

config.json

@@ -1,27 +1,27 @@
 {
-  "_name_or_path": "facebook/esm1b_t33_650M_UR50S",
+  "_name_or_path": "facebook/esm2_t12_35M_UR50D",
   "architectures": [
     "EsmForMaskedLM"
   ],
   "attention_probs_dropout_prob": 0.0,
   "classifier_dropout": null,
-  "emb_layer_norm_before": true,
+  "emb_layer_norm_before": false,
   "esmfold_config": null,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.0,
-  "hidden_size": 1280,
+  "hidden_size": 480,
   "initializer_range": 0.02,
-  "intermediate_size": 5120,
+  "intermediate_size": 1920,
   "is_folding_model": false,
   "layer_norm_eps": 1e-05,
   "mask_token_id": 32,
   "max_position_embeddings": 1026,
   "model_type": "esm",
   "num_attention_heads": 20,
-  "num_hidden_layers": 33,
+  "num_hidden_layers": 12,
   "output_attentions": true,
   "pad_token_id": 1,
-  "position_embedding_type": "absolute",
+  "position_embedding_type": "rotary",
   "token_dropout": true,
   "torch_dtype": "float32",
   "transformers_version": "4.31.0",

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:668e3f23175e955a4f9771f149797dd14fbf6e6697f253480be1d409f2f1747d
-size 2609620225
+oid sha256:b607742916a1ba75e48a0b40df597bc784a1dbf757abf4e975ca0567fccee338
+size 136048297

runs/Jul23_21-05-29_4ab41e0c8300/events.out.tfevents.1690146335.4ab41e0c8300.32683.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b8507e4d7b42c9f946040b763a928756c5de9c83393113b82287f89dfe57032
+size 4161

runs/Jul23_21-05-58_4ab41e0c8300/events.out.tfevents.1690146361.4ab41e0c8300.32683.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5089fcdd4a5a1f9c420cdb8f459840666139c598ce202ad4e98e6112e40169d
+size 4862

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6df29bb7664bd68250f0bdb970548df95c9acffd0ed06ecddd05295347a2dcf
+oid sha256:c3eb365d361fc639e289da00527dca75cf7387b45dec7744e55d9e33616e298c
 size 3963