Training in progress, step 5000

config.json

@@ -3,26 +3,28 @@
     "PIXELForPreTraining"
   ],
   "attention_probs_dropout_prob": 0.1,
-  "decoder_hidden_size": 128,
-  "decoder_intermediate_size": 512,
-  "decoder_num_attention_heads": 4,
+  "decoder_hidden_size": 264,
+  "decoder_intermediate_size": 1056,
+  "decoder_num_attention_heads": 6,
   "decoder_num_hidden_layers": 4,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 192,
+  "hidden_size": 384,
   "image_size": [
     32,
     18432
   ],
   "initializer_range": 0.02,
-  "intermediate_size": 512,
+  "intermediate_size": 1536,
   "layer_norm_eps": 1e-12,
   "mask_ratio": 0.25,
   "model_type": "pixel",
+  "noise_scheduler": "cosine",
   "norm_pix_loss": true,
-  "num_attention_heads": 4,
+  "norm_pix_pred": false,
+  "num_attention_heads": 6,
   "num_channels": 3,
-  "num_hidden_layers": 6,
+  "num_hidden_layers": 12,
   "patch_size": 32,
   "qkv_bias": true,
   "torch_dtype": "float32",

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:981c9658fdc5ac18f12b2f62ef5e7bdaa3b129de485f34d72b01c70ff9e51a84
-size 16339237
+oid sha256:257f26e0c18c8c6efa38b3474c9b5d59a1367b1e71123f77021c48c99d17508f
+size 108604581

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8bb38d74f319322e0c58296ed2f8e7eac3e908cd961e8dd533be5e62a551897
+oid sha256:a290c6ebb75bee38957709a23bfbe42770fa0aa295c474547f4b2350576bbd17
 size 3247