Training in progress, step 500

config.json

@@ -1,20 +1,19 @@
 {
-  "_name_or_path": "run9-phase2",
+  "_name_or_path": "./run9",
   "architectures": [
-    "PipelinedGroupBertForPreTraining"
+    "PoptorchPipelinedGroupBertForPreTraining"
   ],
-  "attention_probs_dropout_prob": 0.1,
+  "attention_probs_dropout_prob": 0.0,
   "classifier_dropout": null,
   "conv_group_size": 16,
   "conv_kernel_size": 7,
   "ffn_groups": 4,
-  "gradient_checkpointing": false,
   "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
+  "hidden_dropout_prob": 0.0,
   "hidden_size": 768,
   "initializer_range": 0.02,
   "intermediate_size": 3072,
-  "layer_norm_eps": 0.001,
+  "layer_norm_eps": 1e-12,
   "max_num_masked_tokens": 96,
   "max_position_embeddings": 512,
   "model_type": "groupbert",

ipu_config.json

@@ -17,7 +17,7 @@
   ],
   "matmul_proportion": 0.22,
   "optimizer_state_offchip": false,
-  "optimum_version": "1.1.1",
+  "optimum_version": "1.4.1",
   "output_mode": "final",
   "recompute_checkpoint_every_layer": true,
   "replicated_tensor_sharding": true,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f3b913dc3f9b1fb1db37b58adc647d4ea8b0a7c266f2eb9aad5e22575be3ac3
+oid sha256:8a406baa7c032227f8ac5eb3f882b8312f8b52f36f38288c34332d2c7d5c9c15
 size 321956797

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f02572ddce2f8070fa4cd8025303dab3b529c6cfee97cf22db3aff4aecdb3a21
-size 2927
+oid sha256:7fcbe6e7f558e3ec8b775c0b6efc35b5d905da365ad127cd316fdd70ecef2c91
+size 2863