Saving weights and logs of step 150000

Load data & train tokenizer.ipynb

@@ -54,12 +54,38 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "348a4dd4",
    "metadata": {},
    "outputs": [],
    "source": [
-    "tokenizer = AutoTokenizer.from_pretrained(\"./\")"
+    "from transformers import pipeline, AutoTokenizer,BigBirdForMaskedLM, pipeline\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\".\")\n",
+    "model = FlaxBigBirdForMaskedLM.from_pretrained(\".\")\n",
+    "\n",
+    "unmasker = pipeline('fill-mask',tokenizer=tokenizer, model=model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4b1bb489",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'unmasker' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_759685/2003099555.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0munmasker\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"test\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m: name 'unmasker' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "unmasker(\"test\")"
    ]
   },
   {

checkpoint_150000

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e3a86c577a503cb52846a97cb2b56106974afab3bedc8617a0c5bc1e89907ea
+size 1530270447

config.json

@@ -1,5 +1,4 @@
 {
-  "_name_or_path": ".",
   "architectures": [
     "BigBirdForMaskedLM"
   ],
@@ -24,7 +23,6 @@
   "position_embedding_type": "absolute",
   "rescale_embeddings": false,
   "sep_token_id": 66,
-  "torch_dtype": "float32",
   "transformers_version": "4.9.0.dev0",
   "type_vocab_size": 2,
   "use_bias": true,

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29ed0773269721507328c69a2ccd94dd845503587ae4a89525ec4f204ab05f0d
+oid sha256:0ec0fb67c52b43bbdf61a82273bba1919d00f297e8de79d79148b5f14a013a41
 size 510090043

wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 136400, "learning_rate": 2.4548056899220683e-05, "train_loss": 2.1764094829559326, "_runtime": 15833, "_timestamp": 1626391358, "_step": 733, "eval_step": 132000, "eval_accuracy": 0.5815154314041138, "eval_loss": 2.216282367706299}
+{"training_step": 150000, "learning_rate": 2.396145646343939e-05, "train_loss": 2.3190174102783203, "_runtime": 21476, "_timestamp": 1626397001, "_step": 1008, "eval_step": 150000, "eval_accuracy": 0.5905819535255432, "eval_loss": 2.15248703956604}