diff --git "a/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/part2.ipynb" "b/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/part2.ipynb"
--- "a/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/part2.ipynb"
+++ "b/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/part2.ipynb"
@@ -1,3 +1,7092 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:12f10826a59669d9dcf90268a0377596a642007f5642c452e87f76612a62939d
-size 52757309
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "6303a399",
+   "metadata": {
+    "papermill": {
+     "duration": 0.005464,
+     "end_time": "2023-09-06T17:17:07.017280",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:07.011816",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# RWKV v5\n",
+    "\n",
+    "Simple memory training for a small model\n",
+    "\n",
+    "**Note:** This project assumes you have the rwkv-infctx conda env setup"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "f39c274d",
+   "metadata": {
+    "papermill": {
+     "duration": 0.004562,
+     "end_time": "2023-09-06T17:17:07.026887",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:07.022325",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# Basic Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "339857f4",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-09-06T17:17:07.038414Z",
+     "iopub.status.busy": "2023-09-06T17:17:07.037888Z",
+     "iopub.status.idle": "2023-09-06T17:17:08.034427Z",
+     "shell.execute_reply": "2023-09-06T17:17:08.033578Z"
+    },
+    "papermill": {
+     "duration": 1.005021,
+     "end_time": "2023-09-06T17:17:08.036897",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:07.031876",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CITATION.cff  RWKV-v4wavenet\t RWKV-v5headsize32  checkpoint\tnotebook\r\n",
+      "LICENSE       RWKV-v5\t\t RWKV-v5r2\t    datapath\toutput\r\n",
+      "README.md     RWKV-v5altwavenet  RWKV-v5rstack\t    docker\r\n",
+      "RWKV-v4neo    RWKV-v5headsize2x  RWKV-v5wavenet     model\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "# First lets setup the various directories, and init the model\n",
+    "!ls ../../../../../\n",
+    "!mkdir -p ../../../../../model/\n",
+    "!mkdir -p ../../../../../datapath/\n",
+    "!mkdir -p ../../../../../checkpoint/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "7b45aafc",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-09-06T17:17:08.049253Z",
+     "iopub.status.busy": "2023-09-06T17:17:08.048613Z",
+     "iopub.status.idle": "2023-09-06T17:17:11.256715Z",
+     "shell.execute_reply": "2023-09-06T17:17:11.255796Z"
+    },
+    "papermill": {
+     "duration": 3.217287,
+     "end_time": "2023-09-06T17:17:11.259293",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:08.042006",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "# Additional dependencies for eval stuff\n",
+    "!pip install -q aiocsv aiofiles"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5a6fa0ec",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-09-06T17:17:11.271967Z",
+     "iopub.status.busy": "2023-09-06T17:17:11.271338Z",
+     "iopub.status.idle": "2023-09-06T17:17:11.279590Z",
+     "shell.execute_reply": "2023-09-06T17:17:11.278754Z"
+    },
+    "papermill": {
+     "duration": 0.016902,
+     "end_time": "2023-09-06T17:17:11.281556",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:11.264654",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DEEPSPEED_STRAT: deepspeed_stage_1\n",
+      "ENABLE_WANDB: True\n",
+      "GPU_DEVICES: auto\n",
+      "DIR_NAME: L6-D2048-E1e-1-ctx4k\n",
+      "NOTEBOOK_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k\n",
+      "INFERENCE_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5\n",
+      "TRAINER_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5\n",
+      "PROJECT_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer\n"
+     ]
+    }
+   ],
+   "source": [
+    "DEEPSPEED_STRAT=\"deepspeed_stage_1\"\n",
+    "GPU_DEVICES=\"auto\"\n",
+    "ENABLE_WANDB=True\n",
+    "\n",
+    "# Layer count and embed dim to start with\n",
+    "LAYER_COUNT=6\n",
+    "EMBED_DIM=2048\n",
+    "\n",
+    "EMBED_SCALE=0.1\n",
+    "EMBED_SCALE_LABEL=str(EMBED_SCALE).replace(\".\", \"_\")\n",
+    "\n",
+    "WANDB_PREFIX=f\"v5r3-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE}\"\n",
+    "FILENAME_PREFIX=f\"v5r3-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE_LABEL}\"\n",
+    "\n",
+    "print(\"DEEPSPEED_STRAT:\", DEEPSPEED_STRAT)\n",
+    "print(\"ENABLE_WANDB:\", ENABLE_WANDB)\n",
+    "print(\"GPU_DEVICES:\", GPU_DEVICES)\n",
+    "\n",
+    "if ENABLE_WANDB:\n",
+    "    WANDB_MODE=\"online\"\n",
+    "else:\n",
+    "    WANDB_MODE=\"disabled\"\n",
+    "\n",
+    "# Computing the notebook, and various paths\n",
+    "import os\n",
+    "NOTEBOOK_DIR=os.path.dirname(os.path.abspath(\"__file__\"))\n",
+    "CONFIG_DIR=os.path.abspath(os.path.join(NOTEBOOK_DIR, \"../\"))\n",
+    "PROJECT_DIR=os.path.abspath(os.path.join(CONFIG_DIR, \"../../../../\"))\n",
+    "TRAINER_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5/\"))\n",
+    "INFERENCE_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5/\"))\n",
+    "\n",
+    "# Get the notebook dir name\n",
+    "DIR_NAME=os.path.basename(NOTEBOOK_DIR)\n",
+    "\n",
+    "# Log names and dir\n",
+    "print(\"DIR_NAME:\", DIR_NAME)\n",
+    "print(\"NOTEBOOK_DIR:\", NOTEBOOK_DIR)\n",
+    "print(\"INFERENCE_DIR:\", INFERENCE_DIR)\n",
+    "print(\"TRAINER_DIR:\", TRAINER_DIR)\n",
+    "print(\"PROJECT_DIR:\", PROJECT_DIR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "5715cb46",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-09-06T17:17:11.294198Z",
+     "iopub.status.busy": "2023-09-06T17:17:11.293662Z",
+     "iopub.status.idle": "2023-09-06T17:17:31.891567Z",
+     "shell.execute_reply": "2023-09-06T17:17:31.890659Z"
+    },
+    "papermill": {
+     "duration": 20.606807,
+     "end_time": "2023-09-06T17:17:31.893643",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:11.286836",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--2023-09-06 17:17:11--  https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/v5r3-L6-D2048-E0_1-enwiki-4k.pth\r\n",
+      "Resolving huggingface.co (huggingface.co)... 18.154.227.87, 18.154.227.67, 18.154.227.69, ...\r\n",
+      "Connecting to huggingface.co (huggingface.co)|18.154.227.87|:443... connected.\r\n",
+      "HTTP request sent, awaiting response... 302 Found\r\n",
+      "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/7eb7abfda2e4cfb2a961ba4d52564f9b330830ba1a836966556e28753468ea1e?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5r3-L6-D2048-E0_1-enwiki-4k.pth%3B+filename%3D%22v5r3-L6-D2048-E0_1-enwiki-4k.pth%22%3B&Expires=1694279831&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NDI3OTgzMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzdlYjdhYmZkYTJlNGNmYjJhOTYxYmE0ZDUyNTY0ZjliMzMwODMwYmExYTgzNjk2NjU1NmUyODc1MzQ2OGVhMWU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=iCSVplgSsG5AjmmXo591m6nLTcfvPpRlnGWiePyk13mX8KhpZ9pdk-CgfNOOMeUtGGTe7Dax%7EqOKvdtmYwBAGLh94g2O8F3Enb9ju0hOqXZa8Z0LoWk7-vfWY-DupCpdaUAS-vgVDxUcUL-VULBW2JnqqXO0Vzm2TaBAOxRSY6u86wfuSrMzfuxu7vPtGKnxU2tO8CFkltBdkumTlWJ8KSHLx8A0BfIn23aR2Wzq77%7EmXJ7dJvBt19%7EiS5p6m-ITw0yXoKhohGLmiubk4%7EmK543ibTfPBbRtxjkxzepwDwuOCLomwXnaGvmVfBjnrHa1ETJTjkro9qITPESxU6TOxg__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n",
+      "--2023-09-06 17:17:11--  https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/7eb7abfda2e4cfb2a961ba4d52564f9b330830ba1a836966556e28753468ea1e?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5r3-L6-D2048-E0_1-enwiki-4k.pth%3B+filename%3D%22v5r3-L6-D2048-E0_1-enwiki-4k.pth%22%3B&Expires=1694279831&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NDI3OTgzMX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzdlYjdhYmZkYTJlNGNmYjJhOTYxYmE0ZDUyNTY0ZjliMzMwODMwYmExYTgzNjk2NjU1NmUyODc1MzQ2OGVhMWU%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=iCSVplgSsG5AjmmXo591m6nLTcfvPpRlnGWiePyk13mX8KhpZ9pdk-CgfNOOMeUtGGTe7Dax%7EqOKvdtmYwBAGLh94g2O8F3Enb9ju0hOqXZa8Z0LoWk7-vfWY-DupCpdaUAS-vgVDxUcUL-VULBW2JnqqXO0Vzm2TaBAOxRSY6u86wfuSrMzfuxu7vPtGKnxU2tO8CFkltBdkumTlWJ8KSHLx8A0BfIn23aR2Wzq77%7EmXJ7dJvBt19%7EiS5p6m-ITw0yXoKhohGLmiubk4%7EmK543ibTfPBbRtxjkxzepwDwuOCLomwXnaGvmVfBjnrHa1ETJTjkro9qITPESxU6TOxg__&Key-Pair-Id=KVTP0A1DKRTAX\r\n",
+      "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 108.138.64.121, 108.138.64.49, 108.138.64.36, ...\r\n",
+      "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|108.138.64.121|:443... connected.\r\n",
+      "HTTP request sent, awaiting response... "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "200 OK\r\n",
+      "Length: 1066536937 (1017M) [binary/octet-stream]\r\n",
+      "Saving to: ‘v5r3-L6-D2048-E0_1-enwiki-4k.pth’\r\n",
+      "\r\n",
+      "\r",
+      "          v5r3-L6-D   0%[                    ]       0  --.-KB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "         v5r3-L6-D2   1%[                    ]  15.26M  53.2MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "        v5r3-L6-D20   3%[                    ]  30.52M  46.6MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "       v5r3-L6-D204   4%[                    ]  45.78M  48.4MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "      v5r3-L6-D2048   5%[>                   ]  60.52M  52.7MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "     v5r3-L6-D2048-   7%[>                   ]  74.46M  46.7MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "    v5r3-L6-D2048-E   8%[>                   ]  91.03M  50.4MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "   v5r3-L6-D2048-E0  10%[=>                  ] 101.80M  47.0MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  v5r3-L6-D2048-E0_  10%[=>                  ] 106.81M  44.8MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      " v5r3-L6-D2048-E0_1  11%[=>                  ] 121.56M  46.7MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "v5r3-L6-D2048-E0_1-  13%[=>                  ] 134.53M  48.0MB/s               "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "5r3-L6-D2048-E0_1-e  14%[=>                  ] 151.99M  50.6MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "r3-L6-D2048-E0_1-en  15%[==>                 ] 155.00M  48.4MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "3-L6-D2048-E0_1-enw  16%[==>                 ] 167.85M  48.7MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-L6-D2048-E0_1-enwi  18%[==>                 ] 189.61M  52.0MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "L6-D2048-E0_1-enwik  19%[==>                 ] 202.61M  52.7MB/s    eta 17s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "6-D2048-E0_1-enwiki  21%[===>                ] 213.62M  48.7MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-D2048-E0_1-enwiki-  22%[===>                ] 226.41M  50.0MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "D2048-E0_1-enwiki-4  23%[===>                ] 243.63M  50.1MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2048-E0_1-enwiki-4k  24%[===>                ] 250.70M  50.7MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "048-E0_1-enwiki-4k.  25%[====>               ] 259.40M  49.3MB/s    eta 16s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "48-E0_1-enwiki-4k.p  27%[====>               ] 274.66M  52.3MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "8-E0_1-enwiki-4k.pt  27%[====>               ] 281.34M  50.0MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-E0_1-enwiki-4k.pth  28%[====>               ] 289.92M  50.7MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "E0_1-enwiki-4k.pth   30%[=====>              ] 305.18M  50.2MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "0_1-enwiki-4k.pth    31%[=====>              ] 320.43M  51.6MB/s    eta 15s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "_1-enwiki-4k.pth     32%[=====>              ] 325.55M  47.9MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "1-enwiki-4k.pth      33%[=====>              ] 335.69M  45.4MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-enwiki-4k.pth       34%[=====>              ] 350.95M  46.9MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "enwiki-4k.pth        37%[======>             ] 377.49M  49.4MB/s    eta 14s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "nwiki-4k.pth         38%[======>             ] 396.22M  50.4MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "wiki-4k.pth          40%[=======>            ] 411.47M  51.8MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "iki-4k.pth           41%[=======>            ] 423.97M  52.9MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "ki-4k.pth            42%[=======>            ] 428.21M  50.3MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "i-4k.pth             43%[=======>            ] 440.68M  50.5MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-4k.pth              43%[=======>            ] 442.66M  50.3MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "4k.pth               44%[=======>            ] 456.72M  49.8MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "k.pth                46%[========>           ] 472.50M  51.1MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      ".pth                 47%[========>           ] 480.25M  51.7MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "pth                  48%[========>           ] 488.28M  50.6MB/s    eta 12s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "th                   50%[=========>          ] 511.80M  53.9MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "h                    51%[=========>          ] 528.83M  53.4MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                     52%[=========>          ] 534.05M  54.8MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                  v  54%[=========>          ] 549.31M  57.4MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                 v5  55%[==========>         ] 566.76M  59.5MB/s    eta 10s    "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "                v5r  56%[==========>         ] 573.51M  56.4MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "               v5r3  57%[==========>         ] 579.83M  49.8MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "              v5r3-  58%[==========>         ] 594.57M  48.9MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "             v5r3-L  60%[===========>        ] 610.35M  50.6MB/s    eta 9s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "            v5r3-L6  61%[===========>        ] 625.61M  51.2MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "           v5r3-L6-  63%[===========>        ] 640.87M  53.4MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "          v5r3-L6-D  64%[===========>        ] 656.13M  53.0MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "         v5r3-L6-D2  66%[============>       ] 671.38M  53.7MB/s    eta 8s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "        v5r3-L6-D20  67%[============>       ] 686.13M  56.4MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "       v5r3-L6-D204  67%[============>       ] 686.77M  52.2MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "      v5r3-L6-D2048  69%[============>       ] 701.90M  52.3MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "     v5r3-L6-D2048-  70%[=============>      ] 717.16M  51.2MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "    v5r3-L6-D2048-E  72%[=============>      ] 732.42M  52.4MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "   v5r3-L6-D2048-E0  73%[=============>      ] 742.67M  51.0MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "  v5r3-L6-D2048-E0_  74%[=============>      ] 756.60M  50.1MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      " v5r3-L6-D2048-E0_1  75%[==============>     ] 771.87M  52.0MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "v5r3-L6-D2048-E0_1-  76%[==============>     ] 781.29M  53.7MB/s    eta 6s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "5r3-L6-D2048-E0_1-e  78%[==============>     ] 793.46M  52.7MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "r3-L6-D2048-E0_1-en  79%[==============>     ] 808.71M  51.9MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "3-L6-D2048-E0_1-enw  81%[===============>    ] 823.97M  52.1MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-L6-D2048-E0_1-enwi  82%[===============>    ] 840.69M  53.9MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "L6-D2048-E0_1-enwik  84%[===============>    ] 854.49M  52.7MB/s    eta 4s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "6-D2048-E0_1-enwiki  85%[================>   ] 869.75M  52.5MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-D2048-E0_1-enwiki-  87%[================>   ] 885.01M  53.8MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "D2048-E0_1-enwiki-4  88%[================>   ] 899.75M  53.7MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "2048-E0_1-enwiki-4k  89%[================>   ] 906.85M  51.7MB/s    eta 3s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "048-E0_1-enwiki-4k.  90%[=================>  ] 915.66M  53.2MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "48-E0_1-enwiki-4k.p  91%[=================>  ] 930.27M  53.4MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "8-E0_1-enwiki-4k.pt  92%[=================>  ] 945.53M  54.0MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "-E0_1-enwiki-4k.pth  94%[=================>  ] 960.79M  51.7MB/s    eta 2s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "E0_1-enwiki-4k.pth   96%[==================> ] 976.55M  53.0MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "0_1-enwiki-4k.pth    97%[==================> ] 986.81M  50.5MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "_1-enwiki-4k.pth     97%[==================> ] 991.82M  49.7MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "1-enwiki-4k.pth      99%[==================> ]   1007M  48.7MB/s    eta 1s     "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "v5r3-L6-D2048-E0_1- 100%[===================>]   1017M  50.0MB/s    in 20s     \r\n",
+      "\r\n",
+      "2023-09-06 17:17:31 (51.1 MB/s) - ‘v5r3-L6-D2048-E0_1-enwiki-4k.pth’ saved [1066536937/1066536937]\r\n",
+      "\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 1018M\r\n",
+      "drwxr-xr-x  2 root root  4.0K Sep  6 17:17 .\r\n",
+      "drwxr-xr-x 20 root root  4.0K Sep  6 17:17 ..\r\n",
+      "-rw-r--r--  1 root root 1018M Sep  6 17:07 v5r3-L6-D2048-E0_1-enwiki-4k.pth\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Download the model directly (stop gap till HF sync issues is resolved)\n",
+    "!cd \"{TRAINER_DIR}\" && cd \"../model/\" && \\\n",
+    "    wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-r3-memory/{DIR_NAME}/{FILENAME_PREFIX}-enwiki-4k.pth\"\n",
+    "\n",
+    "!cd \"{TRAINER_DIR}\" && cd \"../model/\" && \\\n",
+    "    ls -alh ."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "4e6c3e65",
+   "metadata": {
+    "papermill": {
+     "duration": 0.008152,
+     "end_time": "2023-09-06T17:17:31.910423",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:31.902271",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "source": [
+    "# Enwiki Stage 2 : Basic Instruct Tuning"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "b243d3f6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-09-06T17:17:31.929289Z",
+     "iopub.status.busy": "2023-09-06T17:17:31.928880Z",
+     "iopub.status.idle": "2023-09-06T17:17:44.337863Z",
+     "shell.execute_reply": "2023-09-06T17:17:44.336545Z"
+    },
+    "papermill": {
+     "duration": 12.421926,
+     "end_time": "2023-09-06T17:17:44.340880",
+     "exception": false,
+     "start_time": "2023-09-06T17:17:31.918954",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Downloading readme:   0%|                           | 0.00/7.79k [00:00<?, ?B/s]\r",
+      "Downloading readme: 100%|██████████████████| 7.79k/7.79k [00:00<00:00, 5.28MB/s]\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Downloading data files:   0%|                             | 0/1 [00:00<?, ?it/s]\r\n",
+      "\r",
+      "Downloading data:   0%|                             | 0.00/7.80M [00:00<?, ?B/s]\u001b[A"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r\n",
+      "\r",
+      "Downloading data:  54%|██████████▊         | 4.19M/7.80M [00:00<00:00, 10.3MB/s]\u001b[A\r",
+      "Downloading data: 100%|████████████████████| 7.80M/7.80M [00:00<00:00, 18.4MB/s]\r\n",
+      "\r",
+      "Downloading data files: 100%|█████████████████████| 1/1 [00:00<00:00,  2.35it/s]\r",
+      "Downloading data files: 100%|█████████████████████| 1/1 [00:00<00:00,  2.35it/s]\r\n",
+      "\r",
+      "Extracting data files:   0%|                              | 0/1 [00:00<?, ?it/s]\r",
+      "Extracting data files: 100%|████████████████████| 1/1 [00:00<00:00, 1532.45it/s]\r\n",
+      "Setting num_proc from 32 back to 1 for the train split to disable multiprocessing as it only contains one shard.\r\n",
+      "\r",
+      "Generating train split:   0%|                  | 0/15015 [00:00<?, ? examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Generating train split: 100%|██| 15015/15015 [00:00<00:00, 411910.93 examples/s]\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):   0%|                       | 0/15015 [00:00<?, ? examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):   0%|             | 56/15015 [00:00<01:08, 217.92 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):   2%|▏           | 307/15015 [00:00<00:16, 917.22 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):   6%|▋          | 918/15015 [00:00<00:06, 2268.48 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  11%|█         | 1627/15015 [00:00<00:04, 3233.71 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  16%|█▌        | 2349/15015 [00:00<00:03, 4119.76 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  26%|██▌       | 3861/15015 [00:00<00:01, 6063.23 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  32%|███▏      | 4779/15015 [00:01<00:01, 6335.91 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  38%|███▊      | 5703/15015 [00:01<00:01, 6428.44 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  43%|████▎     | 6408/15015 [00:01<00:01, 5980.98 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  47%|████▋     | 7012/15015 [00:01<00:01, 5566.27 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  52%|█████▏    | 7790/15015 [00:01<00:01, 5294.22 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  56%|█████▌    | 8393/15015 [00:01<00:01, 4747.73 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  64%|██████▍   | 9582/15015 [00:01<00:00, 5487.01 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  86%|██████▊ | 12895/15015 [00:02<00:00, 11304.34 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32):  95%|████████▌| 14279/15015 [00:02<00:00, 9003.98 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Map (num_proc=32): 100%|█████████| 15015/15015 [00:02<00:00, 5557.21 examples/s]\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Filter (num_proc=32):   0%|                    | 0/15015 [00:00<?, ? examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Filter (num_proc=32):   3%|▎       | 470/15015 [00:00<00:07, 1955.08 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Filter (num_proc=32):  44%|██▋   | 6573/15015 [00:00<00:00, 23716.64 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Filter (num_proc=32):  94%|████▋| 14077/15015 [00:00<00:00, 41346.10 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Filter (num_proc=32): 100%|█████| 15015/15015 [00:00<00:00, 26920.43 examples/s]\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Saving the dataset (0/1 shards):   0%|         | 0/14932 [00:00<?, ? examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Saving the dataset (0/1 shards):  80%|▊| 12000/14932 [00:00<00:00, 108456.32 exa"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Saving the dataset (1/1 shards): 100%|█| 14932/14932 [00:00<00:00, 108456.32 exa\r",
+      "Saving the dataset (1/1 shards): 100%|█| 14932/14932 [00:00<00:00, 108436.88 exa\r\n",
+      "\r",
+      "Saving the dataset (0/1 shards):   0%|            | 0/76 [00:00<?, ? examples/s]\r",
+      "Saving the dataset (1/1 shards): 100%|█| 76/76 [00:00<00:00, 7375.45 examples/s]\r",
+      "Saving the dataset (1/1 shards): 100%|█| 76/76 [00:00<00:00, 7175.08 examples/s]\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Lets preload the requried dataset\n",
+    "!cd \"{TRAINER_DIR}\" && \\\n",
+    "    python3 preload_datapath.py \"{CONFIG_DIR}/config-enwiki-instruct.yaml\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "067a1fec",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-09-06T17:17:44.370089Z",
+     "iopub.status.busy": "2023-09-06T17:17:44.368709Z"
+    },
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": false,
+     "start_time": "2023-09-06T17:17:44.353735",
+     "status": "running"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[2023-09-06 17:17:49,031] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.10/dist-packages/lightning/pytorch/cli.py:484: UserWarning: LightningCLI's args parameter is intended to run from within Python like if it were from the command line. To prevent mistakes it is not recommended to provide both args and command line arguments, got: sys.argv[1:]=['fit', '-c', '/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/config-enwiki-instruct.yaml', '--trainer.logger.init_args.name=v5r3-L6-D2048-E0.1 - Enwiki-Instruct (train-ctx=4k, deepspeed_stage_1)', '--trainer.strategy=deepspeed_stage_1', '--trainer.devices=auto', '--trainer.callbacks.init_args.dirpath=../checkpoint/v5r3-L6-D2048-E0_1-enwiki-instruct/', '--model.load_model=../model/v5r3-L6-D2048-E0_1-enwiki-4k.pth', '--model.ctx_len=4096', '--model.bptt_learning_range=1'], args=['fit', '-c', '/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/config-enwiki-instruct.yaml', '--trainer.logger.init_args.name=v5r3-L6-D2048-E0.1 - Enwiki-Instruct (train-ctx=4k, deepspeed_stage_1)', '--trainer.strategy=deepspeed_stage_1', '--trainer.devices=auto', '--trainer.callbacks.init_args.dirpath=../checkpoint/v5r3-L6-D2048-E0_1-enwiki-instruct/', '--model.load_model=../model/v5r3-L6-D2048-E0_1-enwiki-4k.pth', '--model.ctx_len=4096', '--model.bptt_learning_range=1'].\r\n",
+      "  rank_zero_warn(\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.10/dist-packages/lightning/fabric/utilities/seed.py:39: UserWarning: No seed found, seed set to 206439180\r\n",
+      "  rank_zero_warn(f\"No seed found, seed set to {seed}\")\r\n",
+      "Global seed set to 206439180\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mpicocreator\u001b[0m (\u001b[33mrwkv-x-dev\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.15.9\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./wandb/run-20230906_171752-8z8chpnp\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mv5r3-L6-D2048-E0.1 - Enwiki-Instruct (train-ctx=4k, deepspeed_stage_1)\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-X-Experiments\u001b[0m\r\n",
+      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-X-Experiments/runs/8z8chpnp\u001b[0m\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.10/dist-packages/lightning/fabric/connector.py:554: UserWarning: bf16 is supported for historical reasons but its usage is discouraged. Please set your precision to bf16-mixed instead!\r\n",
+      "  rank_zero_warn(\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "GPU available: True (cuda), used: True\r\n",
+      "TPU available: False, using: 0 TPU cores\r\n",
+      "IPU available: False, using: 0 IPUs\r\n",
+      "HPU available: False, using: 0 HPUs\r\n",
+      "\r\n",
+      "\r\n",
+      "[RWKV.Trainer] Applying 'target_batch_size' with the following:\r\n",
+      "   - target_batch_size:       32\r\n",
+      "   - num_nodes:               1\r\n",
+      "   - num_devices:             1\r\n",
+      "   - accumulate_grad_batches: 32\r\n",
+      "   - effective_batch_size:    32\r\n",
+      "\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Saving the dataset (0/1 shards):   0%|         | 0/14932 [00:00<?, ? examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Saving the dataset (0/1 shards):  74%|▋| 11000/14932 [00:00<00:00, 92048.62 exam"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Saving the dataset (1/1 shards): 100%|█| 14932/14932 [00:00<00:00, 92048.62 exam\r",
+      "Saving the dataset (1/1 shards): 100%|█| 14932/14932 [00:00<00:00, 82279.50 exam\r\n",
+      "\r",
+      "Saving the dataset (0/1 shards):   0%|            | 0/76 [00:00<?, ? examples/s]\r",
+      "Saving the dataset (1/1 shards): 100%|█| 76/76 [00:00<00:00, 7022.54 examples/s]\r",
+      "Saving the dataset (1/1 shards): 100%|█| 76/76 [00:00<00:00, 6601.78 examples/s]\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[rank: 0] Global seed set to 206439180\r\n",
+      "initializing deepspeed distributed: GLOBAL_RANK: 0, MEMBER: 1/1\r\n",
+      "[2023-09-06 17:18:00,250] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Enabling DeepSpeed BF16.\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\r\n",
+      "#\r\n",
+      "# RWKV lighting_trainer.py important notes \r\n",
+      "# https://github.com/RWKV/RWKV-infctx-trainer \r\n",
+      "#\r\n",
+      "# - Ensure your host is not running cuda 12.0 (use either 11.8, or >=12.1), as this is known to have freeze issues\r\n",
+      "# - The terms used in wandb / the progress bar can be confusing, see the github README.md for beter clarifications\r\n",
+      "# - When resuming from checkpoint, the estimated time is inaccurate\r\n",
+      "#\r\n",
+      "\r\n",
+      "[RWKV.model] Configuring optimizer with\r\n",
+      "    - lr_init:  4.000e-04 (0.0004)\r\n",
+      "    - lr_final: 3.000e-04 (0.0003)\r\n",
+      "\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Using /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\r\n",
+      "Detected CUDA files, patching ldflags\r\n",
+      "Emitting ninja build file /root/.cache/torch_extensions/py310_cu118/fused_adam/build.ninja...\r\n",
+      "Building extension module fused_adam...\r\n",
+      "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ninja: no work to do.\r\n",
+      "Loading extension module fused_adam...\r\n",
+      "Time to load fused_adam op: 0.08282637596130371 seconds\r\n",
+      "Loading `train_dataloader` to estimate number of stepping batches.\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Rank: 0 partition count [1, 1] and sizes[(533245952, False), (384, False)] \r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r\n",
+      "  | Name   | Type       | Params\r\n",
+      "--------------------------------------\r\n",
+      "0 | emb    | Embedding  | 102 M \r\n",
+      "1 | blocks | ModuleList | 327 M \r\n",
+      "2 | ln_out | LayerNorm  | 4.1 K \r\n",
+      "3 | head   | Linear     | 102 M \r\n",
+      "--------------------------------------\r\n",
+      "533 M     Trainable params\r\n",
+      "0         Non-trainable params\r\n",
+      "533 M     Total params\r\n",
+      "2,132.985 Total estimated model params size (MB)\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Training: 0it [00:00, ?it/s]\r",
+      "Training:   0%|                                       | 0/14932 [00:00<?, ?it/s]\r",
+      "Epoch 0:   0%|                                        | 0/14932 [00:00<?, ?it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%|                             | 1/14932 [00:08<35:10:35,  8.48s/it]\r",
+      "Epoch 0:   0%| | 1/14932 [00:08<35:11:00,  8.48s/it, v_num=hpnp, train/loss=4.03"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 2/14932 [00:10<20:59:26,  5.06s/it, v_num=hpnp, train/loss=4.03\r",
+      "Epoch 0:   0%| | 2/14932 [00:10<20:59:33,  5.06s/it, v_num=hpnp, train/loss=6.28"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 3/14932 [00:11<15:12:21,  3.67s/it, v_num=hpnp, train/loss=6.28\r",
+      "Epoch 0:   0%| | 3/14932 [00:11<15:12:25,  3.67s/it, v_num=hpnp, train/loss=5.00"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 4/14932 [00:12<12:31:51,  3.02s/it, v_num=hpnp, train/loss=5.00\r",
+      "Epoch 0:   0%| | 4/14932 [00:12<12:31:54,  3.02s/it, v_num=hpnp, train/loss=3.56"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 5/14932 [00:13<10:54:37,  2.63s/it, v_num=hpnp, train/loss=3.56\r",
+      "Epoch 0:   0%| | 5/14932 [00:13<10:54:41,  2.63s/it, v_num=hpnp, train/loss=4.06"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 6/14932 [00:14<10:14:32,  2.47s/it, v_num=hpnp, train/loss=4.06\r",
+      "Epoch 0:   0%| | 6/14932 [00:14<10:14:48,  2.47s/it, v_num=hpnp, train/loss=3.55"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 7/14932 [00:15<9:23:06,  2.26s/it, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   0%| | 7/14932 [00:15<9:23:33,  2.27s/it, v_num=hpnp, train/loss=4.500"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 8/14932 [00:15<8:15:20,  1.99s/it, v_num=hpnp, train/loss=4.500\r",
+      "Epoch 0:   0%| | 8/14932 [00:15<8:15:22,  1.99s/it, v_num=hpnp, train/loss=4.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 9/14932 [00:16<7:22:18,  1.78s/it, v_num=hpnp, train/loss=4.000\r",
+      "Epoch 0:   0%| | 9/14932 [00:16<7:22:19,  1.78s/it, v_num=hpnp, train/loss=3.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 10/14932 [00:16<6:39:48,  1.61s/it, v_num=hpnp, train/loss=3.84\r",
+      "Epoch 0:   0%| | 10/14932 [00:16<6:39:50,  1.61s/it, v_num=hpnp, train/loss=5.81"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 11/14932 [00:16<6:05:03,  1.47s/it, v_num=hpnp, train/loss=5.81\r",
+      "Epoch 0:   0%| | 11/14932 [00:16<6:05:04,  1.47s/it, v_num=hpnp, train/loss=6.69"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 12/14932 [00:16<5:36:03,  1.35s/it, v_num=hpnp, train/loss=6.69\r",
+      "Epoch 0:   0%| | 12/14932 [00:16<5:36:05,  1.35s/it, v_num=hpnp, train/loss=5.12"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 13/14932 [00:16<5:11:37,  1.25s/it, v_num=hpnp, train/loss=5.12\r",
+      "Epoch 0:   0%| | 13/14932 [00:16<5:11:38,  1.25s/it, v_num=hpnp, train/loss=4.12"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 14/14932 [00:16<4:52:05,  1.17s/it, v_num=hpnp, train/loss=4.12\r",
+      "Epoch 0:   0%| | 14/14932 [00:16<4:52:10,  1.18s/it, v_num=hpnp, train/loss=3.33"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 15/14932 [00:17<4:47:13,  1.16s/it, v_num=hpnp, train/loss=3.33\r",
+      "Epoch 0:   0%| | 15/14932 [00:17<4:47:14,  1.16s/it, v_num=hpnp, train/loss=5.34"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 16/14932 [00:17<4:30:23,  1.09s/it, v_num=hpnp, train/loss=5.34\r",
+      "Epoch 0:   0%| | 16/14932 [00:17<4:30:24,  1.09s/it, v_num=hpnp, train/loss=5.09"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 17/14932 [00:18<4:26:32,  1.07s/it, v_num=hpnp, train/loss=5.09\r",
+      "Epoch 0:   0%| | 17/14932 [00:18<4:26:32,  1.07s/it, v_num=hpnp, train/loss=5.19"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 18/14932 [00:19<4:23:14,  1.06s/it, v_num=hpnp, train/loss=5.19\r",
+      "Epoch 0:   0%| | 18/14932 [00:19<4:23:17,  1.06s/it, v_num=hpnp, train/loss=3.50"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 19/14932 [00:19<4:10:21,  1.01s/it, v_num=hpnp, train/loss=3.50\r",
+      "Epoch 0:   0%| | 19/14932 [00:19<4:10:22,  1.01s/it, v_num=hpnp, train/loss=5.81"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 20/14932 [00:19<3:58:44,  1.04it/s, v_num=hpnp, train/loss=5.81\r",
+      "Epoch 0:   0%| | 20/14932 [00:19<3:58:45,  1.04it/s, v_num=hpnp, train/loss=8.06"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 21/14932 [00:19<3:48:15,  1.09it/s, v_num=hpnp, train/loss=8.06\r",
+      "Epoch 0:   0%| | 21/14932 [00:19<3:48:16,  1.09it/s, v_num=hpnp, train/loss=4.06"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 22/14932 [00:19<3:38:42,  1.14it/s, v_num=hpnp, train/loss=4.06\r",
+      "Epoch 0:   0%| | 22/14932 [00:19<3:38:42,  1.14it/s, v_num=hpnp, train/loss=5.25"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 23/14932 [00:20<3:39:22,  1.13it/s, v_num=hpnp, train/loss=5.25\r",
+      "Epoch 0:   0%| | 23/14932 [00:20<3:39:22,  1.13it/s, v_num=hpnp, train/loss=7.28"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 24/14932 [00:20<3:30:59,  1.18it/s, v_num=hpnp, train/loss=7.28\r",
+      "Epoch 0:   0%| | 24/14932 [00:20<3:30:59,  1.18it/s, v_num=hpnp, train/loss=4.44"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 25/14932 [00:20<3:23:16,  1.22it/s, v_num=hpnp, train/loss=4.44\r",
+      "Epoch 0:   0%| | 25/14932 [00:20<3:23:17,  1.22it/s, v_num=hpnp, train/loss=4.66"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 26/14932 [00:20<3:16:05,  1.27it/s, v_num=hpnp, train/loss=4.66\r",
+      "Epoch 0:   0%| | 26/14932 [00:20<3:16:05,  1.27it/s, v_num=hpnp, train/loss=5.28"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 27/14932 [00:20<3:09:29,  1.31it/s, v_num=hpnp, train/loss=5.28\r",
+      "Epoch 0:   0%| | 27/14932 [00:20<3:09:30,  1.31it/s, v_num=hpnp, train/loss=5.72"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 28/14932 [00:20<3:03:21,  1.35it/s, v_num=hpnp, train/loss=5.72\r",
+      "Epoch 0:   0%| | 28/14932 [00:20<3:03:22,  1.35it/s, v_num=hpnp, train/loss=7.00"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 29/14932 [00:20<2:57:36,  1.40it/s, v_num=hpnp, train/loss=7.00\r",
+      "Epoch 0:   0%| | 29/14932 [00:20<2:57:36,  1.40it/s, v_num=hpnp, train/loss=5.31"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 30/14932 [00:20<2:52:15,  1.44it/s, v_num=hpnp, train/loss=5.31\r",
+      "Epoch 0:   0%| | 30/14932 [00:20<2:52:16,  1.44it/s, v_num=hpnp, train/loss=5.31"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 31/14932 [00:20<2:47:17,  1.48it/s, v_num=hpnp, train/loss=5.31"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 31/14932 [00:20<2:47:17,  1.48it/s, v_num=hpnp, train/loss=5.34"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 32/14932 [00:21<2:43:07,  1.52it/s, v_num=hpnp, train/loss=5.34"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 32/14932 [00:21<2:43:37,  1.52it/s, v_num=hpnp, train/loss=3.94"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 33/14932 [00:21<2:39:11,  1.56it/s, v_num=hpnp, train/loss=3.94\r",
+      "Epoch 0:   0%| | 33/14932 [00:21<2:39:12,  1.56it/s, v_num=hpnp, train/loss=4.94"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 34/14932 [00:21<2:35:03,  1.60it/s, v_num=hpnp, train/loss=4.94\r",
+      "Epoch 0:   0%| | 34/14932 [00:21<2:35:04,  1.60it/s, v_num=hpnp, train/loss=3.38"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 35/14932 [00:21<2:31:08,  1.64it/s, v_num=hpnp, train/loss=3.38\r",
+      "Epoch 0:   0%| | 35/14932 [00:21<2:31:09,  1.64it/s, v_num=hpnp, train/loss=4.09"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 36/14932 [00:21<2:27:26,  1.68it/s, v_num=hpnp, train/loss=4.09\r",
+      "Epoch 0:   0%| | 36/14932 [00:21<2:27:27,  1.68it/s, v_num=hpnp, train/loss=4.03"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 37/14932 [00:21<2:23:56,  1.72it/s, v_num=hpnp, train/loss=4.03\r",
+      "Epoch 0:   0%| | 37/14932 [00:21<2:23:57,  1.72it/s, v_num=hpnp, train/loss=4.78"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 38/14932 [00:21<2:20:37,  1.77it/s, v_num=hpnp, train/loss=4.78\r",
+      "Epoch 0:   0%| | 38/14932 [00:21<2:20:37,  1.77it/s, v_num=hpnp, train/loss=5.59"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 39/14932 [00:21<2:17:29,  1.81it/s, v_num=hpnp, train/loss=5.59\r",
+      "Epoch 0:   0%| | 39/14932 [00:21<2:17:29,  1.81it/s, v_num=hpnp, train/loss=4.12"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 40/14932 [00:21<2:14:29,  1.85it/s, v_num=hpnp, train/loss=4.12\r",
+      "Epoch 0:   0%| | 40/14932 [00:21<2:14:29,  1.85it/s, v_num=hpnp, train/loss=4.44"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 41/14932 [00:21<2:11:38,  1.89it/s, v_num=hpnp, train/loss=4.44\r",
+      "Epoch 0:   0%| | 41/14932 [00:21<2:11:39,  1.89it/s, v_num=hpnp, train/loss=2.58"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 42/14932 [00:21<2:08:56,  1.92it/s, v_num=hpnp, train/loss=2.58\r",
+      "Epoch 0:   0%| | 42/14932 [00:21<2:08:56,  1.92it/s, v_num=hpnp, train/loss=4.50"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 43/14932 [00:21<2:06:21,  1.96it/s, v_num=hpnp, train/loss=4.50\r",
+      "Epoch 0:   0%| | 43/14932 [00:21<2:06:21,  1.96it/s, v_num=hpnp, train/loss=4.91"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 44/14932 [00:21<2:03:53,  2.00it/s, v_num=hpnp, train/loss=4.91\r",
+      "Epoch 0:   0%| | 44/14932 [00:21<2:03:54,  2.00it/s, v_num=hpnp, train/loss=3.70"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 45/14932 [00:22<2:01:31,  2.04it/s, v_num=hpnp, train/loss=3.70\r",
+      "Epoch 0:   0%| | 45/14932 [00:22<2:01:32,  2.04it/s, v_num=hpnp, train/loss=4.53"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 46/14932 [00:22<1:59:15,  2.08it/s, v_num=hpnp, train/loss=4.53\r",
+      "Epoch 0:   0%| | 46/14932 [00:22<1:59:15,  2.08it/s, v_num=hpnp, train/loss=4.75"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 47/14932 [00:22<1:57:03,  2.12it/s, v_num=hpnp, train/loss=4.75\r",
+      "Epoch 0:   0%| | 47/14932 [00:22<1:57:04,  2.12it/s, v_num=hpnp, train/loss=4.69"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 48/14932 [00:22<1:54:59,  2.16it/s, v_num=hpnp, train/loss=4.69\r",
+      "Epoch 0:   0%| | 48/14932 [00:22<1:54:59,  2.16it/s, v_num=hpnp, train/loss=4.47"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 49/14932 [00:22<1:53:00,  2.19it/s, v_num=hpnp, train/loss=4.47\r",
+      "Epoch 0:   0%| | 49/14932 [00:22<1:53:00,  2.19it/s, v_num=hpnp, train/loss=4.28"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 50/14932 [00:22<1:51:27,  2.23it/s, v_num=hpnp, train/loss=4.28\r",
+      "Epoch 0:   0%| | 50/14932 [00:22<1:51:28,  2.23it/s, v_num=hpnp, train/loss=4.50"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 51/14932 [00:22<1:49:43,  2.26it/s, v_num=hpnp, train/loss=4.50"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 51/14932 [00:22<1:49:44,  2.26it/s, v_num=hpnp, train/loss=3.00"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 52/14932 [00:22<1:47:58,  2.30it/s, v_num=hpnp, train/loss=3.00\r",
+      "Epoch 0:   0%| | 52/14932 [00:22<1:47:58,  2.30it/s, v_num=hpnp, train/loss=3.84"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 53/14932 [00:22<1:46:15,  2.33it/s, v_num=hpnp, train/loss=3.84\r",
+      "Epoch 0:   0%| | 53/14932 [00:22<1:46:15,  2.33it/s, v_num=hpnp, train/loss=3.23"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 54/14932 [00:22<1:44:38,  2.37it/s, v_num=hpnp, train/loss=3.23\r",
+      "Epoch 0:   0%| | 54/14932 [00:22<1:44:38,  2.37it/s, v_num=hpnp, train/loss=3.41"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 55/14932 [00:22<1:43:10,  2.40it/s, v_num=hpnp, train/loss=3.41\r",
+      "Epoch 0:   0%| | 55/14932 [00:22<1:43:11,  2.40it/s, v_num=hpnp, train/loss=4.22"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 56/14932 [00:23<1:41:56,  2.43it/s, v_num=hpnp, train/loss=4.22\r",
+      "Epoch 0:   0%| | 56/14932 [00:23<1:41:57,  2.43it/s, v_num=hpnp, train/loss=3.16"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 57/14932 [00:23<1:40:28,  2.47it/s, v_num=hpnp, train/loss=3.16\r",
+      "Epoch 0:   0%| | 57/14932 [00:23<1:40:29,  2.47it/s, v_num=hpnp, train/loss=4.88"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 58/14932 [00:23<1:39:03,  2.50it/s, v_num=hpnp, train/loss=4.88\r",
+      "Epoch 0:   0%| | 58/14932 [00:23<1:39:03,  2.50it/s, v_num=hpnp, train/loss=4.06"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 59/14932 [00:23<1:37:47,  2.53it/s, v_num=hpnp, train/loss=4.06\r",
+      "Epoch 0:   0%| | 59/14932 [00:23<1:37:48,  2.53it/s, v_num=hpnp, train/loss=3.75"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 60/14932 [00:23<1:36:29,  2.57it/s, v_num=hpnp, train/loss=3.75\r",
+      "Epoch 0:   0%| | 60/14932 [00:23<1:36:29,  2.57it/s, v_num=hpnp, train/loss=3.08"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 61/14932 [00:23<1:35:10,  2.60it/s, v_num=hpnp, train/loss=3.08\r",
+      "Epoch 0:   0%| | 61/14932 [00:23<1:35:10,  2.60it/s, v_num=hpnp, train/loss=3.38"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 62/14932 [00:23<1:33:56,  2.64it/s, v_num=hpnp, train/loss=3.38\r",
+      "Epoch 0:   0%| | 62/14932 [00:23<1:33:56,  2.64it/s, v_num=hpnp, train/loss=2.95"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 63/14932 [00:23<1:32:43,  2.67it/s, v_num=hpnp, train/loss=2.95\r",
+      "Epoch 0:   0%| | 63/14932 [00:23<1:32:43,  2.67it/s, v_num=hpnp, train/loss=3.42"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 64/14932 [00:23<1:31:47,  2.70it/s, v_num=hpnp, train/loss=3.42"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 64/14932 [00:23<1:32:02,  2.69it/s, v_num=hpnp, train/loss=4.19"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 65/14932 [00:23<1:30:54,  2.73it/s, v_num=hpnp, train/loss=4.19\r",
+      "Epoch 0:   0%| | 65/14932 [00:23<1:30:54,  2.73it/s, v_num=hpnp, train/loss=3.27"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 66/14932 [00:24<1:32:44,  2.67it/s, v_num=hpnp, train/loss=3.27"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 66/14932 [00:24<1:32:45,  2.67it/s, v_num=hpnp, train/loss=4.38"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 67/14932 [00:24<1:31:39,  2.70it/s, v_num=hpnp, train/loss=4.38\r",
+      "Epoch 0:   0%| | 67/14932 [00:24<1:31:39,  2.70it/s, v_num=hpnp, train/loss=2.86"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 68/14932 [00:24<1:30:50,  2.73it/s, v_num=hpnp, train/loss=2.86\r",
+      "Epoch 0:   0%| | 68/14932 [00:24<1:30:50,  2.73it/s, v_num=hpnp, train/loss=4.38"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 69/14932 [00:25<1:29:49,  2.76it/s, v_num=hpnp, train/loss=4.38\r",
+      "Epoch 0:   0%| | 69/14932 [00:25<1:29:49,  2.76it/s, v_num=hpnp, train/loss=4.84"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 70/14932 [00:25<1:28:46,  2.79it/s, v_num=hpnp, train/loss=4.84\r",
+      "Epoch 0:   0%| | 70/14932 [00:25<1:28:46,  2.79it/s, v_num=hpnp, train/loss=4.25"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 71/14932 [00:25<1:27:47,  2.82it/s, v_num=hpnp, train/loss=4.25"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 71/14932 [00:25<1:27:47,  2.82it/s, v_num=hpnp, train/loss=3.41"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 72/14932 [00:25<1:26:48,  2.85it/s, v_num=hpnp, train/loss=3.41\r",
+      "Epoch 0:   0%| | 72/14932 [00:25<1:26:48,  2.85it/s, v_num=hpnp, train/loss=5.16"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 73/14932 [00:25<1:25:52,  2.88it/s, v_num=hpnp, train/loss=5.16\r",
+      "Epoch 0:   0%| | 73/14932 [00:25<1:25:52,  2.88it/s, v_num=hpnp, train/loss=3.02"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   0%| | 74/14932 [00:25<1:24:56,  2.92it/s, v_num=hpnp, train/loss=3.02\r",
+      "Epoch 0:   0%| | 74/14932 [00:25<1:24:57,  2.92it/s, v_num=hpnp, train/loss=4.03"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 75/14932 [00:25<1:24:03,  2.95it/s, v_num=hpnp, train/loss=4.03\r",
+      "Epoch 0:   1%| | 75/14932 [00:25<1:24:03,  2.95it/s, v_num=hpnp, train/loss=3.50"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 76/14932 [00:25<1:23:11,  2.98it/s, v_num=hpnp, train/loss=3.50\r",
+      "Epoch 0:   1%| | 76/14932 [00:25<1:23:11,  2.98it/s, v_num=hpnp, train/loss=2.55"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 77/14932 [00:25<1:22:20,  3.01it/s, v_num=hpnp, train/loss=2.55\r",
+      "Epoch 0:   1%| | 77/14932 [00:25<1:22:20,  3.01it/s, v_num=hpnp, train/loss=3.75"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 78/14932 [00:25<1:21:29,  3.04it/s, v_num=hpnp, train/loss=3.75\r",
+      "Epoch 0:   1%| | 78/14932 [00:25<1:21:30,  3.04it/s, v_num=hpnp, train/loss=4.59"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 79/14932 [00:25<1:20:41,  3.07it/s, v_num=hpnp, train/loss=4.59\r",
+      "Epoch 0:   1%| | 79/14932 [00:25<1:20:41,  3.07it/s, v_num=hpnp, train/loss=3.78"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 80/14932 [00:25<1:19:58,  3.09it/s, v_num=hpnp, train/loss=3.78\r",
+      "Epoch 0:   1%| | 80/14932 [00:25<1:19:59,  3.09it/s, v_num=hpnp, train/loss=4.22"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 81/14932 [00:25<1:19:13,  3.12it/s, v_num=hpnp, train/loss=4.22\r",
+      "Epoch 0:   1%| | 81/14932 [00:25<1:19:13,  3.12it/s, v_num=hpnp, train/loss=4.44"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 82/14932 [00:25<1:18:28,  3.15it/s, v_num=hpnp, train/loss=4.44\r",
+      "Epoch 0:   1%| | 82/14932 [00:25<1:18:28,  3.15it/s, v_num=hpnp, train/loss=3.36"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 83/14932 [00:26<1:17:45,  3.18it/s, v_num=hpnp, train/loss=3.36\r",
+      "Epoch 0:   1%| | 83/14932 [00:26<1:17:45,  3.18it/s, v_num=hpnp, train/loss=3.73"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 84/14932 [00:26<1:17:01,  3.21it/s, v_num=hpnp, train/loss=3.73\r",
+      "Epoch 0:   1%| | 84/14932 [00:26<1:17:01,  3.21it/s, v_num=hpnp, train/loss=4.34"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 85/14932 [00:26<1:16:35,  3.23it/s, v_num=hpnp, train/loss=4.34\r",
+      "Epoch 0:   1%| | 85/14932 [00:26<1:16:35,  3.23it/s, v_num=hpnp, train/loss=3.81"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 86/14932 [00:26<1:15:56,  3.26it/s, v_num=hpnp, train/loss=3.81\r",
+      "Epoch 0:   1%| | 86/14932 [00:26<1:15:56,  3.26it/s, v_num=hpnp, train/loss=4.91"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 87/14932 [00:26<1:15:16,  3.29it/s, v_num=hpnp, train/loss=4.91\r",
+      "Epoch 0:   1%| | 87/14932 [00:26<1:15:16,  3.29it/s, v_num=hpnp, train/loss=4.19"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 88/14932 [00:26<1:14:37,  3.32it/s, v_num=hpnp, train/loss=4.19\r",
+      "Epoch 0:   1%| | 88/14932 [00:26<1:14:37,  3.32it/s, v_num=hpnp, train/loss=4.41"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 89/14932 [00:26<1:13:59,  3.34it/s, v_num=hpnp, train/loss=4.41\r",
+      "Epoch 0:   1%| | 89/14932 [00:26<1:13:59,  3.34it/s, v_num=hpnp, train/loss=2.81"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 90/14932 [00:26<1:13:22,  3.37it/s, v_num=hpnp, train/loss=2.81\r",
+      "Epoch 0:   1%| | 90/14932 [00:26<1:13:22,  3.37it/s, v_num=hpnp, train/loss=4.00"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 91/14932 [00:26<1:12:45,  3.40it/s, v_num=hpnp, train/loss=4.00\r",
+      "Epoch 0:   1%| | 91/14932 [00:26<1:12:45,  3.40it/s, v_num=hpnp, train/loss=4.09"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 92/14932 [00:26<1:12:09,  3.43it/s, v_num=hpnp, train/loss=4.09\r",
+      "Epoch 0:   1%| | 92/14932 [00:26<1:12:09,  3.43it/s, v_num=hpnp, train/loss=4.44"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 93/14932 [00:26<1:11:34,  3.46it/s, v_num=hpnp, train/loss=4.44\r",
+      "Epoch 0:   1%| | 93/14932 [00:26<1:11:34,  3.46it/s, v_num=hpnp, train/loss=3.47"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 94/14932 [00:26<1:11:00,  3.48it/s, v_num=hpnp, train/loss=3.47\r",
+      "Epoch 0:   1%| | 94/14932 [00:26<1:11:00,  3.48it/s, v_num=hpnp, train/loss=3.67"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 95/14932 [00:27<1:10:30,  3.51it/s, v_num=hpnp, train/loss=3.67\r",
+      "Epoch 0:   1%| | 95/14932 [00:27<1:10:30,  3.51it/s, v_num=hpnp, train/loss=3.73"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 96/14932 [00:27<1:10:16,  3.52it/s, v_num=hpnp, train/loss=3.73\r",
+      "Epoch 0:   1%| | 96/14932 [00:27<1:10:17,  3.52it/s, v_num=hpnp, train/loss=4.31"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 97/14932 [00:27<1:09:45,  3.54it/s, v_num=hpnp, train/loss=4.31\r",
+      "Epoch 0:   1%| | 97/14932 [00:27<1:09:45,  3.54it/s, v_num=hpnp, train/loss=3.88"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 98/14932 [00:27<1:09:15,  3.57it/s, v_num=hpnp, train/loss=3.88\r",
+      "Epoch 0:   1%| | 98/14932 [00:27<1:09:15,  3.57it/s, v_num=hpnp, train/loss=3.55"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 99/14932 [00:27<1:08:44,  3.60it/s, v_num=hpnp, train/loss=3.55\r",
+      "Epoch 0:   1%| | 99/14932 [00:27<1:08:44,  3.60it/s, v_num=hpnp, train/loss=4.22"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 100/14932 [00:27<1:08:14,  3.62it/s, v_num=hpnp, train/loss=4.2\r",
+      "Epoch 0:   1%| | 100/14932 [00:27<1:08:14,  3.62it/s, v_num=hpnp, train/loss=4.0"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 101/14932 [00:27<1:07:43,  3.65it/s, v_num=hpnp, train/loss=4.0\r",
+      "Epoch 0:   1%| | 101/14932 [00:27<1:07:43,  3.65it/s, v_num=hpnp, train/loss=4.0"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 102/14932 [00:27<1:07:13,  3.68it/s, v_num=hpnp, train/loss=4.0\r",
+      "Epoch 0:   1%| | 102/14932 [00:27<1:07:14,  3.68it/s, v_num=hpnp, train/loss=4.1"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 103/14932 [00:27<1:06:45,  3.70it/s, v_num=hpnp, train/loss=4.1\r",
+      "Epoch 0:   1%| | 103/14932 [00:27<1:06:45,  3.70it/s, v_num=hpnp, train/loss=4.8"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 104/14932 [00:27<1:06:18,  3.73it/s, v_num=hpnp, train/loss=4.8\r",
+      "Epoch 0:   1%| | 104/14932 [00:27<1:06:18,  3.73it/s, v_num=hpnp, train/loss=2.4"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 105/14932 [00:27<1:05:50,  3.75it/s, v_num=hpnp, train/loss=2.4\r",
+      "Epoch 0:   1%| | 105/14932 [00:27<1:05:50,  3.75it/s, v_num=hpnp, train/loss=3.6"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 106/14932 [00:28<1:05:23,  3.78it/s, v_num=hpnp, train/loss=3.6\r",
+      "Epoch 0:   1%| | 106/14932 [00:28<1:05:23,  3.78it/s, v_num=hpnp, train/loss=3.4"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 107/14932 [00:28<1:04:56,  3.81it/s, v_num=hpnp, train/loss=3.4\r",
+      "Epoch 0:   1%| | 107/14932 [00:28<1:04:56,  3.81it/s, v_num=hpnp, train/loss=2.7"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 108/14932 [00:28<1:04:30,  3.83it/s, v_num=hpnp, train/loss=2.7\r",
+      "Epoch 0:   1%| | 108/14932 [00:28<1:04:30,  3.83it/s, v_num=hpnp, train/loss=4.1"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 109/14932 [00:28<1:04:04,  3.86it/s, v_num=hpnp, train/loss=4.1\r",
+      "Epoch 0:   1%| | 109/14932 [00:28<1:04:05,  3.86it/s, v_num=hpnp, train/loss=4.5"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 110/14932 [00:28<1:03:38,  3.88it/s, v_num=hpnp, train/loss=4.5\r",
+      "Epoch 0:   1%| | 110/14932 [00:28<1:03:38,  3.88it/s, v_num=hpnp, train/loss=4.3"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 111/14932 [00:28<1:03:13,  3.91it/s, v_num=hpnp, train/loss=4.3\r",
+      "Epoch 0:   1%| | 111/14932 [00:28<1:03:13,  3.91it/s, v_num=hpnp, train/loss=3.6"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 112/14932 [00:28<1:02:49,  3.93it/s, v_num=hpnp, train/loss=3.6\r",
+      "Epoch 0:   1%| | 112/14932 [00:28<1:02:49,  3.93it/s, v_num=hpnp, train/loss=4.1"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 113/14932 [00:29<1:04:13,  3.85it/s, v_num=hpnp, train/loss=4.1\r",
+      "Epoch 0:   1%| | 113/14932 [00:29<1:04:13,  3.85it/s, v_num=hpnp, train/loss=3.3"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 114/14932 [00:29<1:03:48,  3.87it/s, v_num=hpnp, train/loss=3.3\r",
+      "Epoch 0:   1%| | 114/14932 [00:29<1:03:48,  3.87it/s, v_num=hpnp, train/loss=3.5"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 115/14932 [00:29<1:03:24,  3.89it/s, v_num=hpnp, train/loss=3.5\r",
+      "Epoch 0:   1%| | 115/14932 [00:29<1:03:24,  3.89it/s, v_num=hpnp, train/loss=3.9"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 116/14932 [00:29<1:03:00,  3.92it/s, v_num=hpnp, train/loss=3.9\r",
+      "Epoch 0:   1%| | 116/14932 [00:29<1:03:00,  3.92it/s, v_num=hpnp, train/loss=2.6"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 117/14932 [00:29<1:02:36,  3.94it/s, v_num=hpnp, train/loss=2.6\r",
+      "Epoch 0:   1%| | 117/14932 [00:29<1:02:36,  3.94it/s, v_num=hpnp, train/loss=3.2"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 118/14932 [00:29<1:02:13,  3.97it/s, v_num=hpnp, train/loss=3.2\r",
+      "Epoch 0:   1%| | 118/14932 [00:29<1:02:14,  3.97it/s, v_num=hpnp, train/loss=4.5"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 119/14932 [00:29<1:01:52,  3.99it/s, v_num=hpnp, train/loss=4.5\r",
+      "Epoch 0:   1%| | 119/14932 [00:29<1:01:52,  3.99it/s, v_num=hpnp, train/loss=3.1"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 120/14932 [00:29<1:01:30,  4.01it/s, v_num=hpnp, train/loss=3.1\r",
+      "Epoch 0:   1%| | 120/14932 [00:29<1:01:31,  4.01it/s, v_num=hpnp, train/loss=3.0"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 121/14932 [00:29<1:01:09,  4.04it/s, v_num=hpnp, train/loss=3.0\r",
+      "Epoch 0:   1%| | 121/14932 [00:29<1:01:09,  4.04it/s, v_num=hpnp, train/loss=3.7"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 122/14932 [00:30<1:01:08,  4.04it/s, v_num=hpnp, train/loss=3.7"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 122/14932 [00:30<1:01:08,  4.04it/s, v_num=hpnp, train/loss=4.0"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 123/14932 [00:30<1:01:12,  4.03it/s, v_num=hpnp, train/loss=4.0\r",
+      "Epoch 0:   1%| | 123/14932 [00:30<1:01:12,  4.03it/s, v_num=hpnp, train/loss=4.6"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 124/14932 [00:30<1:00:50,  4.06it/s, v_num=hpnp, train/loss=4.6\r",
+      "Epoch 0:   1%| | 124/14932 [00:30<1:00:51,  4.06it/s, v_num=hpnp, train/loss=3.0"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 125/14932 [00:30<1:00:29,  4.08it/s, v_num=hpnp, train/loss=3.0\r",
+      "Epoch 0:   1%| | 125/14932 [00:30<1:00:29,  4.08it/s, v_num=hpnp, train/loss=3.1"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 126/14932 [00:30<1:00:09,  4.10it/s, v_num=hpnp, train/loss=3.1\r",
+      "Epoch 0:   1%| | 126/14932 [00:30<1:00:09,  4.10it/s, v_num=hpnp, train/loss=3.0"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 127/14932 [00:30<59:48,  4.13it/s, v_num=hpnp, train/loss=3.030\r",
+      "Epoch 0:   1%| | 127/14932 [00:30<59:48,  4.13it/s, v_num=hpnp, train/loss=2.360"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 128/14932 [00:30<59:42,  4.13it/s, v_num=hpnp, train/loss=2.360\r",
+      "Epoch 0:   1%| | 128/14932 [00:30<59:42,  4.13it/s, v_num=hpnp, train/loss=2.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 129/14932 [00:31<59:22,  4.16it/s, v_num=hpnp, train/loss=2.840\r",
+      "Epoch 0:   1%| | 129/14932 [00:31<59:22,  4.15it/s, v_num=hpnp, train/loss=2.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 130/14932 [00:31<59:03,  4.18it/s, v_num=hpnp, train/loss=2.880\r",
+      "Epoch 0:   1%| | 130/14932 [00:31<59:03,  4.18it/s, v_num=hpnp, train/loss=4.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 131/14932 [00:31<58:45,  4.20it/s, v_num=hpnp, train/loss=4.750\r",
+      "Epoch 0:   1%| | 131/14932 [00:31<58:45,  4.20it/s, v_num=hpnp, train/loss=3.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 132/14932 [00:31<58:26,  4.22it/s, v_num=hpnp, train/loss=3.610\r",
+      "Epoch 0:   1%| | 132/14932 [00:31<58:26,  4.22it/s, v_num=hpnp, train/loss=3.580"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 133/14932 [00:31<58:07,  4.24it/s, v_num=hpnp, train/loss=3.580\r",
+      "Epoch 0:   1%| | 133/14932 [00:31<58:07,  4.24it/s, v_num=hpnp, train/loss=1.980"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 134/14932 [00:31<57:50,  4.26it/s, v_num=hpnp, train/loss=1.980\r",
+      "Epoch 0:   1%| | 134/14932 [00:31<57:50,  4.26it/s, v_num=hpnp, train/loss=3.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 135/14932 [00:31<57:33,  4.28it/s, v_num=hpnp, train/loss=3.730\r",
+      "Epoch 0:   1%| | 135/14932 [00:31<57:33,  4.28it/s, v_num=hpnp, train/loss=4.060"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 136/14932 [00:31<57:16,  4.31it/s, v_num=hpnp, train/loss=4.060\r",
+      "Epoch 0:   1%| | 136/14932 [00:31<57:16,  4.31it/s, v_num=hpnp, train/loss=2.640"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 137/14932 [00:31<56:58,  4.33it/s, v_num=hpnp, train/loss=2.640\r",
+      "Epoch 0:   1%| | 137/14932 [00:31<56:58,  4.33it/s, v_num=hpnp, train/loss=2.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 138/14932 [00:31<56:41,  4.35it/s, v_num=hpnp, train/loss=2.830\r",
+      "Epoch 0:   1%| | 138/14932 [00:31<56:41,  4.35it/s, v_num=hpnp, train/loss=2.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 139/14932 [00:31<56:24,  4.37it/s, v_num=hpnp, train/loss=2.560\r",
+      "Epoch 0:   1%| | 139/14932 [00:31<56:24,  4.37it/s, v_num=hpnp, train/loss=1.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 140/14932 [00:31<56:17,  4.38it/s, v_num=hpnp, train/loss=1.660\r",
+      "Epoch 0:   1%| | 140/14932 [00:31<56:17,  4.38it/s, v_num=hpnp, train/loss=4.090"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 141/14932 [00:32<56:00,  4.40it/s, v_num=hpnp, train/loss=4.090\r",
+      "Epoch 0:   1%| | 141/14932 [00:32<56:00,  4.40it/s, v_num=hpnp, train/loss=3.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 142/14932 [00:32<55:44,  4.42it/s, v_num=hpnp, train/loss=3.160\r",
+      "Epoch 0:   1%| | 142/14932 [00:32<55:44,  4.42it/s, v_num=hpnp, train/loss=3.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 143/14932 [00:32<55:27,  4.44it/s, v_num=hpnp, train/loss=3.970\r",
+      "Epoch 0:   1%| | 143/14932 [00:32<55:27,  4.44it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 144/14932 [00:32<55:12,  4.46it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   1%| | 144/14932 [00:32<55:12,  4.46it/s, v_num=hpnp, train/loss=3.450"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 145/14932 [00:32<54:57,  4.48it/s, v_num=hpnp, train/loss=3.450\r",
+      "Epoch 0:   1%| | 145/14932 [00:32<54:57,  4.48it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 146/14932 [00:32<54:41,  4.51it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   1%| | 146/14932 [00:32<54:41,  4.51it/s, v_num=hpnp, train/loss=3.500"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 147/14932 [00:32<54:26,  4.53it/s, v_num=hpnp, train/loss=3.500\r",
+      "Epoch 0:   1%| | 147/14932 [00:32<54:26,  4.53it/s, v_num=hpnp, train/loss=3.700"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 148/14932 [00:32<54:11,  4.55it/s, v_num=hpnp, train/loss=3.700\r",
+      "Epoch 0:   1%| | 148/14932 [00:32<54:11,  4.55it/s, v_num=hpnp, train/loss=3.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 149/14932 [00:32<53:56,  4.57it/s, v_num=hpnp, train/loss=3.590\r",
+      "Epoch 0:   1%| | 149/14932 [00:32<53:56,  4.57it/s, v_num=hpnp, train/loss=3.520"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 150/14932 [00:32<53:44,  4.58it/s, v_num=hpnp, train/loss=3.520\r",
+      "Epoch 0:   1%| | 150/14932 [00:32<53:44,  4.58it/s, v_num=hpnp, train/loss=3.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 151/14932 [00:32<53:36,  4.60it/s, v_num=hpnp, train/loss=3.810\r",
+      "Epoch 0:   1%| | 151/14932 [00:32<53:36,  4.60it/s, v_num=hpnp, train/loss=2.890"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 152/14932 [00:32<53:21,  4.62it/s, v_num=hpnp, train/loss=2.890\r",
+      "Epoch 0:   1%| | 152/14932 [00:32<53:22,  4.62it/s, v_num=hpnp, train/loss=3.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 153/14932 [00:33<53:07,  4.64it/s, v_num=hpnp, train/loss=3.000\r",
+      "Epoch 0:   1%| | 153/14932 [00:33<53:07,  4.64it/s, v_num=hpnp, train/loss=3.120"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 154/14932 [00:33<52:53,  4.66it/s, v_num=hpnp, train/loss=3.120\r",
+      "Epoch 0:   1%| | 154/14932 [00:33<52:53,  4.66it/s, v_num=hpnp, train/loss=3.280"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 155/14932 [00:33<52:40,  4.68it/s, v_num=hpnp, train/loss=3.280\r",
+      "Epoch 0:   1%| | 155/14932 [00:33<52:40,  4.67it/s, v_num=hpnp, train/loss=3.080"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 156/14932 [00:33<52:27,  4.69it/s, v_num=hpnp, train/loss=3.080\r",
+      "Epoch 0:   1%| | 156/14932 [00:33<52:27,  4.69it/s, v_num=hpnp, train/loss=3.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 157/14932 [00:33<52:14,  4.71it/s, v_num=hpnp, train/loss=3.300\r",
+      "Epoch 0:   1%| | 157/14932 [00:33<52:14,  4.71it/s, v_num=hpnp, train/loss=2.470"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 158/14932 [00:33<52:01,  4.73it/s, v_num=hpnp, train/loss=2.470\r",
+      "Epoch 0:   1%| | 158/14932 [00:33<52:01,  4.73it/s, v_num=hpnp, train/loss=2.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 159/14932 [00:33<51:48,  4.75it/s, v_num=hpnp, train/loss=2.730\r",
+      "Epoch 0:   1%| | 159/14932 [00:33<51:48,  4.75it/s, v_num=hpnp, train/loss=3.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 160/14932 [00:33<51:48,  4.75it/s, v_num=hpnp, train/loss=3.610\r",
+      "Epoch 0:   1%| | 160/14932 [00:33<51:48,  4.75it/s, v_num=hpnp, train/loss=4.690"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 161/14932 [00:33<51:34,  4.77it/s, v_num=hpnp, train/loss=4.690\r",
+      "Epoch 0:   1%| | 161/14932 [00:33<51:34,  4.77it/s, v_num=hpnp, train/loss=3.170"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 162/14932 [00:33<51:22,  4.79it/s, v_num=hpnp, train/loss=3.170\r",
+      "Epoch 0:   1%| | 162/14932 [00:33<51:22,  4.79it/s, v_num=hpnp, train/loss=3.200"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 163/14932 [00:33<51:10,  4.81it/s, v_num=hpnp, train/loss=3.200\r",
+      "Epoch 0:   1%| | 163/14932 [00:33<51:10,  4.81it/s, v_num=hpnp, train/loss=2.580"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 164/14932 [00:33<50:57,  4.83it/s, v_num=hpnp, train/loss=2.580\r",
+      "Epoch 0:   1%| | 164/14932 [00:33<50:57,  4.83it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 165/14932 [00:34<50:45,  4.85it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   1%| | 165/14932 [00:34<50:45,  4.85it/s, v_num=hpnp, train/loss=2.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 166/14932 [00:34<50:33,  4.87it/s, v_num=hpnp, train/loss=2.810\r",
+      "Epoch 0:   1%| | 166/14932 [00:34<50:33,  4.87it/s, v_num=hpnp, train/loss=3.720"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 167/14932 [00:34<50:22,  4.89it/s, v_num=hpnp, train/loss=3.720\r",
+      "Epoch 0:   1%| | 167/14932 [00:34<50:22,  4.89it/s, v_num=hpnp, train/loss=4.280"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 168/14932 [00:34<50:11,  4.90it/s, v_num=hpnp, train/loss=4.280\r",
+      "Epoch 0:   1%| | 168/14932 [00:34<50:11,  4.90it/s, v_num=hpnp, train/loss=3.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 169/14932 [00:34<49:59,  4.92it/s, v_num=hpnp, train/loss=3.860\r",
+      "Epoch 0:   1%| | 169/14932 [00:34<49:59,  4.92it/s, v_num=hpnp, train/loss=2.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 170/14932 [00:34<49:53,  4.93it/s, v_num=hpnp, train/loss=2.550\r",
+      "Epoch 0:   1%| | 170/14932 [00:34<49:53,  4.93it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 171/14932 [00:34<49:42,  4.95it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   1%| | 171/14932 [00:34<49:42,  4.95it/s, v_num=hpnp, train/loss=3.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 172/14932 [00:34<49:31,  4.97it/s, v_num=hpnp, train/loss=3.560\r",
+      "Epoch 0:   1%| | 172/14932 [00:34<49:31,  4.97it/s, v_num=hpnp, train/loss=4.280"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 173/14932 [00:34<49:22,  4.98it/s, v_num=hpnp, train/loss=4.280\r",
+      "Epoch 0:   1%| | 173/14932 [00:34<49:22,  4.98it/s, v_num=hpnp, train/loss=3.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 174/14932 [00:34<49:10,  5.00it/s, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   1%| | 174/14932 [00:34<49:10,  5.00it/s, v_num=hpnp, train/loss=3.020"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 175/14932 [00:34<49:00,  5.02it/s, v_num=hpnp, train/loss=3.020\r",
+      "Epoch 0:   1%| | 175/14932 [00:34<49:00,  5.02it/s, v_num=hpnp, train/loss=3.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 176/14932 [00:34<48:49,  5.04it/s, v_num=hpnp, train/loss=3.160\r",
+      "Epoch 0:   1%| | 176/14932 [00:34<48:49,  5.04it/s, v_num=hpnp, train/loss=3.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 177/14932 [00:35<48:39,  5.05it/s, v_num=hpnp, train/loss=3.970\r",
+      "Epoch 0:   1%| | 177/14932 [00:35<48:39,  5.05it/s, v_num=hpnp, train/loss=2.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 178/14932 [00:35<48:28,  5.07it/s, v_num=hpnp, train/loss=2.940\r",
+      "Epoch 0:   1%| | 178/14932 [00:35<48:28,  5.07it/s, v_num=hpnp, train/loss=1.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 179/14932 [00:35<48:24,  5.08it/s, v_num=hpnp, train/loss=1.940\r",
+      "Epoch 0:   1%| | 179/14932 [00:35<48:24,  5.08it/s, v_num=hpnp, train/loss=3.670"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 180/14932 [00:35<48:14,  5.10it/s, v_num=hpnp, train/loss=3.670\r",
+      "Epoch 0:   1%| | 180/14932 [00:35<48:14,  5.10it/s, v_num=hpnp, train/loss=4.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 181/14932 [00:35<48:03,  5.11it/s, v_num=hpnp, train/loss=4.030\r",
+      "Epoch 0:   1%| | 181/14932 [00:35<48:04,  5.11it/s, v_num=hpnp, train/loss=3.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 182/14932 [00:35<47:53,  5.13it/s, v_num=hpnp, train/loss=3.220\r",
+      "Epoch 0:   1%| | 182/14932 [00:35<47:53,  5.13it/s, v_num=hpnp, train/loss=3.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 183/14932 [00:35<47:44,  5.15it/s, v_num=hpnp, train/loss=3.300\r",
+      "Epoch 0:   1%| | 183/14932 [00:35<47:44,  5.15it/s, v_num=hpnp, train/loss=3.230"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 184/14932 [00:35<47:33,  5.17it/s, v_num=hpnp, train/loss=3.230\r",
+      "Epoch 0:   1%| | 184/14932 [00:35<47:33,  5.17it/s, v_num=hpnp, train/loss=2.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 185/14932 [00:35<47:23,  5.19it/s, v_num=hpnp, train/loss=2.610\r",
+      "Epoch 0:   1%| | 185/14932 [00:35<47:23,  5.19it/s, v_num=hpnp, train/loss=4.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 186/14932 [00:35<47:14,  5.20it/s, v_num=hpnp, train/loss=4.310\r",
+      "Epoch 0:   1%| | 186/14932 [00:35<47:14,  5.20it/s, v_num=hpnp, train/loss=3.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 187/14932 [00:35<47:04,  5.22it/s, v_num=hpnp, train/loss=3.000\r",
+      "Epoch 0:   1%| | 187/14932 [00:35<47:04,  5.22it/s, v_num=hpnp, train/loss=3.050"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 188/14932 [00:35<46:55,  5.24it/s, v_num=hpnp, train/loss=3.050\r",
+      "Epoch 0:   1%| | 188/14932 [00:35<46:55,  5.24it/s, v_num=hpnp, train/loss=2.890"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 189/14932 [00:35<46:46,  5.25it/s, v_num=hpnp, train/loss=2.890\r",
+      "Epoch 0:   1%| | 189/14932 [00:35<46:46,  5.25it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 190/14932 [00:36<46:37,  5.27it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   1%| | 190/14932 [00:36<46:37,  5.27it/s, v_num=hpnp, train/loss=4.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 191/14932 [00:36<46:28,  5.29it/s, v_num=hpnp, train/loss=4.310\r",
+      "Epoch 0:   1%| | 191/14932 [00:36<46:28,  5.29it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 192/14932 [00:36<46:31,  5.28it/s, v_num=hpnp, train/loss=3.780\r",
+      "Epoch 0:   1%| | 192/14932 [00:36<46:31,  5.28it/s, v_num=hpnp, train/loss=3.520"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 193/14932 [00:36<46:21,  5.30it/s, v_num=hpnp, train/loss=3.520\r",
+      "Epoch 0:   1%| | 193/14932 [00:36<46:21,  5.30it/s, v_num=hpnp, train/loss=1.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 194/14932 [00:36<46:13,  5.31it/s, v_num=hpnp, train/loss=1.880\r",
+      "Epoch 0:   1%| | 194/14932 [00:36<46:13,  5.31it/s, v_num=hpnp, train/loss=3.410"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 195/14932 [00:36<46:04,  5.33it/s, v_num=hpnp, train/loss=3.410\r",
+      "Epoch 0:   1%| | 195/14932 [00:36<46:04,  5.33it/s, v_num=hpnp, train/loss=2.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 196/14932 [00:36<45:57,  5.34it/s, v_num=hpnp, train/loss=2.380\r",
+      "Epoch 0:   1%| | 196/14932 [00:36<45:57,  5.34it/s, v_num=hpnp, train/loss=2.920"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 197/14932 [00:36<45:49,  5.36it/s, v_num=hpnp, train/loss=2.920\r",
+      "Epoch 0:   1%| | 197/14932 [00:36<45:49,  5.36it/s, v_num=hpnp, train/loss=4.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 198/14932 [00:36<45:40,  5.38it/s, v_num=hpnp, train/loss=4.840\r",
+      "Epoch 0:   1%| | 198/14932 [00:36<45:40,  5.38it/s, v_num=hpnp, train/loss=2.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 199/14932 [00:36<45:32,  5.39it/s, v_num=hpnp, train/loss=2.950\r",
+      "Epoch 0:   1%| | 199/14932 [00:36<45:32,  5.39it/s, v_num=hpnp, train/loss=3.450"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 200/14932 [00:36<45:23,  5.41it/s, v_num=hpnp, train/loss=3.450\r",
+      "Epoch 0:   1%| | 200/14932 [00:36<45:23,  5.41it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 201/14932 [00:37<45:15,  5.43it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   1%| | 201/14932 [00:37<45:15,  5.43it/s, v_num=hpnp, train/loss=2.620"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 202/14932 [00:37<45:06,  5.44it/s, v_num=hpnp, train/loss=2.620\r",
+      "Epoch 0:   1%| | 202/14932 [00:37<45:06,  5.44it/s, v_num=hpnp, train/loss=2.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 203/14932 [00:37<44:59,  5.46it/s, v_num=hpnp, train/loss=2.250\r",
+      "Epoch 0:   1%| | 203/14932 [00:37<44:59,  5.46it/s, v_num=hpnp, train/loss=3.640"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 204/14932 [00:37<44:51,  5.47it/s, v_num=hpnp, train/loss=3.640\r",
+      "Epoch 0:   1%| | 204/14932 [00:37<44:51,  5.47it/s, v_num=hpnp, train/loss=3.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 205/14932 [00:37<44:44,  5.49it/s, v_num=hpnp, train/loss=3.610\r",
+      "Epoch 0:   1%| | 205/14932 [00:37<44:44,  5.49it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 206/14932 [00:37<44:35,  5.50it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   1%| | 206/14932 [00:37<44:35,  5.50it/s, v_num=hpnp, train/loss=3.110"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 207/14932 [00:37<44:27,  5.52it/s, v_num=hpnp, train/loss=3.110\r",
+      "Epoch 0:   1%| | 207/14932 [00:37<44:27,  5.52it/s, v_num=hpnp, train/loss=3.920"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 208/14932 [00:37<44:20,  5.53it/s, v_num=hpnp, train/loss=3.920\r",
+      "Epoch 0:   1%| | 208/14932 [00:37<44:20,  5.53it/s, v_num=hpnp, train/loss=3.690"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 209/14932 [00:37<44:12,  5.55it/s, v_num=hpnp, train/loss=3.690\r",
+      "Epoch 0:   1%| | 209/14932 [00:37<44:12,  5.55it/s, v_num=hpnp, train/loss=3.120"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 210/14932 [00:37<44:05,  5.57it/s, v_num=hpnp, train/loss=3.120\r",
+      "Epoch 0:   1%| | 210/14932 [00:37<44:05,  5.57it/s, v_num=hpnp, train/loss=2.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 211/14932 [00:37<43:57,  5.58it/s, v_num=hpnp, train/loss=2.830\r",
+      "Epoch 0:   1%| | 211/14932 [00:37<43:58,  5.58it/s, v_num=hpnp, train/loss=2.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 212/14932 [00:37<43:51,  5.59it/s, v_num=hpnp, train/loss=2.970\r",
+      "Epoch 0:   1%| | 212/14932 [00:37<43:51,  5.59it/s, v_num=hpnp, train/loss=3.110"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 213/14932 [00:37<43:43,  5.61it/s, v_num=hpnp, train/loss=3.110\r",
+      "Epoch 0:   1%| | 213/14932 [00:37<43:43,  5.61it/s, v_num=hpnp, train/loss=4.690"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 214/14932 [00:38<43:35,  5.63it/s, v_num=hpnp, train/loss=4.690\r",
+      "Epoch 0:   1%| | 214/14932 [00:38<43:35,  5.63it/s, v_num=hpnp, train/loss=3.580"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 215/14932 [00:38<43:28,  5.64it/s, v_num=hpnp, train/loss=3.580\r",
+      "Epoch 0:   1%| | 215/14932 [00:38<43:28,  5.64it/s, v_num=hpnp, train/loss=3.110"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 216/14932 [00:38<43:20,  5.66it/s, v_num=hpnp, train/loss=3.110\r",
+      "Epoch 0:   1%| | 216/14932 [00:38<43:20,  5.66it/s, v_num=hpnp, train/loss=3.410"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 217/14932 [00:38<43:13,  5.67it/s, v_num=hpnp, train/loss=3.410\r",
+      "Epoch 0:   1%| | 217/14932 [00:38<43:13,  5.67it/s, v_num=hpnp, train/loss=2.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 218/14932 [00:38<43:05,  5.69it/s, v_num=hpnp, train/loss=2.340\r",
+      "Epoch 0:   1%| | 218/14932 [00:38<43:06,  5.69it/s, v_num=hpnp, train/loss=1.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 219/14932 [00:38<42:59,  5.70it/s, v_num=hpnp, train/loss=1.830\r",
+      "Epoch 0:   1%| | 219/14932 [00:38<42:59,  5.70it/s, v_num=hpnp, train/loss=2.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 220/14932 [00:38<42:52,  5.72it/s, v_num=hpnp, train/loss=2.950\r",
+      "Epoch 0:   1%| | 220/14932 [00:38<42:52,  5.72it/s, v_num=hpnp, train/loss=3.670"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 221/14932 [00:38<42:45,  5.73it/s, v_num=hpnp, train/loss=3.670\r",
+      "Epoch 0:   1%| | 221/14932 [00:38<42:45,  5.73it/s, v_num=hpnp, train/loss=3.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 222/14932 [00:38<42:39,  5.75it/s, v_num=hpnp, train/loss=3.380\r",
+      "Epoch 0:   1%| | 222/14932 [00:38<42:39,  5.75it/s, v_num=hpnp, train/loss=4.500"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   1%| | 223/14932 [00:38<42:41,  5.74it/s, v_num=hpnp, train/loss=4.500\r",
+      "Epoch 0:   1%| | 223/14932 [00:38<42:41,  5.74it/s, v_num=hpnp, train/loss=4.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 224/14932 [00:39<42:43,  5.74it/s, v_num=hpnp, train/loss=4.030\r",
+      "Epoch 0:   2%| | 224/14932 [00:39<42:43,  5.74it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 225/14932 [00:39<42:37,  5.75it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   2%| | 225/14932 [00:39<42:37,  5.75it/s, v_num=hpnp, train/loss=4.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 226/14932 [00:39<42:31,  5.76it/s, v_num=hpnp, train/loss=4.220\r",
+      "Epoch 0:   2%| | 226/14932 [00:39<42:31,  5.76it/s, v_num=hpnp, train/loss=3.620"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 227/14932 [00:39<42:24,  5.78it/s, v_num=hpnp, train/loss=3.620\r",
+      "Epoch 0:   2%| | 227/14932 [00:39<42:24,  5.78it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 228/14932 [00:39<42:18,  5.79it/s, v_num=hpnp, train/loss=3.780\r",
+      "Epoch 0:   2%| | 228/14932 [00:39<42:18,  5.79it/s, v_num=hpnp, train/loss=3.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 229/14932 [00:39<42:11,  5.81it/s, v_num=hpnp, train/loss=3.940\r",
+      "Epoch 0:   2%| | 229/14932 [00:39<42:12,  5.81it/s, v_num=hpnp, train/loss=2.390"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 230/14932 [00:39<42:05,  5.82it/s, v_num=hpnp, train/loss=2.390\r",
+      "Epoch 0:   2%| | 230/14932 [00:39<42:05,  5.82it/s, v_num=hpnp, train/loss=5.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 231/14932 [00:39<41:59,  5.83it/s, v_num=hpnp, train/loss=5.310\r",
+      "Epoch 0:   2%| | 231/14932 [00:39<41:59,  5.83it/s, v_num=hpnp, train/loss=3.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 232/14932 [00:39<41:53,  5.85it/s, v_num=hpnp, train/loss=3.830\r",
+      "Epoch 0:   2%| | 232/14932 [00:39<41:53,  5.85it/s, v_num=hpnp, train/loss=2.910"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 233/14932 [00:39<41:51,  5.85it/s, v_num=hpnp, train/loss=2.910\r",
+      "Epoch 0:   2%| | 233/14932 [00:39<41:51,  5.85it/s, v_num=hpnp, train/loss=2.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 234/14932 [00:39<41:45,  5.87it/s, v_num=hpnp, train/loss=2.860\r",
+      "Epoch 0:   2%| | 234/14932 [00:39<41:45,  5.87it/s, v_num=hpnp, train/loss=2.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 235/14932 [00:39<41:39,  5.88it/s, v_num=hpnp, train/loss=2.810\r",
+      "Epoch 0:   2%| | 235/14932 [00:39<41:39,  5.88it/s, v_num=hpnp, train/loss=2.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 236/14932 [00:40<41:33,  5.89it/s, v_num=hpnp, train/loss=2.380\r",
+      "Epoch 0:   2%| | 236/14932 [00:40<41:33,  5.89it/s, v_num=hpnp, train/loss=2.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 237/14932 [00:40<41:27,  5.91it/s, v_num=hpnp, train/loss=2.840\r",
+      "Epoch 0:   2%| | 237/14932 [00:40<41:27,  5.91it/s, v_num=hpnp, train/loss=3.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 238/14932 [00:40<41:22,  5.92it/s, v_num=hpnp, train/loss=3.560\r",
+      "Epoch 0:   2%| | 238/14932 [00:40<41:22,  5.92it/s, v_num=hpnp, train/loss=3.720"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 239/14932 [00:40<41:16,  5.93it/s, v_num=hpnp, train/loss=3.720\r",
+      "Epoch 0:   2%| | 239/14932 [00:40<41:16,  5.93it/s, v_num=hpnp, train/loss=4.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 240/14932 [00:40<41:10,  5.95it/s, v_num=hpnp, train/loss=4.030\r",
+      "Epoch 0:   2%| | 240/14932 [00:40<41:10,  5.95it/s, v_num=hpnp, train/loss=3.200"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 241/14932 [00:40<41:04,  5.96it/s, v_num=hpnp, train/loss=3.200\r",
+      "Epoch 0:   2%| | 241/14932 [00:40<41:05,  5.96it/s, v_num=hpnp, train/loss=2.500"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 242/14932 [00:40<40:59,  5.97it/s, v_num=hpnp, train/loss=2.500\r",
+      "Epoch 0:   2%| | 242/14932 [00:40<40:59,  5.97it/s, v_num=hpnp, train/loss=3.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 243/14932 [00:40<40:54,  5.99it/s, v_num=hpnp, train/loss=3.860\r",
+      "Epoch 0:   2%| | 243/14932 [00:40<40:54,  5.99it/s, v_num=hpnp, train/loss=2.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 244/14932 [00:40<40:48,  6.00it/s, v_num=hpnp, train/loss=2.730\r",
+      "Epoch 0:   2%| | 244/14932 [00:40<40:48,  6.00it/s, v_num=hpnp, train/loss=4.470"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 245/14932 [00:40<40:43,  6.01it/s, v_num=hpnp, train/loss=4.470\r",
+      "Epoch 0:   2%| | 245/14932 [00:40<40:43,  6.01it/s, v_num=hpnp, train/loss=3.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 246/14932 [00:40<40:37,  6.02it/s, v_num=hpnp, train/loss=3.880\r",
+      "Epoch 0:   2%| | 246/14932 [00:40<40:38,  6.02it/s, v_num=hpnp, train/loss=3.620"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 247/14932 [00:40<40:32,  6.04it/s, v_num=hpnp, train/loss=3.620\r",
+      "Epoch 0:   2%| | 247/14932 [00:40<40:32,  6.04it/s, v_num=hpnp, train/loss=3.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 248/14932 [00:40<40:26,  6.05it/s, v_num=hpnp, train/loss=3.750\r",
+      "Epoch 0:   2%| | 248/14932 [00:40<40:27,  6.05it/s, v_num=hpnp, train/loss=2.440"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 249/14932 [00:41<40:21,  6.06it/s, v_num=hpnp, train/loss=2.440\r",
+      "Epoch 0:   2%| | 249/14932 [00:41<40:21,  6.06it/s, v_num=hpnp, train/loss=3.530"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 250/14932 [00:41<40:15,  6.08it/s, v_num=hpnp, train/loss=3.530\r",
+      "Epoch 0:   2%| | 250/14932 [00:41<40:15,  6.08it/s, v_num=hpnp, train/loss=4.060"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 251/14932 [00:41<40:10,  6.09it/s, v_num=hpnp, train/loss=4.060\r",
+      "Epoch 0:   2%| | 251/14932 [00:41<40:10,  6.09it/s, v_num=hpnp, train/loss=3.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 252/14932 [00:41<40:05,  6.10it/s, v_num=hpnp, train/loss=3.190\r",
+      "Epoch 0:   2%| | 252/14932 [00:41<40:05,  6.10it/s, v_num=hpnp, train/loss=3.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 253/14932 [00:41<39:59,  6.12it/s, v_num=hpnp, train/loss=3.610\r",
+      "Epoch 0:   2%| | 253/14932 [00:41<39:59,  6.12it/s, v_num=hpnp, train/loss=4.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 254/14932 [00:41<39:54,  6.13it/s, v_num=hpnp, train/loss=4.310\r",
+      "Epoch 0:   2%| | 254/14932 [00:41<39:54,  6.13it/s, v_num=hpnp, train/loss=3.140"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 255/14932 [00:41<39:49,  6.14it/s, v_num=hpnp, train/loss=3.140\r",
+      "Epoch 0:   2%| | 255/14932 [00:41<39:49,  6.14it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 256/14932 [00:41<39:51,  6.14it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   2%| | 256/14932 [00:41<39:51,  6.14it/s, v_num=hpnp, train/loss=3.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 257/14932 [00:41<39:45,  6.15it/s, v_num=hpnp, train/loss=3.300\r",
+      "Epoch 0:   2%| | 257/14932 [00:41<39:45,  6.15it/s, v_num=hpnp, train/loss=3.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 258/14932 [00:41<39:39,  6.17it/s, v_num=hpnp, train/loss=3.560\r",
+      "Epoch 0:   2%| | 258/14932 [00:41<39:39,  6.17it/s, v_num=hpnp, train/loss=3.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 259/14932 [00:41<39:34,  6.18it/s, v_num=hpnp, train/loss=3.730\r",
+      "Epoch 0:   2%| | 259/14932 [00:41<39:34,  6.18it/s, v_num=hpnp, train/loss=3.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 260/14932 [00:42<39:39,  6.17it/s, v_num=hpnp, train/loss=3.660\r",
+      "Epoch 0:   2%| | 260/14932 [00:42<39:39,  6.17it/s, v_num=hpnp, train/loss=4.090"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 261/14932 [00:42<39:34,  6.18it/s, v_num=hpnp, train/loss=4.090\r",
+      "Epoch 0:   2%| | 261/14932 [00:42<39:34,  6.18it/s, v_num=hpnp, train/loss=2.920"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 262/14932 [00:42<39:29,  6.19it/s, v_num=hpnp, train/loss=2.920\r",
+      "Epoch 0:   2%| | 262/14932 [00:42<39:29,  6.19it/s, v_num=hpnp, train/loss=3.770"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 263/14932 [00:42<39:24,  6.20it/s, v_num=hpnp, train/loss=3.770\r",
+      "Epoch 0:   2%| | 263/14932 [00:42<39:24,  6.20it/s, v_num=hpnp, train/loss=4.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 264/14932 [00:42<39:19,  6.22it/s, v_num=hpnp, train/loss=4.000\r",
+      "Epoch 0:   2%| | 264/14932 [00:42<39:19,  6.22it/s, v_num=hpnp, train/loss=3.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 265/14932 [00:42<39:14,  6.23it/s, v_num=hpnp, train/loss=3.750\r",
+      "Epoch 0:   2%| | 265/14932 [00:42<39:14,  6.23it/s, v_num=hpnp, train/loss=3.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 266/14932 [00:42<39:09,  6.24it/s, v_num=hpnp, train/loss=3.190\r",
+      "Epoch 0:   2%| | 266/14932 [00:42<39:09,  6.24it/s, v_num=hpnp, train/loss=3.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 267/14932 [00:42<39:04,  6.26it/s, v_num=hpnp, train/loss=3.810\r",
+      "Epoch 0:   2%| | 267/14932 [00:42<39:04,  6.26it/s, v_num=hpnp, train/loss=3.520"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 268/14932 [00:42<38:59,  6.27it/s, v_num=hpnp, train/loss=3.520\r",
+      "Epoch 0:   2%| | 268/14932 [00:42<38:59,  6.27it/s, v_num=hpnp, train/loss=2.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 269/14932 [00:42<38:55,  6.28it/s, v_num=hpnp, train/loss=2.950\r",
+      "Epoch 0:   2%| | 269/14932 [00:42<38:55,  6.28it/s, v_num=hpnp, train/loss=3.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 270/14932 [00:42<38:50,  6.29it/s, v_num=hpnp, train/loss=3.810\r",
+      "Epoch 0:   2%| | 270/14932 [00:42<38:50,  6.29it/s, v_num=hpnp, train/loss=4.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 271/14932 [00:42<38:45,  6.30it/s, v_num=hpnp, train/loss=4.220\r",
+      "Epoch 0:   2%| | 271/14932 [00:42<38:45,  6.30it/s, v_num=hpnp, train/loss=3.110"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 272/14932 [00:43<38:41,  6.32it/s, v_num=hpnp, train/loss=3.110\r",
+      "Epoch 0:   2%| | 272/14932 [00:43<38:41,  6.32it/s, v_num=hpnp, train/loss=1.740"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 273/14932 [00:43<38:36,  6.33it/s, v_num=hpnp, train/loss=1.740\r",
+      "Epoch 0:   2%| | 273/14932 [00:43<38:36,  6.33it/s, v_num=hpnp, train/loss=3.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 274/14932 [00:43<38:31,  6.34it/s, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   2%| | 274/14932 [00:43<38:31,  6.34it/s, v_num=hpnp, train/loss=3.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 275/14932 [00:43<38:27,  6.35it/s, v_num=hpnp, train/loss=3.830\r",
+      "Epoch 0:   2%| | 275/14932 [00:43<38:27,  6.35it/s, v_num=hpnp, train/loss=2.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 276/14932 [00:43<38:22,  6.36it/s, v_num=hpnp, train/loss=2.590\r",
+      "Epoch 0:   2%| | 276/14932 [00:43<38:22,  6.36it/s, v_num=hpnp, train/loss=4.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 277/14932 [00:43<38:18,  6.38it/s, v_num=hpnp, train/loss=4.220\r",
+      "Epoch 0:   2%| | 277/14932 [00:43<38:18,  6.38it/s, v_num=hpnp, train/loss=3.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 278/14932 [00:43<38:13,  6.39it/s, v_num=hpnp, train/loss=3.660\r",
+      "Epoch 0:   2%| | 278/14932 [00:43<38:13,  6.39it/s, v_num=hpnp, train/loss=2.690"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 279/14932 [00:43<38:09,  6.40it/s, v_num=hpnp, train/loss=2.690\r",
+      "Epoch 0:   2%| | 279/14932 [00:43<38:09,  6.40it/s, v_num=hpnp, train/loss=5.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 280/14932 [00:43<38:05,  6.41it/s, v_num=hpnp, train/loss=5.190\r",
+      "Epoch 0:   2%| | 280/14932 [00:43<38:05,  6.41it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 281/14932 [00:43<38:00,  6.42it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   2%| | 281/14932 [00:43<38:00,  6.42it/s, v_num=hpnp, train/loss=3.720"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 282/14932 [00:43<37:56,  6.44it/s, v_num=hpnp, train/loss=3.720\r",
+      "Epoch 0:   2%| | 282/14932 [00:43<37:56,  6.44it/s, v_num=hpnp, train/loss=2.910"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 283/14932 [00:43<37:51,  6.45it/s, v_num=hpnp, train/loss=2.910\r",
+      "Epoch 0:   2%| | 283/14932 [00:43<37:51,  6.45it/s, v_num=hpnp, train/loss=3.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 284/14932 [00:43<37:47,  6.46it/s, v_num=hpnp, train/loss=3.860\r",
+      "Epoch 0:   2%| | 284/14932 [00:43<37:47,  6.46it/s, v_num=hpnp, train/loss=3.470"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 285/14932 [00:44<37:43,  6.47it/s, v_num=hpnp, train/loss=3.470\r",
+      "Epoch 0:   2%| | 285/14932 [00:44<37:43,  6.47it/s, v_num=hpnp, train/loss=3.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 286/14932 [00:44<37:40,  6.48it/s, v_num=hpnp, train/loss=3.590\r",
+      "Epoch 0:   2%| | 286/14932 [00:44<37:40,  6.48it/s, v_num=hpnp, train/loss=4.060"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 287/14932 [00:44<37:35,  6.49it/s, v_num=hpnp, train/loss=4.060\r",
+      "Epoch 0:   2%| | 287/14932 [00:44<37:35,  6.49it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 288/14932 [00:44<37:38,  6.49it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   2%| | 288/14932 [00:44<37:38,  6.48it/s, v_num=hpnp, train/loss=3.910"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 289/14932 [00:44<37:33,  6.50it/s, v_num=hpnp, train/loss=3.910\r",
+      "Epoch 0:   2%| | 289/14932 [00:44<37:33,  6.50it/s, v_num=hpnp, train/loss=2.440"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 290/14932 [00:44<37:29,  6.51it/s, v_num=hpnp, train/loss=2.440\r",
+      "Epoch 0:   2%| | 290/14932 [00:44<37:29,  6.51it/s, v_num=hpnp, train/loss=3.230"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 291/14932 [00:44<37:25,  6.52it/s, v_num=hpnp, train/loss=3.230\r",
+      "Epoch 0:   2%| | 291/14932 [00:44<37:25,  6.52it/s, v_num=hpnp, train/loss=4.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 292/14932 [00:44<37:21,  6.53it/s, v_num=hpnp, train/loss=4.220\r",
+      "Epoch 0:   2%| | 292/14932 [00:44<37:21,  6.53it/s, v_num=hpnp, train/loss=2.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 293/14932 [00:44<37:17,  6.54it/s, v_num=hpnp, train/loss=2.190\r",
+      "Epoch 0:   2%| | 293/14932 [00:44<37:17,  6.54it/s, v_num=hpnp, train/loss=4.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 294/14932 [00:44<37:12,  6.56it/s, v_num=hpnp, train/loss=4.000\r",
+      "Epoch 0:   2%| | 294/14932 [00:44<37:12,  6.56it/s, v_num=hpnp, train/loss=3.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 295/14932 [00:44<37:08,  6.57it/s, v_num=hpnp, train/loss=3.000\r",
+      "Epoch 0:   2%| | 295/14932 [00:44<37:08,  6.57it/s, v_num=hpnp, train/loss=4.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 296/14932 [00:44<37:04,  6.58it/s, v_num=hpnp, train/loss=4.560\r",
+      "Epoch 0:   2%| | 296/14932 [00:44<37:05,  6.58it/s, v_num=hpnp, train/loss=3.360"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 297/14932 [00:45<37:00,  6.59it/s, v_num=hpnp, train/loss=3.360\r",
+      "Epoch 0:   2%| | 297/14932 [00:45<37:01,  6.59it/s, v_num=hpnp, train/loss=4.470"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 298/14932 [00:45<36:57,  6.60it/s, v_num=hpnp, train/loss=4.470\r",
+      "Epoch 0:   2%| | 298/14932 [00:45<36:57,  6.60it/s, v_num=hpnp, train/loss=3.700"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 299/14932 [00:45<36:52,  6.61it/s, v_num=hpnp, train/loss=3.700\r",
+      "Epoch 0:   2%| | 299/14932 [00:45<36:52,  6.61it/s, v_num=hpnp, train/loss=2.910"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 300/14932 [00:45<36:49,  6.62it/s, v_num=hpnp, train/loss=2.910"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 300/14932 [00:45<36:49,  6.62it/s, v_num=hpnp, train/loss=2.700"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 301/14932 [00:45<36:45,  6.63it/s, v_num=hpnp, train/loss=2.700\r",
+      "Epoch 0:   2%| | 301/14932 [00:45<36:45,  6.63it/s, v_num=hpnp, train/loss=3.920"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 302/14932 [00:45<36:41,  6.65it/s, v_num=hpnp, train/loss=3.920\r",
+      "Epoch 0:   2%| | 302/14932 [00:45<36:41,  6.65it/s, v_num=hpnp, train/loss=3.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 303/14932 [00:45<36:37,  6.66it/s, v_num=hpnp, train/loss=3.160\r",
+      "Epoch 0:   2%| | 303/14932 [00:45<36:37,  6.66it/s, v_num=hpnp, train/loss=4.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 304/14932 [00:45<36:34,  6.67it/s, v_num=hpnp, train/loss=4.220\r",
+      "Epoch 0:   2%| | 304/14932 [00:45<36:34,  6.67it/s, v_num=hpnp, train/loss=4.530"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 305/14932 [00:45<36:30,  6.68it/s, v_num=hpnp, train/loss=4.530\r",
+      "Epoch 0:   2%| | 305/14932 [00:45<36:31,  6.68it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 306/14932 [00:45<36:27,  6.69it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   2%| | 306/14932 [00:45<36:27,  6.69it/s, v_num=hpnp, train/loss=3.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 307/14932 [00:45<36:23,  6.70it/s, v_num=hpnp, train/loss=3.830\r",
+      "Epoch 0:   2%| | 307/14932 [00:45<36:23,  6.70it/s, v_num=hpnp, train/loss=3.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 308/14932 [00:45<36:19,  6.71it/s, v_num=hpnp, train/loss=3.300\r",
+      "Epoch 0:   2%| | 308/14932 [00:45<36:19,  6.71it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 309/14932 [00:45<36:15,  6.72it/s, v_num=hpnp, train/loss=3.780\r",
+      "Epoch 0:   2%| | 309/14932 [00:45<36:15,  6.72it/s, v_num=hpnp, train/loss=3.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 310/14932 [00:46<36:12,  6.73it/s, v_num=hpnp, train/loss=3.560\r",
+      "Epoch 0:   2%| | 310/14932 [00:46<36:12,  6.73it/s, v_num=hpnp, train/loss=3.980"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 311/14932 [00:46<36:08,  6.74it/s, v_num=hpnp, train/loss=3.980\r",
+      "Epoch 0:   2%| | 311/14932 [00:46<36:08,  6.74it/s, v_num=hpnp, train/loss=2.800"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 312/14932 [00:46<36:04,  6.75it/s, v_num=hpnp, train/loss=2.800\r",
+      "Epoch 0:   2%| | 312/14932 [00:46<36:04,  6.75it/s, v_num=hpnp, train/loss=2.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 313/14932 [00:46<36:01,  6.76it/s, v_num=hpnp, train/loss=2.780\r",
+      "Epoch 0:   2%| | 313/14932 [00:46<36:01,  6.76it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 314/14932 [00:46<35:57,  6.78it/s, v_num=hpnp, train/loss=3.780\r",
+      "Epoch 0:   2%| | 314/14932 [00:46<35:57,  6.78it/s, v_num=hpnp, train/loss=3.520"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 315/14932 [00:46<35:53,  6.79it/s, v_num=hpnp, train/loss=3.520\r",
+      "Epoch 0:   2%| | 315/14932 [00:46<35:53,  6.79it/s, v_num=hpnp, train/loss=2.390"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 316/14932 [00:46<35:49,  6.80it/s, v_num=hpnp, train/loss=2.390\r",
+      "Epoch 0:   2%| | 316/14932 [00:46<35:49,  6.80it/s, v_num=hpnp, train/loss=1.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 317/14932 [00:46<35:46,  6.81it/s, v_num=hpnp, train/loss=1.950\r",
+      "Epoch 0:   2%| | 317/14932 [00:46<35:46,  6.81it/s, v_num=hpnp, train/loss=3.050"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 318/14932 [00:46<35:43,  6.82it/s, v_num=hpnp, train/loss=3.050\r",
+      "Epoch 0:   2%| | 318/14932 [00:46<35:43,  6.82it/s, v_num=hpnp, train/loss=4.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 319/14932 [00:46<35:39,  6.83it/s, v_num=hpnp, train/loss=4.250\r",
+      "Epoch 0:   2%| | 319/14932 [00:46<35:39,  6.83it/s, v_num=hpnp, train/loss=3.480"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 320/14932 [00:46<35:41,  6.82it/s, v_num=hpnp, train/loss=3.480\r",
+      "Epoch 0:   2%| | 320/14932 [00:46<35:41,  6.82it/s, v_num=hpnp, train/loss=3.440"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 321/14932 [00:46<35:37,  6.83it/s, v_num=hpnp, train/loss=3.440\r",
+      "Epoch 0:   2%| | 321/14932 [00:46<35:37,  6.83it/s, v_num=hpnp, train/loss=3.640"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 322/14932 [00:47<35:34,  6.85it/s, v_num=hpnp, train/loss=3.640\r",
+      "Epoch 0:   2%| | 322/14932 [00:47<35:34,  6.85it/s, v_num=hpnp, train/loss=4.500"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 323/14932 [00:47<35:30,  6.86it/s, v_num=hpnp, train/loss=4.500\r",
+      "Epoch 0:   2%| | 323/14932 [00:47<35:30,  6.86it/s, v_num=hpnp, train/loss=3.890"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 324/14932 [00:47<35:28,  6.86it/s, v_num=hpnp, train/loss=3.890\r",
+      "Epoch 0:   2%| | 324/14932 [00:47<35:28,  6.86it/s, v_num=hpnp, train/loss=2.920"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 325/14932 [00:47<35:26,  6.87it/s, v_num=hpnp, train/loss=2.920\r",
+      "Epoch 0:   2%| | 325/14932 [00:47<35:26,  6.87it/s, v_num=hpnp, train/loss=3.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 326/14932 [00:47<35:22,  6.88it/s, v_num=hpnp, train/loss=3.030\r",
+      "Epoch 0:   2%| | 326/14932 [00:47<35:22,  6.88it/s, v_num=hpnp, train/loss=4.090"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 327/14932 [00:47<35:19,  6.89it/s, v_num=hpnp, train/loss=4.090\r",
+      "Epoch 0:   2%| | 327/14932 [00:47<35:19,  6.89it/s, v_num=hpnp, train/loss=4.410"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 328/14932 [00:47<35:16,  6.90it/s, v_num=hpnp, train/loss=4.410\r",
+      "Epoch 0:   2%| | 328/14932 [00:47<35:16,  6.90it/s, v_num=hpnp, train/loss=3.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 329/14932 [00:47<35:12,  6.91it/s, v_num=hpnp, train/loss=3.950\r",
+      "Epoch 0:   2%| | 329/14932 [00:47<35:12,  6.91it/s, v_num=hpnp, train/loss=3.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 330/14932 [00:47<35:09,  6.92it/s, v_num=hpnp, train/loss=3.950\r",
+      "Epoch 0:   2%| | 330/14932 [00:47<35:09,  6.92it/s, v_num=hpnp, train/loss=3.980"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 331/14932 [00:47<35:06,  6.93it/s, v_num=hpnp, train/loss=3.980\r",
+      "Epoch 0:   2%| | 331/14932 [00:47<35:06,  6.93it/s, v_num=hpnp, train/loss=4.280"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 332/14932 [00:47<35:02,  6.94it/s, v_num=hpnp, train/loss=4.280\r",
+      "Epoch 0:   2%| | 332/14932 [00:47<35:02,  6.94it/s, v_num=hpnp, train/loss=2.410"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 333/14932 [00:47<34:59,  6.95it/s, v_num=hpnp, train/loss=2.410\r",
+      "Epoch 0:   2%| | 333/14932 [00:47<34:59,  6.95it/s, v_num=hpnp, train/loss=3.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 334/14932 [00:47<34:56,  6.96it/s, v_num=hpnp, train/loss=3.950\r",
+      "Epoch 0:   2%| | 334/14932 [00:47<34:56,  6.96it/s, v_num=hpnp, train/loss=3.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 335/14932 [00:48<34:53,  6.97it/s, v_num=hpnp, train/loss=3.610\r",
+      "Epoch 0:   2%| | 335/14932 [00:48<34:53,  6.97it/s, v_num=hpnp, train/loss=3.500"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 336/14932 [00:48<34:53,  6.97it/s, v_num=hpnp, train/loss=3.500\r",
+      "Epoch 0:   2%| | 336/14932 [00:48<34:53,  6.97it/s, v_num=hpnp, train/loss=3.800"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 337/14932 [00:48<34:49,  6.98it/s, v_num=hpnp, train/loss=3.800\r",
+      "Epoch 0:   2%| | 337/14932 [00:48<34:49,  6.98it/s, v_num=hpnp, train/loss=3.090"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 338/14932 [00:48<34:46,  6.99it/s, v_num=hpnp, train/loss=3.090\r",
+      "Epoch 0:   2%| | 338/14932 [00:48<34:46,  6.99it/s, v_num=hpnp, train/loss=3.020"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 339/14932 [00:48<34:43,  7.00it/s, v_num=hpnp, train/loss=3.020\r",
+      "Epoch 0:   2%| | 339/14932 [00:48<34:43,  7.00it/s, v_num=hpnp, train/loss=3.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 340/14932 [00:48<34:40,  7.01it/s, v_num=hpnp, train/loss=3.860\r",
+      "Epoch 0:   2%| | 340/14932 [00:48<34:40,  7.01it/s, v_num=hpnp, train/loss=4.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 341/14932 [00:48<34:37,  7.02it/s, v_num=hpnp, train/loss=4.030\r",
+      "Epoch 0:   2%| | 341/14932 [00:48<34:37,  7.02it/s, v_num=hpnp, train/loss=2.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 342/14932 [00:48<34:34,  7.03it/s, v_num=hpnp, train/loss=2.160\r",
+      "Epoch 0:   2%| | 342/14932 [00:48<34:34,  7.03it/s, v_num=hpnp, train/loss=3.420"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 343/14932 [00:48<34:34,  7.03it/s, v_num=hpnp, train/loss=3.420\r",
+      "Epoch 0:   2%| | 343/14932 [00:48<34:34,  7.03it/s, v_num=hpnp, train/loss=4.720"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 344/14932 [00:48<34:31,  7.04it/s, v_num=hpnp, train/loss=4.720\r",
+      "Epoch 0:   2%| | 344/14932 [00:48<34:31,  7.04it/s, v_num=hpnp, train/loss=2.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 345/14932 [00:48<34:28,  7.05it/s, v_num=hpnp, train/loss=2.880\r",
+      "Epoch 0:   2%| | 345/14932 [00:48<34:28,  7.05it/s, v_num=hpnp, train/loss=2.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 346/14932 [00:48<34:25,  7.06it/s, v_num=hpnp, train/loss=2.560\r",
+      "Epoch 0:   2%| | 346/14932 [00:48<34:25,  7.06it/s, v_num=hpnp, train/loss=4.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 347/14932 [00:49<34:22,  7.07it/s, v_num=hpnp, train/loss=4.560\r",
+      "Epoch 0:   2%| | 347/14932 [00:49<34:22,  7.07it/s, v_num=hpnp, train/loss=3.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 348/14932 [00:49<34:19,  7.08it/s, v_num=hpnp, train/loss=3.160\r",
+      "Epoch 0:   2%| | 348/14932 [00:49<34:19,  7.08it/s, v_num=hpnp, train/loss=4.720"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 349/14932 [00:49<34:16,  7.09it/s, v_num=hpnp, train/loss=4.720\r",
+      "Epoch 0:   2%| | 349/14932 [00:49<34:16,  7.09it/s, v_num=hpnp, train/loss=4.060"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 350/14932 [00:49<34:13,  7.10it/s, v_num=hpnp, train/loss=4.060\r",
+      "Epoch 0:   2%| | 350/14932 [00:49<34:13,  7.10it/s, v_num=hpnp, train/loss=2.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 351/14932 [00:49<34:10,  7.11it/s, v_num=hpnp, train/loss=2.940\r",
+      "Epoch 0:   2%| | 351/14932 [00:49<34:10,  7.11it/s, v_num=hpnp, train/loss=3.480"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 352/14932 [00:49<34:15,  7.09it/s, v_num=hpnp, train/loss=3.480\r",
+      "Epoch 0:   2%| | 352/14932 [00:49<34:15,  7.09it/s, v_num=hpnp, train/loss=3.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 353/14932 [00:49<34:12,  7.10it/s, v_num=hpnp, train/loss=3.840\r",
+      "Epoch 0:   2%| | 353/14932 [00:49<34:12,  7.10it/s, v_num=hpnp, train/loss=2.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 354/14932 [00:49<34:10,  7.11it/s, v_num=hpnp, train/loss=2.950\r",
+      "Epoch 0:   2%| | 354/14932 [00:49<34:10,  7.11it/s, v_num=hpnp, train/loss=4.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 355/14932 [00:49<34:07,  7.12it/s, v_num=hpnp, train/loss=4.220\r",
+      "Epoch 0:   2%| | 355/14932 [00:49<34:07,  7.12it/s, v_num=hpnp, train/loss=3.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 356/14932 [00:49<34:05,  7.12it/s, v_num=hpnp, train/loss=3.340\r",
+      "Epoch 0:   2%| | 356/14932 [00:49<34:05,  7.12it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 357/14932 [00:50<34:03,  7.13it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   2%| | 357/14932 [00:50<34:03,  7.13it/s, v_num=hpnp, train/loss=3.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 358/14932 [00:50<34:01,  7.14it/s, v_num=hpnp, train/loss=3.190\r",
+      "Epoch 0:   2%| | 358/14932 [00:50<34:01,  7.14it/s, v_num=hpnp, train/loss=3.980"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 359/14932 [00:50<33:58,  7.15it/s, v_num=hpnp, train/loss=3.980\r",
+      "Epoch 0:   2%| | 359/14932 [00:50<33:58,  7.15it/s, v_num=hpnp, train/loss=4.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 360/14932 [00:50<33:56,  7.16it/s, v_num=hpnp, train/loss=4.000\r",
+      "Epoch 0:   2%| | 360/14932 [00:50<33:56,  7.16it/s, v_num=hpnp, train/loss=2.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 361/14932 [00:50<33:55,  7.16it/s, v_num=hpnp, train/loss=2.810\r",
+      "Epoch 0:   2%| | 361/14932 [00:50<33:56,  7.16it/s, v_num=hpnp, train/loss=3.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 362/14932 [00:50<33:52,  7.17it/s, v_num=hpnp, train/loss=3.880\r",
+      "Epoch 0:   2%| | 362/14932 [00:50<33:52,  7.17it/s, v_num=hpnp, train/loss=3.530"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 363/14932 [00:50<33:50,  7.18it/s, v_num=hpnp, train/loss=3.530\r",
+      "Epoch 0:   2%| | 363/14932 [00:50<33:50,  7.18it/s, v_num=hpnp, train/loss=3.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 364/14932 [00:50<33:47,  7.19it/s, v_num=hpnp, train/loss=3.300\r",
+      "Epoch 0:   2%| | 364/14932 [00:50<33:47,  7.18it/s, v_num=hpnp, train/loss=2.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 365/14932 [00:50<33:44,  7.20it/s, v_num=hpnp, train/loss=2.840\r",
+      "Epoch 0:   2%| | 365/14932 [00:50<33:44,  7.19it/s, v_num=hpnp, train/loss=3.120"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 366/14932 [00:50<33:41,  7.21it/s, v_num=hpnp, train/loss=3.120\r",
+      "Epoch 0:   2%| | 366/14932 [00:50<33:41,  7.21it/s, v_num=hpnp, train/loss=2.770"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 367/14932 [00:50<33:38,  7.21it/s, v_num=hpnp, train/loss=2.770"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 367/14932 [00:50<33:38,  7.21it/s, v_num=hpnp, train/loss=3.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 368/14932 [00:50<33:36,  7.22it/s, v_num=hpnp, train/loss=3.160\r",
+      "Epoch 0:   2%| | 368/14932 [00:50<33:36,  7.22it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 369/14932 [00:51<33:33,  7.23it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   2%| | 369/14932 [00:51<33:33,  7.23it/s, v_num=hpnp, train/loss=3.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 370/14932 [00:51<33:30,  7.24it/s, v_num=hpnp, train/loss=3.220\r",
+      "Epoch 0:   2%| | 370/14932 [00:51<33:30,  7.24it/s, v_num=hpnp, train/loss=3.520"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 371/14932 [00:51<33:27,  7.25it/s, v_num=hpnp, train/loss=3.520\r",
+      "Epoch 0:   2%| | 371/14932 [00:51<33:27,  7.25it/s, v_num=hpnp, train/loss=3.170"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 372/14932 [00:51<33:25,  7.26it/s, v_num=hpnp, train/loss=3.170\r",
+      "Epoch 0:   2%| | 372/14932 [00:51<33:25,  7.26it/s, v_num=hpnp, train/loss=2.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   2%| | 373/14932 [00:51<33:22,  7.27it/s, v_num=hpnp, train/loss=2.780\r",
+      "Epoch 0:   2%| | 373/14932 [00:51<33:22,  7.27it/s, v_num=hpnp, train/loss=3.270"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 374/14932 [00:51<33:20,  7.28it/s, v_num=hpnp, train/loss=3.270\r",
+      "Epoch 0:   3%| | 374/14932 [00:51<33:20,  7.28it/s, v_num=hpnp, train/loss=3.110"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 375/14932 [00:51<33:17,  7.29it/s, v_num=hpnp, train/loss=3.110\r",
+      "Epoch 0:   3%| | 375/14932 [00:51<33:17,  7.29it/s, v_num=hpnp, train/loss=3.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 376/14932 [00:51<33:15,  7.30it/s, v_num=hpnp, train/loss=3.300\r",
+      "Epoch 0:   3%| | 376/14932 [00:51<33:15,  7.30it/s, v_num=hpnp, train/loss=3.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 377/14932 [00:51<33:12,  7.30it/s, v_num=hpnp, train/loss=3.950\r",
+      "Epoch 0:   3%| | 377/14932 [00:51<33:12,  7.30it/s, v_num=hpnp, train/loss=3.890"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 378/14932 [00:51<33:10,  7.31it/s, v_num=hpnp, train/loss=3.890\r",
+      "Epoch 0:   3%| | 378/14932 [00:51<33:10,  7.31it/s, v_num=hpnp, train/loss=3.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 379/14932 [00:51<33:07,  7.32it/s, v_num=hpnp, train/loss=3.730\r",
+      "Epoch 0:   3%| | 379/14932 [00:51<33:07,  7.32it/s, v_num=hpnp, train/loss=3.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 380/14932 [00:51<33:04,  7.33it/s, v_num=hpnp, train/loss=3.560\r",
+      "Epoch 0:   3%| | 380/14932 [00:51<33:05,  7.33it/s, v_num=hpnp, train/loss=2.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 381/14932 [00:51<33:02,  7.34it/s, v_num=hpnp, train/loss=2.970\r",
+      "Epoch 0:   3%| | 381/14932 [00:51<33:02,  7.34it/s, v_num=hpnp, train/loss=3.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 382/14932 [00:51<33:00,  7.35it/s, v_num=hpnp, train/loss=3.160\r",
+      "Epoch 0:   3%| | 382/14932 [00:51<33:00,  7.35it/s, v_num=hpnp, train/loss=2.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 383/14932 [00:52<32:57,  7.36it/s, v_num=hpnp, train/loss=2.660\r",
+      "Epoch 0:   3%| | 383/14932 [00:52<32:57,  7.36it/s, v_num=hpnp, train/loss=2.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 384/14932 [00:52<33:00,  7.35it/s, v_num=hpnp, train/loss=2.560\r",
+      "Epoch 0:   3%| | 384/14932 [00:52<33:00,  7.35it/s, v_num=hpnp, train/loss=3.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 385/14932 [00:52<32:57,  7.36it/s, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   3%| | 385/14932 [00:52<32:57,  7.36it/s, v_num=hpnp, train/loss=3.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 386/14932 [00:52<32:55,  7.36it/s, v_num=hpnp, train/loss=3.840\r",
+      "Epoch 0:   3%| | 386/14932 [00:52<32:55,  7.36it/s, v_num=hpnp, train/loss=4.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 387/14932 [00:52<32:52,  7.37it/s, v_num=hpnp, train/loss=4.250\r",
+      "Epoch 0:   3%| | 387/14932 [00:52<32:52,  7.37it/s, v_num=hpnp, train/loss=4.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 388/14932 [00:52<32:50,  7.38it/s, v_num=hpnp, train/loss=4.000\r",
+      "Epoch 0:   3%| | 388/14932 [00:52<32:50,  7.38it/s, v_num=hpnp, train/loss=1.420"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 389/14932 [00:52<32:47,  7.39it/s, v_num=hpnp, train/loss=1.420\r",
+      "Epoch 0:   3%| | 389/14932 [00:52<32:47,  7.39it/s, v_num=hpnp, train/loss=3.980"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 390/14932 [00:52<32:45,  7.40it/s, v_num=hpnp, train/loss=3.980\r",
+      "Epoch 0:   3%| | 390/14932 [00:52<32:45,  7.40it/s, v_num=hpnp, train/loss=3.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 391/14932 [00:52<32:43,  7.41it/s, v_num=hpnp, train/loss=3.590\r",
+      "Epoch 0:   3%| | 391/14932 [00:52<32:43,  7.41it/s, v_num=hpnp, train/loss=3.060"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 392/14932 [00:52<32:41,  7.41it/s, v_num=hpnp, train/loss=3.060\r",
+      "Epoch 0:   3%| | 392/14932 [00:52<32:41,  7.41it/s, v_num=hpnp, train/loss=3.640"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 393/14932 [00:52<32:39,  7.42it/s, v_num=hpnp, train/loss=3.640\r",
+      "Epoch 0:   3%| | 393/14932 [00:52<32:39,  7.42it/s, v_num=hpnp, train/loss=3.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 394/14932 [00:53<32:36,  7.43it/s, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   3%| | 394/14932 [00:53<32:36,  7.43it/s, v_num=hpnp, train/loss=1.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 395/14932 [00:53<32:34,  7.44it/s, v_num=hpnp, train/loss=1.750\r",
+      "Epoch 0:   3%| | 395/14932 [00:53<32:34,  7.44it/s, v_num=hpnp, train/loss=3.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 396/14932 [00:53<32:31,  7.45it/s, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   3%| | 396/14932 [00:53<32:31,  7.45it/s, v_num=hpnp, train/loss=2.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 397/14932 [00:53<32:29,  7.46it/s, v_num=hpnp, train/loss=2.970\r",
+      "Epoch 0:   3%| | 397/14932 [00:53<32:29,  7.46it/s, v_num=hpnp, train/loss=4.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 398/14932 [00:53<32:26,  7.47it/s, v_num=hpnp, train/loss=4.190\r",
+      "Epoch 0:   3%| | 398/14932 [00:53<32:26,  7.47it/s, v_num=hpnp, train/loss=4.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 399/14932 [00:53<32:24,  7.48it/s, v_num=hpnp, train/loss=4.250\r",
+      "Epoch 0:   3%| | 399/14932 [00:53<32:24,  7.48it/s, v_num=hpnp, train/loss=2.480"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 400/14932 [00:53<32:21,  7.48it/s, v_num=hpnp, train/loss=2.480\r",
+      "Epoch 0:   3%| | 400/14932 [00:53<32:21,  7.48it/s, v_num=hpnp, train/loss=3.720"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 401/14932 [00:53<32:22,  7.48it/s, v_num=hpnp, train/loss=3.720\r",
+      "Epoch 0:   3%| | 401/14932 [00:53<32:22,  7.48it/s, v_num=hpnp, train/loss=4.910"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 402/14932 [00:53<32:19,  7.49it/s, v_num=hpnp, train/loss=4.910\r",
+      "Epoch 0:   3%| | 402/14932 [00:53<32:19,  7.49it/s, v_num=hpnp, train/loss=1.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 403/14932 [00:53<32:17,  7.50it/s, v_num=hpnp, train/loss=1.660\r",
+      "Epoch 0:   3%| | 403/14932 [00:53<32:17,  7.50it/s, v_num=hpnp, train/loss=4.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 404/14932 [00:53<32:15,  7.51it/s, v_num=hpnp, train/loss=4.250\r",
+      "Epoch 0:   3%| | 404/14932 [00:53<32:15,  7.51it/s, v_num=hpnp, train/loss=3.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 405/14932 [00:53<32:13,  7.51it/s, v_num=hpnp, train/loss=3.660\r",
+      "Epoch 0:   3%| | 405/14932 [00:53<32:13,  7.51it/s, v_num=hpnp, train/loss=3.330"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 406/14932 [00:53<32:11,  7.52it/s, v_num=hpnp, train/loss=3.330\r",
+      "Epoch 0:   3%| | 406/14932 [00:53<32:11,  7.52it/s, v_num=hpnp, train/loss=3.140"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 407/14932 [00:54<32:09,  7.53it/s, v_num=hpnp, train/loss=3.140\r",
+      "Epoch 0:   3%| | 407/14932 [00:54<32:09,  7.53it/s, v_num=hpnp, train/loss=3.640"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 408/14932 [00:54<32:07,  7.54it/s, v_num=hpnp, train/loss=3.640\r",
+      "Epoch 0:   3%| | 408/14932 [00:54<32:07,  7.54it/s, v_num=hpnp, train/loss=3.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 409/14932 [00:54<32:04,  7.54it/s, v_num=hpnp, train/loss=3.310\r",
+      "Epoch 0:   3%| | 409/14932 [00:54<32:05,  7.54it/s, v_num=hpnp, train/loss=1.480"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 410/14932 [00:54<32:02,  7.55it/s, v_num=hpnp, train/loss=1.480\r",
+      "Epoch 0:   3%| | 410/14932 [00:54<32:02,  7.55it/s, v_num=hpnp, train/loss=3.140"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 411/14932 [00:54<32:00,  7.56it/s, v_num=hpnp, train/loss=3.140\r",
+      "Epoch 0:   3%| | 411/14932 [00:54<32:00,  7.56it/s, v_num=hpnp, train/loss=3.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 412/14932 [00:54<31:58,  7.57it/s, v_num=hpnp, train/loss=3.660\r",
+      "Epoch 0:   3%| | 412/14932 [00:54<31:58,  7.57it/s, v_num=hpnp, train/loss=3.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 413/14932 [00:54<31:56,  7.58it/s, v_num=hpnp, train/loss=3.590\r",
+      "Epoch 0:   3%| | 413/14932 [00:54<31:56,  7.58it/s, v_num=hpnp, train/loss=4.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 414/14932 [00:54<31:54,  7.58it/s, v_num=hpnp, train/loss=4.250\r",
+      "Epoch 0:   3%| | 414/14932 [00:54<31:54,  7.58it/s, v_num=hpnp, train/loss=3.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 415/14932 [00:54<31:54,  7.58it/s, v_num=hpnp, train/loss=3.860\r",
+      "Epoch 0:   3%| | 415/14932 [00:54<31:54,  7.58it/s, v_num=hpnp, train/loss=4.280"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 416/14932 [00:54<31:56,  7.57it/s, v_num=hpnp, train/loss=4.280\r",
+      "Epoch 0:   3%| | 416/14932 [00:54<31:56,  7.57it/s, v_num=hpnp, train/loss=5.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 417/14932 [00:54<31:54,  7.58it/s, v_num=hpnp, train/loss=5.160\r",
+      "Epoch 0:   3%| | 417/14932 [00:54<31:54,  7.58it/s, v_num=hpnp, train/loss=3.700"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 418/14932 [00:55<31:52,  7.59it/s, v_num=hpnp, train/loss=3.700\r",
+      "Epoch 0:   3%| | 418/14932 [00:55<31:52,  7.59it/s, v_num=hpnp, train/loss=3.110"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 419/14932 [00:55<31:50,  7.60it/s, v_num=hpnp, train/loss=3.110\r",
+      "Epoch 0:   3%| | 419/14932 [00:55<31:50,  7.60it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 420/14932 [00:55<31:47,  7.61it/s, v_num=hpnp, train/loss=3.780\r",
+      "Epoch 0:   3%| | 420/14932 [00:55<31:47,  7.61it/s, v_num=hpnp, train/loss=2.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 421/14932 [00:55<31:45,  7.61it/s, v_num=hpnp, train/loss=2.660\r",
+      "Epoch 0:   3%| | 421/14932 [00:55<31:45,  7.61it/s, v_num=hpnp, train/loss=2.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 422/14932 [00:55<31:44,  7.62it/s, v_num=hpnp, train/loss=2.830\r",
+      "Epoch 0:   3%| | 422/14932 [00:55<31:44,  7.62it/s, v_num=hpnp, train/loss=2.700"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 423/14932 [00:55<31:42,  7.63it/s, v_num=hpnp, train/loss=2.700\r",
+      "Epoch 0:   3%| | 423/14932 [00:55<31:42,  7.63it/s, v_num=hpnp, train/loss=3.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 424/14932 [00:55<31:40,  7.63it/s, v_num=hpnp, train/loss=3.860\r",
+      "Epoch 0:   3%| | 424/14932 [00:55<31:40,  7.63it/s, v_num=hpnp, train/loss=3.270"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 425/14932 [00:55<31:38,  7.64it/s, v_num=hpnp, train/loss=3.270\r",
+      "Epoch 0:   3%| | 425/14932 [00:55<31:38,  7.64it/s, v_num=hpnp, train/loss=4.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 426/14932 [00:55<31:36,  7.65it/s, v_num=hpnp, train/loss=4.970\r",
+      "Epoch 0:   3%| | 426/14932 [00:55<31:36,  7.65it/s, v_num=hpnp, train/loss=2.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 427/14932 [00:55<31:34,  7.66it/s, v_num=hpnp, train/loss=2.300\r",
+      "Epoch 0:   3%| | 427/14932 [00:55<31:34,  7.66it/s, v_num=hpnp, train/loss=3.110"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 428/14932 [00:55<31:32,  7.66it/s, v_num=hpnp, train/loss=3.110\r",
+      "Epoch 0:   3%| | 428/14932 [00:55<31:32,  7.66it/s, v_num=hpnp, train/loss=4.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 429/14932 [00:55<31:30,  7.67it/s, v_num=hpnp, train/loss=4.310\r",
+      "Epoch 0:   3%| | 429/14932 [00:55<31:30,  7.67it/s, v_num=hpnp, train/loss=3.410"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 430/14932 [00:56<31:31,  7.67it/s, v_num=hpnp, train/loss=3.410\r",
+      "Epoch 0:   3%| | 430/14932 [00:56<31:31,  7.67it/s, v_num=hpnp, train/loss=3.670"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 431/14932 [00:56<31:30,  7.67it/s, v_num=hpnp, train/loss=3.670\r",
+      "Epoch 0:   3%| | 431/14932 [00:56<31:30,  7.67it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 432/14932 [00:56<31:28,  7.68it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   3%| | 432/14932 [00:56<31:28,  7.68it/s, v_num=hpnp, train/loss=2.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 433/14932 [00:56<31:26,  7.69it/s, v_num=hpnp, train/loss=2.590\r",
+      "Epoch 0:   3%| | 433/14932 [00:56<31:26,  7.69it/s, v_num=hpnp, train/loss=4.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 434/14932 [00:56<31:24,  7.69it/s, v_num=hpnp, train/loss=4.000\r",
+      "Epoch 0:   3%| | 434/14932 [00:56<31:24,  7.69it/s, v_num=hpnp, train/loss=3.700"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 435/14932 [00:56<31:22,  7.70it/s, v_num=hpnp, train/loss=3.700\r",
+      "Epoch 0:   3%| | 435/14932 [00:56<31:22,  7.70it/s, v_num=hpnp, train/loss=4.500"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 436/14932 [00:56<31:20,  7.71it/s, v_num=hpnp, train/loss=4.500\r",
+      "Epoch 0:   3%| | 436/14932 [00:56<31:20,  7.71it/s, v_num=hpnp, train/loss=2.170"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 437/14932 [00:56<31:18,  7.71it/s, v_num=hpnp, train/loss=2.170\r",
+      "Epoch 0:   3%| | 437/14932 [00:56<31:18,  7.71it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 438/14932 [00:56<31:16,  7.72it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   3%| | 438/14932 [00:56<31:16,  7.72it/s, v_num=hpnp, train/loss=3.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 439/14932 [00:56<31:14,  7.73it/s, v_num=hpnp, train/loss=3.830\r",
+      "Epoch 0:   3%| | 439/14932 [00:56<31:14,  7.73it/s, v_num=hpnp, train/loss=3.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 440/14932 [00:56<31:12,  7.74it/s, v_num=hpnp, train/loss=3.590\r",
+      "Epoch 0:   3%| | 440/14932 [00:56<31:12,  7.74it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 441/14932 [00:57<31:13,  7.73it/s, v_num=hpnp, train/loss=3.780\r",
+      "Epoch 0:   3%| | 441/14932 [00:57<31:13,  7.73it/s, v_num=hpnp, train/loss=3.450"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 442/14932 [00:57<31:11,  7.74it/s, v_num=hpnp, train/loss=3.450\r",
+      "Epoch 0:   3%| | 442/14932 [00:57<31:11,  7.74it/s, v_num=hpnp, train/loss=3.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 443/14932 [00:57<31:10,  7.75it/s, v_num=hpnp, train/loss=3.190\r",
+      "Epoch 0:   3%| | 443/14932 [00:57<31:10,  7.75it/s, v_num=hpnp, train/loss=3.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 444/14932 [00:57<31:08,  7.76it/s, v_num=hpnp, train/loss=3.220\r",
+      "Epoch 0:   3%| | 444/14932 [00:57<31:08,  7.76it/s, v_num=hpnp, train/loss=3.410"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 445/14932 [00:57<31:06,  7.76it/s, v_num=hpnp, train/loss=3.410\r",
+      "Epoch 0:   3%| | 445/14932 [00:57<31:06,  7.76it/s, v_num=hpnp, train/loss=3.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 446/14932 [00:57<31:04,  7.77it/s, v_num=hpnp, train/loss=3.840\r",
+      "Epoch 0:   3%| | 446/14932 [00:57<31:04,  7.77it/s, v_num=hpnp, train/loss=2.390"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 447/14932 [00:57<31:02,  7.78it/s, v_num=hpnp, train/loss=2.390\r",
+      "Epoch 0:   3%| | 447/14932 [00:57<31:02,  7.78it/s, v_num=hpnp, train/loss=1.670"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 448/14932 [00:57<31:04,  7.77it/s, v_num=hpnp, train/loss=1.670\r",
+      "Epoch 0:   3%| | 448/14932 [00:57<31:04,  7.77it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 449/14932 [00:57<31:02,  7.78it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   3%| | 449/14932 [00:57<31:02,  7.78it/s, v_num=hpnp, train/loss=4.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 450/14932 [00:57<31:00,  7.78it/s, v_num=hpnp, train/loss=4.030\r",
+      "Epoch 0:   3%| | 450/14932 [00:57<31:00,  7.78it/s, v_num=hpnp, train/loss=4.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 451/14932 [00:57<30:58,  7.79it/s, v_num=hpnp, train/loss=4.310\r",
+      "Epoch 0:   3%| | 451/14932 [00:57<30:58,  7.79it/s, v_num=hpnp, train/loss=2.700"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 452/14932 [00:57<30:56,  7.80it/s, v_num=hpnp, train/loss=2.700\r",
+      "Epoch 0:   3%| | 452/14932 [00:57<30:56,  7.80it/s, v_num=hpnp, train/loss=3.170"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 453/14932 [00:58<30:54,  7.81it/s, v_num=hpnp, train/loss=3.170\r",
+      "Epoch 0:   3%| | 453/14932 [00:58<30:54,  7.81it/s, v_num=hpnp, train/loss=2.860"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 454/14932 [00:58<30:52,  7.81it/s, v_num=hpnp, train/loss=2.860\r",
+      "Epoch 0:   3%| | 454/14932 [00:58<30:52,  7.81it/s, v_num=hpnp, train/loss=3.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 455/14932 [00:58<30:51,  7.82it/s, v_num=hpnp, train/loss=3.880\r",
+      "Epoch 0:   3%| | 455/14932 [00:58<30:51,  7.82it/s, v_num=hpnp, train/loss=3.920"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 456/14932 [00:58<30:49,  7.83it/s, v_num=hpnp, train/loss=3.920\r",
+      "Epoch 0:   3%| | 456/14932 [00:58<30:49,  7.83it/s, v_num=hpnp, train/loss=3.330"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 457/14932 [00:58<30:50,  7.82it/s, v_num=hpnp, train/loss=3.330\r",
+      "Epoch 0:   3%| | 457/14932 [00:58<30:50,  7.82it/s, v_num=hpnp, train/loss=3.890"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 458/14932 [00:58<30:49,  7.83it/s, v_num=hpnp, train/loss=3.890\r",
+      "Epoch 0:   3%| | 458/14932 [00:58<30:49,  7.83it/s, v_num=hpnp, train/loss=3.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 459/14932 [00:58<30:47,  7.83it/s, v_num=hpnp, train/loss=3.940\r",
+      "Epoch 0:   3%| | 459/14932 [00:58<30:47,  7.83it/s, v_num=hpnp, train/loss=3.440"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 460/14932 [00:58<30:48,  7.83it/s, v_num=hpnp, train/loss=3.440\r",
+      "Epoch 0:   3%| | 460/14932 [00:58<30:48,  7.83it/s, v_num=hpnp, train/loss=2.310"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 461/14932 [00:58<30:46,  7.84it/s, v_num=hpnp, train/loss=2.310\r",
+      "Epoch 0:   3%| | 461/14932 [00:58<30:46,  7.84it/s, v_num=hpnp, train/loss=1.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 462/14932 [00:58<30:44,  7.85it/s, v_num=hpnp, train/loss=1.250\r",
+      "Epoch 0:   3%| | 462/14932 [00:58<30:44,  7.85it/s, v_num=hpnp, train/loss=2.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 463/14932 [00:58<30:42,  7.85it/s, v_num=hpnp, train/loss=2.840\r",
+      "Epoch 0:   3%| | 463/14932 [00:58<30:42,  7.85it/s, v_num=hpnp, train/loss=3.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 464/14932 [00:59<30:40,  7.86it/s, v_num=hpnp, train/loss=3.730\r",
+      "Epoch 0:   3%| | 464/14932 [00:59<30:40,  7.86it/s, v_num=hpnp, train/loss=3.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 465/14932 [00:59<30:39,  7.86it/s, v_num=hpnp, train/loss=3.730\r",
+      "Epoch 0:   3%| | 465/14932 [00:59<30:39,  7.86it/s, v_num=hpnp, train/loss=4.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 466/14932 [00:59<30:37,  7.87it/s, v_num=hpnp, train/loss=4.560\r",
+      "Epoch 0:   3%| | 466/14932 [00:59<30:37,  7.87it/s, v_num=hpnp, train/loss=3.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 467/14932 [00:59<30:36,  7.88it/s, v_num=hpnp, train/loss=3.610\r",
+      "Epoch 0:   3%| | 467/14932 [00:59<30:36,  7.88it/s, v_num=hpnp, train/loss=2.360"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 468/14932 [00:59<30:34,  7.88it/s, v_num=hpnp, train/loss=2.360\r",
+      "Epoch 0:   3%| | 468/14932 [00:59<30:34,  7.88it/s, v_num=hpnp, train/loss=1.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 469/14932 [00:59<30:32,  7.89it/s, v_num=hpnp, train/loss=1.970\r",
+      "Epoch 0:   3%| | 469/14932 [00:59<30:32,  7.89it/s, v_num=hpnp, train/loss=1.650"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 470/14932 [00:59<30:31,  7.90it/s, v_num=hpnp, train/loss=1.650\r",
+      "Epoch 0:   3%| | 470/14932 [00:59<30:31,  7.90it/s, v_num=hpnp, train/loss=3.480"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 471/14932 [00:59<30:29,  7.90it/s, v_num=hpnp, train/loss=3.480\r",
+      "Epoch 0:   3%| | 471/14932 [00:59<30:29,  7.90it/s, v_num=hpnp, train/loss=2.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 472/14932 [00:59<30:27,  7.91it/s, v_num=hpnp, train/loss=2.660\r",
+      "Epoch 0:   3%| | 472/14932 [00:59<30:27,  7.91it/s, v_num=hpnp, train/loss=3.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 473/14932 [00:59<30:26,  7.92it/s, v_num=hpnp, train/loss=3.590\r",
+      "Epoch 0:   3%| | 473/14932 [00:59<30:26,  7.92it/s, v_num=hpnp, train/loss=4.060"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 474/14932 [00:59<30:24,  7.92it/s, v_num=hpnp, train/loss=4.060\r",
+      "Epoch 0:   3%| | 474/14932 [00:59<30:24,  7.92it/s, v_num=hpnp, train/loss=3.360"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 475/14932 [00:59<30:25,  7.92it/s, v_num=hpnp, train/loss=3.360\r",
+      "Epoch 0:   3%| | 475/14932 [00:59<30:25,  7.92it/s, v_num=hpnp, train/loss=2.980"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 476/14932 [01:00<30:25,  7.92it/s, v_num=hpnp, train/loss=2.980\r",
+      "Epoch 0:   3%| | 476/14932 [01:00<30:25,  7.92it/s, v_num=hpnp, train/loss=3.640"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 477/14932 [01:00<30:23,  7.93it/s, v_num=hpnp, train/loss=3.640\r",
+      "Epoch 0:   3%| | 477/14932 [01:00<30:23,  7.93it/s, v_num=hpnp, train/loss=3.050"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 478/14932 [01:00<30:21,  7.93it/s, v_num=hpnp, train/loss=3.050\r",
+      "Epoch 0:   3%| | 478/14932 [01:00<30:21,  7.93it/s, v_num=hpnp, train/loss=4.000"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 479/14932 [01:00<30:19,  7.94it/s, v_num=hpnp, train/loss=4.000\r",
+      "Epoch 0:   3%| | 479/14932 [01:00<30:19,  7.94it/s, v_num=hpnp, train/loss=5.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 480/14932 [01:00<30:22,  7.93it/s, v_num=hpnp, train/loss=5.220\r",
+      "Epoch 0:   3%| | 480/14932 [01:00<30:22,  7.93it/s, v_num=hpnp, train/loss=3.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 481/14932 [01:00<30:20,  7.94it/s, v_num=hpnp, train/loss=3.660\r",
+      "Epoch 0:   3%| | 481/14932 [01:00<30:20,  7.94it/s, v_num=hpnp, train/loss=4.220"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 482/14932 [01:00<30:19,  7.94it/s, v_num=hpnp, train/loss=4.220\r",
+      "Epoch 0:   3%| | 482/14932 [01:00<30:19,  7.94it/s, v_num=hpnp, train/loss=2.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 483/14932 [01:00<30:17,  7.95it/s, v_num=hpnp, train/loss=2.880\r",
+      "Epoch 0:   3%| | 483/14932 [01:00<30:17,  7.95it/s, v_num=hpnp, train/loss=2.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 484/14932 [01:00<30:15,  7.96it/s, v_num=hpnp, train/loss=2.950\r",
+      "Epoch 0:   3%| | 484/14932 [01:00<30:15,  7.96it/s, v_num=hpnp, train/loss=3.250"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 485/14932 [01:00<30:14,  7.96it/s, v_num=hpnp, train/loss=3.250\r",
+      "Epoch 0:   3%| | 485/14932 [01:00<30:14,  7.96it/s, v_num=hpnp, train/loss=2.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 486/14932 [01:00<30:12,  7.97it/s, v_num=hpnp, train/loss=2.830\r",
+      "Epoch 0:   3%| | 486/14932 [01:00<30:12,  7.97it/s, v_num=hpnp, train/loss=3.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 487/14932 [01:01<30:10,  7.98it/s, v_num=hpnp, train/loss=3.560\r",
+      "Epoch 0:   3%| | 487/14932 [01:01<30:10,  7.98it/s, v_num=hpnp, train/loss=3.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 488/14932 [01:01<30:09,  7.98it/s, v_num=hpnp, train/loss=3.660\r",
+      "Epoch 0:   3%| | 488/14932 [01:01<30:09,  7.98it/s, v_num=hpnp, train/loss=2.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 489/14932 [01:01<30:07,  7.99it/s, v_num=hpnp, train/loss=2.750\r",
+      "Epoch 0:   3%| | 489/14932 [01:01<30:07,  7.99it/s, v_num=hpnp, train/loss=3.200"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 490/14932 [01:01<30:05,  8.00it/s, v_num=hpnp, train/loss=3.200\r",
+      "Epoch 0:   3%| | 490/14932 [01:01<30:05,  8.00it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 491/14932 [01:01<30:04,  8.00it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   3%| | 491/14932 [01:01<30:04,  8.00it/s, v_num=hpnp, train/loss=3.120"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 492/14932 [01:01<30:05,  8.00it/s, v_num=hpnp, train/loss=3.120\r",
+      "Epoch 0:   3%| | 492/14932 [01:01<30:05,  8.00it/s, v_num=hpnp, train/loss=3.230"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 493/14932 [01:01<30:03,  8.00it/s, v_num=hpnp, train/loss=3.230"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 493/14932 [01:01<30:03,  8.00it/s, v_num=hpnp, train/loss=4.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 494/14932 [01:01<30:02,  8.01it/s, v_num=hpnp, train/loss=4.970\r",
+      "Epoch 0:   3%| | 494/14932 [01:01<30:02,  8.01it/s, v_num=hpnp, train/loss=2.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 495/14932 [01:01<30:01,  8.02it/s, v_num=hpnp, train/loss=2.550\r",
+      "Epoch 0:   3%| | 495/14932 [01:01<30:01,  8.02it/s, v_num=hpnp, train/loss=1.320"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 496/14932 [01:01<29:59,  8.02it/s, v_num=hpnp, train/loss=1.320\r",
+      "Epoch 0:   3%| | 496/14932 [01:01<29:59,  8.02it/s, v_num=hpnp, train/loss=4.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 497/14932 [01:01<29:57,  8.03it/s, v_num=hpnp, train/loss=4.190\r",
+      "Epoch 0:   3%| | 497/14932 [01:01<29:57,  8.03it/s, v_num=hpnp, train/loss=1.630"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 498/14932 [01:01<29:56,  8.04it/s, v_num=hpnp, train/loss=1.630\r",
+      "Epoch 0:   3%| | 498/14932 [01:01<29:56,  8.03it/s, v_num=hpnp, train/loss=3.480"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 499/14932 [01:02<29:54,  8.04it/s, v_num=hpnp, train/loss=3.480\r",
+      "Epoch 0:   3%| | 499/14932 [01:02<29:54,  8.04it/s, v_num=hpnp, train/loss=3.730"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 500/14932 [01:02<29:53,  8.05it/s, v_num=hpnp, train/loss=3.730\r",
+      "Epoch 0:   3%| | 500/14932 [01:02<29:53,  8.05it/s, v_num=hpnp, train/loss=3.080"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 501/14932 [01:02<29:51,  8.05it/s, v_num=hpnp, train/loss=3.080\r",
+      "Epoch 0:   3%| | 501/14932 [01:02<29:51,  8.05it/s, v_num=hpnp, train/loss=2.280"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 502/14932 [01:02<29:50,  8.06it/s, v_num=hpnp, train/loss=2.280\r",
+      "Epoch 0:   3%| | 502/14932 [01:02<29:50,  8.06it/s, v_num=hpnp, train/loss=4.690"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 503/14932 [01:02<29:48,  8.07it/s, v_num=hpnp, train/loss=4.690\r",
+      "Epoch 0:   3%| | 503/14932 [01:02<29:48,  8.07it/s, v_num=hpnp, train/loss=3.770"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 504/14932 [01:02<29:47,  8.07it/s, v_num=hpnp, train/loss=3.770\r",
+      "Epoch 0:   3%| | 504/14932 [01:02<29:47,  8.07it/s, v_num=hpnp, train/loss=4.120"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 505/14932 [01:02<29:45,  8.08it/s, v_num=hpnp, train/loss=4.120\r",
+      "Epoch 0:   3%| | 505/14932 [01:02<29:45,  8.08it/s, v_num=hpnp, train/loss=3.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 506/14932 [01:02<29:43,  8.09it/s, v_num=hpnp, train/loss=3.970\r",
+      "Epoch 0:   3%| | 506/14932 [01:02<29:43,  8.09it/s, v_num=hpnp, train/loss=2.120"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 507/14932 [01:02<29:42,  8.09it/s, v_num=hpnp, train/loss=2.120\r",
+      "Epoch 0:   3%| | 507/14932 [01:02<29:42,  8.09it/s, v_num=hpnp, train/loss=2.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 508/14932 [01:02<29:40,  8.10it/s, v_num=hpnp, train/loss=2.940\r",
+      "Epoch 0:   3%| | 508/14932 [01:02<29:40,  8.10it/s, v_num=hpnp, train/loss=3.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 509/14932 [01:02<29:39,  8.11it/s, v_num=hpnp, train/loss=3.750\r",
+      "Epoch 0:   3%| | 509/14932 [01:02<29:39,  8.11it/s, v_num=hpnp, train/loss=3.230"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 510/14932 [01:02<29:38,  8.11it/s, v_num=hpnp, train/loss=3.230\r",
+      "Epoch 0:   3%| | 510/14932 [01:02<29:38,  8.11it/s, v_num=hpnp, train/loss=4.280"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 511/14932 [01:02<29:37,  8.11it/s, v_num=hpnp, train/loss=4.280\r",
+      "Epoch 0:   3%| | 511/14932 [01:02<29:37,  8.11it/s, v_num=hpnp, train/loss=2.080"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 512/14932 [01:03<29:39,  8.10it/s, v_num=hpnp, train/loss=2.080\r",
+      "Epoch 0:   3%| | 512/14932 [01:03<29:39,  8.10it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 513/14932 [01:03<29:37,  8.11it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   3%| | 513/14932 [01:03<29:37,  8.11it/s, v_num=hpnp, train/loss=3.300"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 514/14932 [01:03<29:35,  8.12it/s, v_num=hpnp, train/loss=3.300\r",
+      "Epoch 0:   3%| | 514/14932 [01:03<29:35,  8.12it/s, v_num=hpnp, train/loss=3.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 515/14932 [01:03<29:34,  8.12it/s, v_num=hpnp, train/loss=3.380\r",
+      "Epoch 0:   3%| | 515/14932 [01:03<29:34,  8.12it/s, v_num=hpnp, train/loss=2.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 516/14932 [01:03<29:37,  8.11it/s, v_num=hpnp, train/loss=2.940\r",
+      "Epoch 0:   3%| | 516/14932 [01:03<29:37,  8.11it/s, v_num=hpnp, train/loss=3.670"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 517/14932 [01:03<29:35,  8.12it/s, v_num=hpnp, train/loss=3.670\r",
+      "Epoch 0:   3%| | 517/14932 [01:03<29:35,  8.12it/s, v_num=hpnp, train/loss=3.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 518/14932 [01:03<29:33,  8.13it/s, v_num=hpnp, train/loss=3.750\r",
+      "Epoch 0:   3%| | 518/14932 [01:03<29:34,  8.13it/s, v_num=hpnp, train/loss=3.470"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 519/14932 [01:03<29:32,  8.13it/s, v_num=hpnp, train/loss=3.470\r",
+      "Epoch 0:   3%| | 519/14932 [01:03<29:32,  8.13it/s, v_num=hpnp, train/loss=4.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 520/14932 [01:03<29:31,  8.14it/s, v_num=hpnp, train/loss=4.030\r",
+      "Epoch 0:   3%| | 520/14932 [01:03<29:31,  8.14it/s, v_num=hpnp, train/loss=5.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 521/14932 [01:04<29:31,  8.13it/s, v_num=hpnp, train/loss=5.340\r",
+      "Epoch 0:   3%| | 521/14932 [01:04<29:31,  8.13it/s, v_num=hpnp, train/loss=3.170"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   3%| | 522/14932 [01:04<29:30,  8.14it/s, v_num=hpnp, train/loss=3.170\r",
+      "Epoch 0:   3%| | 522/14932 [01:04<29:30,  8.14it/s, v_num=hpnp, train/loss=3.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 523/14932 [01:04<29:28,  8.15it/s, v_num=hpnp, train/loss=3.940\r",
+      "Epoch 0:   4%| | 523/14932 [01:04<29:28,  8.15it/s, v_num=hpnp, train/loss=4.030"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 524/14932 [01:04<29:27,  8.15it/s, v_num=hpnp, train/loss=4.030\r",
+      "Epoch 0:   4%| | 524/14932 [01:04<29:27,  8.15it/s, v_num=hpnp, train/loss=3.920"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 525/14932 [01:04<29:26,  8.16it/s, v_num=hpnp, train/loss=3.920\r",
+      "Epoch 0:   4%| | 525/14932 [01:04<29:26,  8.16it/s, v_num=hpnp, train/loss=2.660"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 526/14932 [01:04<29:24,  8.16it/s, v_num=hpnp, train/loss=2.660\r",
+      "Epoch 0:   4%| | 526/14932 [01:04<29:24,  8.16it/s, v_num=hpnp, train/loss=2.940"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 527/14932 [01:04<29:22,  8.17it/s, v_num=hpnp, train/loss=2.940\r",
+      "Epoch 0:   4%| | 527/14932 [01:04<29:23,  8.17it/s, v_num=hpnp, train/loss=1.270"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 528/14932 [01:04<29:21,  8.18it/s, v_num=hpnp, train/loss=1.270\r",
+      "Epoch 0:   4%| | 528/14932 [01:04<29:21,  8.18it/s, v_num=hpnp, train/loss=2.720"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 529/14932 [01:04<29:20,  8.18it/s, v_num=hpnp, train/loss=2.720\r",
+      "Epoch 0:   4%| | 529/14932 [01:04<29:20,  8.18it/s, v_num=hpnp, train/loss=2.880"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 530/14932 [01:04<29:18,  8.19it/s, v_num=hpnp, train/loss=2.880\r",
+      "Epoch 0:   4%| | 530/14932 [01:04<29:18,  8.19it/s, v_num=hpnp, train/loss=3.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 531/14932 [01:04<29:17,  8.20it/s, v_num=hpnp, train/loss=3.380\r",
+      "Epoch 0:   4%| | 531/14932 [01:04<29:17,  8.20it/s, v_num=hpnp, train/loss=3.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 532/14932 [01:04<29:15,  8.20it/s, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   4%| | 532/14932 [01:04<29:15,  8.20it/s, v_num=hpnp, train/loss=2.770"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 533/14932 [01:04<29:14,  8.21it/s, v_num=hpnp, train/loss=2.770\r",
+      "Epoch 0:   4%| | 533/14932 [01:04<29:14,  8.21it/s, v_num=hpnp, train/loss=2.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 534/14932 [01:05<29:12,  8.21it/s, v_num=hpnp, train/loss=2.380\r",
+      "Epoch 0:   4%| | 534/14932 [01:05<29:12,  8.21it/s, v_num=hpnp, train/loss=3.530"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 535/14932 [01:05<29:11,  8.22it/s, v_num=hpnp, train/loss=3.530\r",
+      "Epoch 0:   4%| | 535/14932 [01:05<29:11,  8.22it/s, v_num=hpnp, train/loss=3.800"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 536/14932 [01:05<29:10,  8.23it/s, v_num=hpnp, train/loss=3.800\r",
+      "Epoch 0:   4%| | 536/14932 [01:05<29:10,  8.23it/s, v_num=hpnp, train/loss=4.190"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 537/14932 [01:05<29:08,  8.23it/s, v_num=hpnp, train/loss=4.190\r",
+      "Epoch 0:   4%| | 537/14932 [01:05<29:08,  8.23it/s, v_num=hpnp, train/loss=3.390"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 538/14932 [01:05<29:07,  8.24it/s, v_num=hpnp, train/loss=3.390\r",
+      "Epoch 0:   4%| | 538/14932 [01:05<29:07,  8.24it/s, v_num=hpnp, train/loss=4.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 539/14932 [01:05<29:05,  8.24it/s, v_num=hpnp, train/loss=4.380\r",
+      "Epoch 0:   4%| | 539/14932 [01:05<29:05,  8.24it/s, v_num=hpnp, train/loss=1.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 540/14932 [01:05<29:04,  8.25it/s, v_num=hpnp, train/loss=1.950\r",
+      "Epoch 0:   4%| | 540/14932 [01:05<29:04,  8.25it/s, v_num=hpnp, train/loss=3.620"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 541/14932 [01:05<29:02,  8.26it/s, v_num=hpnp, train/loss=3.620\r",
+      "Epoch 0:   4%| | 541/14932 [01:05<29:02,  8.26it/s, v_num=hpnp, train/loss=3.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 542/14932 [01:05<29:03,  8.25it/s, v_num=hpnp, train/loss=3.560\r",
+      "Epoch 0:   4%| | 542/14932 [01:05<29:03,  8.25it/s, v_num=hpnp, train/loss=2.980"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 543/14932 [01:05<29:01,  8.26it/s, v_num=hpnp, train/loss=2.980\r",
+      "Epoch 0:   4%| | 543/14932 [01:05<29:01,  8.26it/s, v_num=hpnp, train/loss=2.170"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 544/14932 [01:05<29:03,  8.25it/s, v_num=hpnp, train/loss=2.170\r",
+      "Epoch 0:   4%| | 544/14932 [01:05<29:03,  8.25it/s, v_num=hpnp, train/loss=2.020"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 545/14932 [01:06<29:05,  8.24it/s, v_num=hpnp, train/loss=2.020\r",
+      "Epoch 0:   4%| | 545/14932 [01:06<29:05,  8.24it/s, v_num=hpnp, train/loss=4.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 546/14932 [01:06<29:04,  8.25it/s, v_num=hpnp, train/loss=4.810\r",
+      "Epoch 0:   4%| | 546/14932 [01:06<29:04,  8.25it/s, v_num=hpnp, train/loss=2.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 547/14932 [01:06<29:03,  8.25it/s, v_num=hpnp, train/loss=2.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 547/14932 [01:06<29:03,  8.25it/s, v_num=hpnp, train/loss=3.440"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 548/14932 [01:06<29:01,  8.26it/s, v_num=hpnp, train/loss=3.440\r",
+      "Epoch 0:   4%| | 548/14932 [01:06<29:01,  8.26it/s, v_num=hpnp, train/loss=3.050"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 549/14932 [01:06<29:00,  8.27it/s, v_num=hpnp, train/loss=3.050\r",
+      "Epoch 0:   4%| | 549/14932 [01:06<29:00,  8.26it/s, v_num=hpnp, train/loss=2.830"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 550/14932 [01:06<28:58,  8.27it/s, v_num=hpnp, train/loss=2.830\r",
+      "Epoch 0:   4%| | 550/14932 [01:06<28:58,  8.27it/s, v_num=hpnp, train/loss=3.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 551/14932 [01:06<28:59,  8.27it/s, v_num=hpnp, train/loss=3.950\r",
+      "Epoch 0:   4%| | 551/14932 [01:06<28:59,  8.27it/s, v_num=hpnp, train/loss=4.340"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 552/14932 [01:06<28:57,  8.27it/s, v_num=hpnp, train/loss=4.340\r",
+      "Epoch 0:   4%| | 552/14932 [01:06<28:57,  8.27it/s, v_num=hpnp, train/loss=3.610"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 553/14932 [01:06<28:56,  8.28it/s, v_num=hpnp, train/loss=3.610\r",
+      "Epoch 0:   4%| | 553/14932 [01:06<28:56,  8.28it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 554/14932 [01:06<28:55,  8.29it/s, v_num=hpnp, train/loss=3.780\r",
+      "Epoch 0:   4%| | 554/14932 [01:06<28:55,  8.29it/s, v_num=hpnp, train/loss=3.800"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 555/14932 [01:06<28:53,  8.29it/s, v_num=hpnp, train/loss=3.800\r",
+      "Epoch 0:   4%| | 555/14932 [01:06<28:53,  8.29it/s, v_num=hpnp, train/loss=3.590"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 556/14932 [01:07<28:52,  8.30it/s, v_num=hpnp, train/loss=3.590\r",
+      "Epoch 0:   4%| | 556/14932 [01:07<28:52,  8.30it/s, v_num=hpnp, train/loss=3.810"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 557/14932 [01:07<28:50,  8.30it/s, v_num=hpnp, train/loss=3.810\r",
+      "Epoch 0:   4%| | 557/14932 [01:07<28:50,  8.30it/s, v_num=hpnp, train/loss=3.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 558/14932 [01:07<28:49,  8.31it/s, v_num=hpnp, train/loss=3.160\r",
+      "Epoch 0:   4%| | 558/14932 [01:07<28:49,  8.31it/s, v_num=hpnp, train/loss=2.910"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 559/14932 [01:07<28:48,  8.32it/s, v_num=hpnp, train/loss=2.910\r",
+      "Epoch 0:   4%| | 559/14932 [01:07<28:48,  8.31it/s, v_num=hpnp, train/loss=3.970"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 560/14932 [01:07<28:47,  8.32it/s, v_num=hpnp, train/loss=3.970\r",
+      "Epoch 0:   4%| | 560/14932 [01:07<28:47,  8.32it/s, v_num=hpnp, train/loss=1.840"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 561/14932 [01:07<28:45,  8.33it/s, v_num=hpnp, train/loss=1.840\r",
+      "Epoch 0:   4%| | 561/14932 [01:07<28:45,  8.33it/s, v_num=hpnp, train/loss=2.950"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 562/14932 [01:07<28:44,  8.33it/s, v_num=hpnp, train/loss=2.950\r",
+      "Epoch 0:   4%| | 562/14932 [01:07<28:44,  8.33it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 563/14932 [01:07<28:42,  8.34it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   4%| | 563/14932 [01:07<28:42,  8.34it/s, v_num=hpnp, train/loss=3.080"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 564/14932 [01:07<28:41,  8.35it/s, v_num=hpnp, train/loss=3.080\r",
+      "Epoch 0:   4%| | 564/14932 [01:07<28:41,  8.35it/s, v_num=hpnp, train/loss=3.770"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 565/14932 [01:07<28:40,  8.35it/s, v_num=hpnp, train/loss=3.770\r",
+      "Epoch 0:   4%| | 565/14932 [01:07<28:40,  8.35it/s, v_num=hpnp, train/loss=3.480"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 566/14932 [01:07<28:38,  8.36it/s, v_num=hpnp, train/loss=3.480\r",
+      "Epoch 0:   4%| | 566/14932 [01:07<28:38,  8.36it/s, v_num=hpnp, train/loss=2.750"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 567/14932 [01:07<28:37,  8.36it/s, v_num=hpnp, train/loss=2.750\r",
+      "Epoch 0:   4%| | 567/14932 [01:07<28:37,  8.36it/s, v_num=hpnp, train/loss=3.380"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 568/14932 [01:07<28:37,  8.36it/s, v_num=hpnp, train/loss=3.380\r",
+      "Epoch 0:   4%| | 568/14932 [01:07<28:37,  8.36it/s, v_num=hpnp, train/loss=4.160"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 569/14932 [01:07<28:36,  8.37it/s, v_num=hpnp, train/loss=4.160\r",
+      "Epoch 0:   4%| | 569/14932 [01:07<28:36,  8.37it/s, v_num=hpnp, train/loss=3.470"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 570/14932 [01:08<28:35,  8.37it/s, v_num=hpnp, train/loss=3.470\r",
+      "Epoch 0:   4%| | 570/14932 [01:08<28:35,  8.37it/s, v_num=hpnp, train/loss=3.620"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 571/14932 [01:08<28:34,  8.38it/s, v_num=hpnp, train/loss=3.620\r",
+      "Epoch 0:   4%| | 571/14932 [01:08<28:34,  8.38it/s, v_num=hpnp, train/loss=3.080"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 572/14932 [01:08<28:32,  8.39it/s, v_num=hpnp, train/loss=3.080\r",
+      "Epoch 0:   4%| | 572/14932 [01:08<28:32,  8.38it/s, v_num=hpnp, train/loss=2.640"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 573/14932 [01:08<28:31,  8.39it/s, v_num=hpnp, train/loss=2.640\r",
+      "Epoch 0:   4%| | 573/14932 [01:08<28:31,  8.39it/s, v_num=hpnp, train/loss=2.560"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 574/14932 [01:08<28:30,  8.40it/s, v_num=hpnp, train/loss=2.560\r",
+      "Epoch 0:   4%| | 574/14932 [01:08<28:30,  8.40it/s, v_num=hpnp, train/loss=4.410"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 575/14932 [01:08<28:28,  8.40it/s, v_num=hpnp, train/loss=4.410\r",
+      "Epoch 0:   4%| | 575/14932 [01:08<28:28,  8.40it/s, v_num=hpnp, train/loss=3.550"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\r",
+      "Epoch 0:   4%| | 576/14932 [01:08<28:30,  8.39it/s, v_num=hpnp, train/loss=3.550\r",
+      "Epoch 0:   4%| | 576/14932 [01:08<28:30,  8.39it/s, v_num=hpnp, train/loss=3.780"
+     ]
+    }
+   ],
+   "source": [
+    "# Start the instruct finetuning\n",
+    "!cd \"{TRAINER_DIR}\" && \\\n",
+    "    export WANDB_MODE=\"{WANDB_MODE}\" && \\\n",
+    "    python3 lightning_trainer.py fit \\\n",
+    "        -c \"{CONFIG_DIR}/config-enwiki-instruct.yaml\" \\\n",
+    "        --trainer.logger.init_args.name=\"{WANDB_PREFIX} - Enwiki-Instruct (train-ctx=4k, {DEEPSPEED_STRAT})\" \\\n",
+    "        --trainer.strategy=\"{DEEPSPEED_STRAT}\" \\\n",
+    "        --trainer.devices=\"{GPU_DEVICES}\" \\\n",
+    "        --trainer.callbacks.init_args.dirpath=\"../checkpoint/{FILENAME_PREFIX}-enwiki-instruct/\" \\\n",
+    "        --model.load_model=\"../model/{FILENAME_PREFIX}-enwiki-4k.pth\" \\\n",
+    "        --model.ctx_len=4096 \\\n",
+    "        --model.bptt_learning_range=1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f980e672",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Lets export the model from the checkpoint\n",
+    "!cd \"{TRAINER_DIR}\" && \\\n",
+    "    python3 export_checkpoint.py \"../checkpoint/{FILENAME_PREFIX}-enwiki-instruct/last.ckpt\" \"../model/{FILENAME_PREFIX}-enwiki-instruct.pth\" \"bf16\"\n",
+    "!cd \"{TRAINER_DIR}\" && ls -alh \"../model/{FILENAME_PREFIX}-enwiki-instruct.pth\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30e999ee",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# # Lets do a quick dragon prompt validation\n",
+    "!cd \"{INFERENCE_DIR}\" && \\\n",
+    "    python3 dragon_test.py \"../model/{FILENAME_PREFIX}-enwiki-instruct.pth\" \"cuda fp32\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "79d6eb35",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Lets do a quick memory test\n",
+    "!python3\n",
+    "        python3 ../memory_script/eval_v5_memory_guided.py \"{PROJECT_DIR}/model/{FILENAME_PREFIX}-enwiki-instruct.pth\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1e778681",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "source": [
+    "## Tune 1 : Simple Memory instruct finetuning\n",
+    "\n",
+    "- Tune 1: Low ctx size (512), Training with only the input masked. This does very limited memory training, and is used primarily to train the instruction set."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e284d692",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%%script bash\n",
+    "\n",
+    "########################################\n",
+    "# Generate the required jsonl dataset\n",
+    "########################################\n",
+    "\n",
+    "# Reset the dataset dir\n",
+    "mkdir -p ../dataset\n",
+    "rm -rf ../dataset/*.jsonl\n",
+    "\n",
+    "# Generate the various datasets\n",
+    "echo \"## Generating word reptition dataset ##\"\n",
+    "\n",
+    "# We do a strong bias for smaller word count, to teach the concept from scratch\n",
+    "# so that the model can learn the function. \n",
+    "#\n",
+    "# Note that all document samples, are randomized between the target word count, \n",
+    "# to half of the target word count.\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-2-count.jsonl  2  5000 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-5-count.jsonl  5  5000 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-10-count.jsonl 10 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-15-count.jsonl 15 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-20-count.jsonl 20 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-25-count.jsonl 25 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-40-count.jsonl 40 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-50-count.jsonl 50 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-60-count.jsonl 80 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-80-count.jsonl 80 2500 &\n",
+    "\n",
+    "# With a slight mix of the larger word count\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-100-count.jsonl 100 2500 &\n",
+    "python3 ../memory_script/gen_limited_segmented_jsonl.py ../dataset/word-200-count.jsonl 200 2500 &\n",
+    "\n",
+    "wait\n",
+    "echo \"## Done ##\"\n",
+    "\n",
+    "ls -alh ../dataset/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "682e13a6",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Start the finetune model training\n",
+    "!cd \"{TRAINER_DIR}\" && \\\n",
+    "    export WANDB_MODE=\"{WANDB_MODE}\" && \\\n",
+    "    python3 lightning_trainer.py fit \\\n",
+    "        -c \"{CONFIG_DIR}/config-mem-instruct.yaml\" \\\n",
+    "        --trainer.logger.init_args.name=\"{WANDB_PREFIX} - Mem-Instruct (train-ctx=512, {DEEPSPEED_STRAT})\" \\\n",
+    "        --trainer.strategy=\"{DEEPSPEED_STRAT}\" \\\n",
+    "        --trainer.devices=\"{GPU_DEVICES}\" \\\n",
+    "        --trainer.callbacks.init_args.dirpath=\"../checkpoint/{FILENAME_PREFIX}-mem-instruct/\" \\\n",
+    "        --model.load_model=\"../model/{FILENAME_PREFIX}-enwiki-instruct.pth\" \\\n",
+    "        --model.ctx_len=512 \\\n",
+    "        --model.bptt_learning_range=1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cb00a0de",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Lets export the model from the checkpoint\n",
+    "!cd \"{TRAINER_DIR}\" && \\\n",
+    "    python3 export_checkpoint.py \\\n",
+    "        \"../checkpoint/{FILENAME_PREFIX}-mem-instruct/last.ckpt\" \\\n",
+    "        \"../model/{FILENAME_PREFIX}-mem-instruct.pth\" \"bf16\"\n",
+    "!cd \"{TRAINER_DIR}\" && ls -alh \"../model/{FILENAME_PREFIX}-mem-instruct.pth\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b23e0082",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Lets do a quick memory test\n",
+    "!python3 ../memory_script/eval_v5_memory_guided.py \"{PROJECT_DIR}/model/{FILENAME_PREFIX}-mem-instruct.pth\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a5b04004",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "source": [
+    "## Tune 2 : Low ctx size (512), memory training\n",
+    "\n",
+    "- Tune 2: Low ctx size (512), Training with instruction & input masked. This forces the actual memory training on the output tokens."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13388b95",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%%script bash\n",
+    "\n",
+    "########################################\n",
+    "# Generate the required jsonl dataset\n",
+    "########################################\n",
+    "\n",
+    "# Reset the dataset dir\n",
+    "mkdir -p ../dataset\n",
+    "rm -rf ../dataset/*.jsonl\n",
+    "\n",
+    "# Generate the various datasets\n",
+    "echo \"## Generating word reptition dataset ##\"\n",
+    "\n",
+    "#\n",
+    "# We switch over to fully masked instruct+input, to properly learn the memorization task\n",
+    "#\n",
+    "python3 ../memory_script/gen_limited_prompt_completion_jsonl.py ../dataset/word-2-count.jsonl  2  5000 &\n",
+    "for i in {5..95..5} \n",
+    "do\n",
+    "    python3 ../memory_script/gen_limited_prompt_completion_jsonl.py ../dataset/gen-word-$i-count.jsonl $i 5000 & \n",
+    "done\n",
+    "python3 ../memory_script/gen_limited_prompt_completion_jsonl.py ../dataset/word-100-count.jsonl 100 5000 &\n",
+    "python3 ../memory_script/gen_limited_prompt_completion_jsonl.py ../dataset/word-200-count.jsonl 200 5000 &\n",
+    "\n",
+    "#\n",
+    "# We mixin the shuffled word list, so that we ensure all words / tokens are learned\n",
+    "# however this might intrduce an exclusion bias (if seen this word, never repeat it), \n",
+    "# so we limit the mixture of this data samples\n",
+    "#\n",
+    "python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-10-count.jsonl 10 20 &\n",
+    "python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-15-count.jsonl 15 20 &\n",
+    "python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-25-count.jsonl 25 30 &\n",
+    "python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-50-count.jsonl 50 50 &\n",
+    "python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-75-count.jsonl 75 50 &\n",
+    "python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-100-count.jsonl 100 50 &\n",
+    "python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-200-count.jsonl 200 50 &\n",
+    "\n",
+    "wait\n",
+    "echo \"## Done ##\"\n",
+    "\n",
+    "ls -alh ../dataset/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b97a452e",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Start the finetune model training\n",
+    "!cd \"{TRAINER_DIR}\" && \\\n",
+    "    export WANDB_MODE=\"{WANDB_MODE}\" && \\\n",
+    "    python3 lightning_trainer.py fit \\\n",
+    "        -c \"{CONFIG_DIR}/config-mem-template.yaml\" \\\n",
+    "        --trainer.logger.init_args.name=\"{WANDB_PREFIX} - Mem-Tune ctx-512 (train-ctx=512, {DEEPSPEED_STRAT})\" \\\n",
+    "        --trainer.strategy=\"{DEEPSPEED_STRAT}\" \\\n",
+    "        --trainer.devices=\"{GPU_DEVICES}\"  \\\n",
+    "        --trainer.callbacks.init_args.dirpath=\"../checkpoint/{FILENAME_PREFIX}-mem-ctx-512/\" \\\n",
+    "        --model.lr_init=5e-4 \\\n",
+    "        --model.lr_final=4e-4 \\\n",
+    "        --data.max_token_size=512 \\\n",
+    "        --model.ctx_len=512 \\\n",
+    "        --model.bptt_learning_range=1 \\\n",
+    "        --model.load_model=\"../model/{FILENAME_PREFIX}-mem-instruct.pth\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "28c68850",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Lets export the model from the checkpoint\n",
+    "!cd \"{TRAINER_DIR}\" && \\\n",
+    "    python3 export_checkpoint.py \\\n",
+    "        \"../checkpoint/{FILENAME_PREFIX}-mem-ctx-512/last.ckpt\" \\\n",
+    "        \"../model/{FILENAME_PREFIX}-mem-ctx-512.pth\" \"bf16\"\n",
+    "!cd \"{TRAINER_DIR}\" && ls -alh \"../model/{FILENAME_PREFIX}-mem-ctx-512.pth\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21791457",
+   "metadata": {
+    "papermill": {
+     "duration": null,
+     "end_time": null,
+     "exception": null,
+     "start_time": null,
+     "status": "pending"
+    },
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Lets do a quick memory test\n",
+    "!python3 ../memory_script/eval_v5_memory_guided.py \"{PROJECT_DIR}/model/{FILENAME_PREFIX}-mem-ctx-512.pth\""
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  },
+  "papermill": {
+   "default_parameters": {},
+   "duration": null,
+   "end_time": null,
+   "environment_variables": {},
+   "exception": null,
+   "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/part2.ipynb",
+   "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/part2.ipynb",
+   "parameters": {},
+   "start_time": "2023-09-06T17:17:05.666978",
+   "version": "2.4.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
\ No newline at end of file