Delete trainer-v4-unit-test

trainer-v4-unit-test/L6-D512-neox-init.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d3d06f5cb8762098d57586d37f251cecb27464dfe16955f2c0a9da4781f1ed17
-size 143968613

trainer-v4-unit-test/trainer-v4-unit-test/model-init.ipynb

@@ -1,294 +0,0 @@
-{
- "cells": [
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "id": "f8d0f2a8",
-   "metadata": {
-    "papermill": {
-     "duration": 0.001857,
-     "end_time": "2023-08-23T10:30:31.380683",
-     "exception": false,
-     "start_time": "2023-08-23T10:30:31.378826",
-     "status": "completed"
-    },
-    "tags": []
-   },
-   "source": [
-    "# Model Init\n",
-    "\n",
-    "Test that the model init code, runs without issues\n",
-    "\n",
-    "**L6-D512 model with**\n",
-    "- Layer count: 6\n",
-    "- Embed size: 512"
-   ]
-  },
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "id": "5877c2a8",
-   "metadata": {
-    "notebookRunGroups": {
-     "groupValue": ""
-    },
-    "papermill": {
-     "duration": 0.001252,
-     "end_time": "2023-08-23T10:30:31.383486",
-     "exception": false,
-     "start_time": "2023-08-23T10:30:31.382234",
-     "status": "completed"
-    },
-    "tags": []
-   },
-   "source": [
-    "## Preparing the init model and test dataset"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "a6979aad",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2023-08-23T10:30:31.389495Z",
-     "iopub.status.busy": "2023-08-23T10:30:31.388151Z",
-     "iopub.status.idle": "2023-08-23T10:30:32.146278Z",
-     "shell.execute_reply": "2023-08-23T10:30:32.144823Z"
-    },
-    "papermill": {
-     "duration": 0.76389,
-     "end_time": "2023-08-23T10:30:32.148710",
-     "exception": false,
-     "start_time": "2023-08-23T10:30:31.384820",
-     "status": "completed"
-    },
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "# First lets setup the various directories\n",
-    "!mkdir -p ../../model/\n",
-    "!mkdir -p ../../datapath/\n",
-    "!mkdir -p ../../checkpoint/"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "831f03fd",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2023-08-23T10:30:32.155121Z",
-     "iopub.status.busy": "2023-08-23T10:30:32.153892Z",
-     "iopub.status.idle": "2023-08-23T10:30:41.918539Z",
-     "shell.execute_reply": "2023-08-23T10:30:41.917396Z"
-    },
-    "papermill": {
-     "duration": 9.770776,
-     "end_time": "2023-08-23T10:30:41.921168",
-     "exception": false,
-     "start_time": "2023-08-23T10:30:32.150392",
-     "status": "completed"
-    },
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[2023-08-23 10:30:36,724] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n",
-      "---- Initializing model ----\r\n",
-      "No of layers: 6\r\n",
-      "Embedding size: 512\r\n",
-      "Output model path: ../model/L6-D512-neox-init.pth\r\n",
-      "Vocab size: 50277\r\n",
-      "Note: this process takes a significant time (and ram) for large models\r\n",
-      "---- ----- ----\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Using /root/.cache/torch_extensions/py310_cu118 as PyTorch extensions root...\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Detected CUDA files, patching ldflags\r\n",
-      "Emitting ninja build file /root/.cache/torch_extensions/py310_cu118/wkv_1_bf16/build.ninja...\r\n",
-      "Building extension module wkv_1_bf16...\r\n",
-      "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "ninja: no work to do.\r\n",
-      "Loading extension module wkv_1_bf16...\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[RWKV.model]: Finished initial model load\r\n",
-      "50277 512   -0.0001 emb.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.0.att.key.weight\r\n",
-      "512   512   1.0  blocks.0.att.value.weight\r\n",
-      "512   512   0    blocks.0.att.receptance.weight\r\n",
-      "512   512   0    blocks.0.att.output.weight\r\n",
-      "2048  512   1.0  blocks.0.ffn.key.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.0.ffn.receptance.weight\r\n",
-      "512   2048  0    blocks.0.ffn.value.weight\r\n",
-      "512   512   0    blocks.1.att.key.weight\r\n",
-      "512   512   1.0  blocks.1.att.value.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.1.att.receptance.weight\r\n",
-      "512   512   0    blocks.1.att.output.weight\r\n",
-      "2048  512   1.0  blocks.1.ffn.key.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.1.ffn.receptance.weight\r\n",
-      "512   2048  0    blocks.1.ffn.value.weight\r\n",
-      "512   512   0    blocks.2.att.key.weight\r\n",
-      "512   512   1.0  blocks.2.att.value.weight\r\n",
-      "512   512   0    blocks.2.att.receptance.weight\r\n",
-      "512   512   0    blocks.2.att.output.weight\r\n",
-      "2048  512   1.0  blocks.2.ffn.key.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.2.ffn.receptance.weight\r\n",
-      "512   2048  0    blocks.2.ffn.value.weight\r\n",
-      "512   512   0    blocks.3.att.key.weight\r\n",
-      "512   512   1.0  blocks.3.att.value.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.3.att.receptance.weight\r\n",
-      "512   512   0    blocks.3.att.output.weight\r\n",
-      "2048  512   1.0  blocks.3.ffn.key.weight\r\n",
-      "512   512   0    blocks.3.ffn.receptance.weight\r\n",
-      "512   2048  0    blocks.3.ffn.value.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.4.att.key.weight\r\n",
-      "512   512   1.0  blocks.4.att.value.weight\r\n",
-      "512   512   0    blocks.4.att.receptance.weight\r\n",
-      "512   512   0    blocks.4.att.output.weight\r\n",
-      "2048  512   1.0  blocks.4.ffn.key.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "512   512   0    blocks.4.ffn.receptance.weight\r\n",
-      "512   2048  0    blocks.4.ffn.value.weight\r\n",
-      "512   512   0    blocks.5.att.key.weight\r\n",
-      "512   512   1.0  blocks.5.att.value.weight\r\n",
-      "512   512   0    blocks.5.att.receptance.weight\r\n",
-      "512   512   0    blocks.5.att.output.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "2048  512   1.0  blocks.5.ffn.key.weight\r\n",
-      "512   512   0    blocks.5.ffn.receptance.weight\r\n",
-      "512   2048  0    blocks.5.ffn.value.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "50277 512   0.5  head.weight\r\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Lets initialized the L6-D512 model with the init_model.py code\n",
-    "!cd ../../RWKV-v4neo/ && python3 init_model.py --n_layer 6 --n_embd 512 --vocab_size neox ../model/L6-D512-neox-init.pth"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "rwkv-exp",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
-  },
-  "papermill": {
-   "default_parameters": {},
-   "duration": 12.324741,
-   "end_time": "2023-08-23T10:30:42.347605",
-   "environment_variables": {},
-   "exception": null,
-   "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/trainer-v4-unit-test/model-init.ipynb",
-   "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/trainer-v4-unit-test/model-init.ipynb",
-   "parameters": {},
-   "start_time": "2023-08-23T10:30:30.022864",
-   "version": "2.4.0"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}