{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b60de1f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-24T06:38:13.233516Z",
     "iopub.status.busy": "2023-08-24T06:38:13.233344Z",
     "iopub.status.idle": "2023-08-24T06:38:13.475603Z",
     "shell.execute_reply": "2023-08-24T06:38:13.474843Z"
    },
    "papermill": {
     "duration": 0.24656,
     "end_time": "2023-08-24T06:38:13.476917",
     "exception": false,
     "start_time": "2023-08-24T06:38:13.230357",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello world\r\n"
     ]
    }
   ],
   "source": [
    "!echo \"Hello world\" # this file is for debugging the runner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf387939",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-24T06:38:13.480448Z",
     "iopub.status.busy": "2023-08-24T06:38:13.480263Z",
     "iopub.status.idle": "2023-08-24T06:38:13.486354Z",
     "shell.execute_reply": "2023-08-24T06:38:13.485719Z"
    },
    "papermill": {
     "duration": 0.008964,
     "end_time": "2023-08-24T06:38:13.487435",
     "exception": false,
     "start_time": "2023-08-24T06:38:13.478471",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DEEPSPEED_STRAT: deepspeed_stage_2_offload\n",
      "ENABLE_WANDB: True\n",
      "GPU_DEVICES: auto\n",
      "NOTEBOOK_DIR: /actions-runner-lane2/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-headsize2x\n",
      "INFERENCE_DIR: /actions-runner-lane2/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5headsize2x\n",
      "TRAINER_DIR: /actions-runner-lane2/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5headsize2x\n",
      "PROJECT_DIR: /actions-runner-lane2/_work/RWKV-infctx-trainer/RWKV-infctx-trainer\n"
     ]
    }
   ],
   "source": [
    "DEEPSPEED_STRAT=\"deepspeed_stage_2_offload\"\n",
    "GPU_DEVICES=\"auto\"\n",
    "ENABLE_WANDB=True\n",
    "\n",
    "RWKV_WAVENET_LAYERS=1\n",
    "\n",
    "EMBED_SCALE=0.1\n",
    "EMBED_SCALE_LABEL=str(EMBED_SCALE).replace(\".\", \"_\")\n",
    "\n",
    "LAYER_COUNT=6\n",
    "EMBED_DIM=4096\n",
    "\n",
    "WANDB_PREFIX=f\"v5-hs2x-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE}\"\n",
    "FILENAME_PREFIX=f\"v5-hs2x-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE_LABEL}\"\n",
    "\n",
    "print(\"DEEPSPEED_STRAT:\", DEEPSPEED_STRAT)\n",
    "print(\"ENABLE_WANDB:\", ENABLE_WANDB)\n",
    "print(\"GPU_DEVICES:\", GPU_DEVICES)\n",
    "\n",
    "if ENABLE_WANDB:\n",
    "    WANDB_MODE=\"online\"\n",
    "else:\n",
    "    WANDB_MODE=\"disabled\"\n",
    "\n",
    "# Computing the notebook, and various paths\n",
    "import os\n",
    "NOTEBOOK_DIR=os.path.dirname(os.path.abspath(\"__file__\"))\n",
    "PROJECT_DIR=os.path.abspath(os.path.join(NOTEBOOK_DIR, \"../../../../\"))\n",
    "TRAINER_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5headsize2x/\"))\n",
    "INFERENCE_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5headsize2x/\"))\n",
    "\n",
    "print(\"NOTEBOOK_DIR:\", NOTEBOOK_DIR)\n",
    "print(\"INFERENCE_DIR:\", INFERENCE_DIR)\n",
    "print(\"TRAINER_DIR:\", TRAINER_DIR)\n",
    "print(\"PROJECT_DIR:\", PROJECT_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "07c2dc29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-24T06:38:13.490328Z",
     "iopub.status.busy": "2023-08-24T06:38:13.490163Z",
     "iopub.status.idle": "2023-08-24T06:38:13.957453Z",
     "shell.execute_reply": "2023-08-24T06:38:13.956594Z"
    },
    "papermill": {
     "duration": 0.470639,
     "end_time": "2023-08-24T06:38:13.959185",
     "exception": false,
     "start_time": "2023-08-24T06:38:13.488546",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/actions-runner-lane2/_work/RWKV-infctx-trainer/RWKV-infctx-trainer\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 4.0K\r\n",
      "drwxr-xr-x  2 root root   10 Aug 24 06:38 .\r\n",
      "drwxr-xr-x 19 root root 4.0K Aug 24 06:38 ..\r\n"
     ]
    }
   ],
   "source": [
    "# Get the current model listing\n",
    "!cd \"{TRAINER_DIR}\" && cd \"../\" && pwd\n",
    "!cd \"{TRAINER_DIR}\" && ls -alh \"../model/\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rwkv-infctx",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1.889768,
   "end_time": "2023-08-24T06:38:14.177579",
   "environment_variables": {},
   "exception": null,
   "input_path": "/actions-runner-lane2/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-headsize2x/hello-world.ipynb",
   "output_path": "/actions-runner-lane2/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/rwkv-x-exp/v5-headsize2x/hello-world.ipynb",
   "parameters": {},
   "start_time": "2023-08-24T06:38:12.287811",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}