Upload 8 files

adapter_config.json

@@ -0,0 +1,22 @@
+{
+  "base_model_name_or_path": "/workspace/WizardLM-7b",
+  "bias": "all",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "lora_alpha": 64,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "out_proj",
+    "fc_in",
+    "fc_out",
+    "wte"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7e509a37773ddff40c0df2df2979f2da94c3e92374b6ef8903962172a871397
+size 25234701

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[PAD]": 32000
+}

finetune.ipynb

@@ -0,0 +1,1130 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3139c31d-2ca2-472e-8e29-c404032e4a55",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "#!git clone https://github.com/IlyaGusev/rulm.git\n",
+    "#!git clone https://github.com/ggerganov/llama.cpp.git\n",
+    "\n",
+    "self_instruct_dir = Path('rulm/self_instruct').resolve()\n",
+    "\n",
+    "#!cd {self_instruct_dir} && pip install -r ../requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6f04525b-5567-4335-98b2-7bb60d4be6ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import snapshot_download\n",
+    "content_dir = Path('.').resolve()\n",
+    "\n",
+    "model_dir = content_dir / \"WizardLM-7b\"\n",
+    "base_model = \"ehartford/WizardLM-7B-V1.0-Uncensored\" #@param {type:\"string\"}\n",
+    "#snapshot_download(repo_id=base_model, local_dir=model_dir, ignore_patterns=[\"LICENSE\", \"README.md\", \".gitattributes\"])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5321ea58-116a-4501-b6c3-3521ac5e6e7f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Overriding torch_dtype=None with `torch_dtype=torch.float16` due to requirements of `bitsandbytes` to enable model loading in mixed int8. Either pass torch_dtype=torch.float16 or don't pass this argument at all to remove this warning.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "================================================================================\n",
+      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\n",
+      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
+      "CUDA SETUP: Detected CUDA version 117\n",
+      "CUDA SETUP: Loading binary /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/nvidia/lib64'), PosixPath('/usr/local/nvidia/lib')}\n",
+      "  warn(msg)\n",
+      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/lib/x86_64-linux-gnu did not contain libcudart.so as expected! Searching further paths...\n",
+      "  warn(msg)\n",
+      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('yCIZccJum+9/ZLm7c7rDHLQYnn/OPUl4MSQ9SgINSks'), PosixPath('SHA256')}\n",
+      "  warn(msg)\n",
+      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('noninteractive    SHELL=/bin/bash')}\n",
+      "  warn(msg)\n",
+      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/workspace/Untitled.ipynb')}\n",
+      "  warn(msg)\n",
+      "/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//matplotlib_inline.backend_inline'), PosixPath('module')}\n",
+      "  warn(msg)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5861e63248924916a4e574e3b559d5f9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
+    "\n",
+    "model_id = \"/workspace/WizardLM-7b\"\n",
+    "bnb_config = BitsAndBytesConfig(\n",
+    "    load_in_8bit=True,\n",
+    "    bnb_8bit_use_double_quant=True,\n",
+    "    bnb_8bit_quant_type=\"nf4\",\n",
+    "    bnb_8bit_compute_dtype=torch.float32,\n",
+    "    llm_int8_enable_fp32_cpu_offload=True,\n",
+    "    load_in_8bit_fp32_cpu_offload=True\n",
+    ")\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
+    "model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map=\"auto\", max_memory={0:'8GiB', 'cpu':'50GiB'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6e575c22-66de-457c-82b2-da57db38ead8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from peft import prepare_model_for_int8_training\n",
+    "\n",
+    "model.gradient_checkpointing_enable()\n",
+    "model = prepare_model_for_int8_training(model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "16fd3d91-6468-4268-bc48-3b1c5bcf39cc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def print_trainable_parameters(model):\n",
+    "    \"\"\"\n",
+    "    Prints the number of trainable parameters in the model.\n",
+    "    \"\"\"\n",
+    "    trainable_params = 0\n",
+    "    all_param = 0\n",
+    "    for _, param in model.named_parameters():\n",
+    "        all_param += param.numel()\n",
+    "        if param.requires_grad:\n",
+    "            trainable_params += param.numel()\n",
+    "    print(\n",
+    "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "79566f41-ee3f-4662-bbcc-0c8ebc684b8d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "trainable params: 6291456 || all params: 6744707072 || trainable%: 0.09327989982127426\n"
+     ]
+    }
+   ],
+   "source": [
+    "from peft import LoraConfig, get_peft_model\n",
+    "\n",
+    "\n",
+    "config = LoraConfig(\n",
+    "    r=8,\n",
+    "    lora_alpha=64,\n",
+    "    target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"out_proj\", \"fc_in\", \"fc_out\", \"wte\"],\n",
+    "    lora_dropout=0.1,\n",
+    "    bias=\"all\",\n",
+    "    task_type=\"CAUSAL_LM\"\n",
+    ")\n",
+    "\n",
+    "model = get_peft_model(model, config)\n",
+    "print_trainable_parameters(model)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "68ca2e1a-eeca-41a0-a220-f27971309477",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Found cached dataset json (/root/.cache/huggingface/datasets/json/default-6d1f9620cbe5783f/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)\n",
+      "Loading cached processed dataset at /root/.cache/huggingface/datasets/json/default-6d1f9620cbe5783f/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-6b78e4d017496646.arrow\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "# Define a function to tokenize the dialogues\n",
+    "def tokenize_function(examples):\n",
+    "    return tokenizer(examples[\"dialogue\"], truncation=True, padding='max_length', max_length=128)\n",
+    "\n",
+    "# Load the new dataset\n",
+    "dataset = load_dataset('json', data_files='/workspace/new_output.jsonl', split='train')\n",
+    "\n",
+    "# Tokenize the dataset\n",
+    "dataset = dataset.map(tokenize_function, batched=True)\n",
+    "\n",
+    "# Split the dataset into training and evaluation sets\n",
+    "dataset = dataset.train_test_split(test_size=0.1)\n",
+    "\n",
+    "# Get the training and evaluation datasets\n",
+    "train_dataset = dataset['train']\n",
+    "eval_dataset = dataset['test']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ba14b1b-26ec-420c-bb41-6dacfaa5093a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mdj1150277539\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "Tracking run with wandb version 0.15.4"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Run data is saved locally in <code>/workspace/wandb/run-20230704_050553-7zgg0kn5</code>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Syncing run <strong><a href='https://wandb.ai/dj1150277539/huggingface/runs/7zgg0kn5' target=\"_blank\">distinctive-feather-33</a></strong> to <a href='https://wandb.ai/dj1150277539/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View project at <a href='https://wandb.ai/dj1150277539/huggingface' target=\"_blank\">https://wandb.ai/dj1150277539/huggingface</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View run at <a href='https://wandb.ai/dj1150277539/huggingface/runs/7zgg0kn5' target=\"_blank\">https://wandb.ai/dj1150277539/huggingface/runs/7zgg0kn5</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='292' max='300' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [292/300 13:37:21 < 22:32, 0.01 it/s, Epoch 166/300]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "      <th>Validation Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>3.336500</td>\n",
+       "      <td>14.149993</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>6.517100</td>\n",
+       "      <td>13.468672</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>9.206500</td>\n",
+       "      <td>12.650669</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>9.328000</td>\n",
+       "      <td>11.601873</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>6.599700</td>\n",
+       "      <td>10.404304</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>3.456600</td>\n",
+       "      <td>8.535113</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14</td>\n",
+       "      <td>0.919500</td>\n",
+       "      <td>6.562298</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16</td>\n",
+       "      <td>0.691900</td>\n",
+       "      <td>6.245927</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18</td>\n",
+       "      <td>1.991800</td>\n",
+       "      <td>5.460762</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>2.966800</td>\n",
+       "      <td>4.838455</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>22</td>\n",
+       "      <td>3.540500</td>\n",
+       "      <td>4.411865</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>24</td>\n",
+       "      <td>3.101700</td>\n",
+       "      <td>4.204891</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>26</td>\n",
+       "      <td>1.990200</td>\n",
+       "      <td>4.001011</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>28</td>\n",
+       "      <td>0.980100</td>\n",
+       "      <td>3.855425</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>0.058200</td>\n",
+       "      <td>3.714956</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>32</td>\n",
+       "      <td>0.800900</td>\n",
+       "      <td>3.611897</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>34</td>\n",
+       "      <td>1.613400</td>\n",
+       "      <td>3.506055</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>36</td>\n",
+       "      <td>2.387300</td>\n",
+       "      <td>3.416133</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>38</td>\n",
+       "      <td>2.598900</td>\n",
+       "      <td>3.339494</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>2.018800</td>\n",
+       "      <td>3.274912</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>42</td>\n",
+       "      <td>1.217900</td>\n",
+       "      <td>3.217013</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>44</td>\n",
+       "      <td>0.448600</td>\n",
+       "      <td>3.172593</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>46</td>\n",
+       "      <td>0.293700</td>\n",
+       "      <td>3.141093</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>48</td>\n",
+       "      <td>1.022200</td>\n",
+       "      <td>3.091443</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>50</td>\n",
+       "      <td>1.731100</td>\n",
+       "      <td>3.060865</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>52</td>\n",
+       "      <td>2.338500</td>\n",
+       "      <td>3.032907</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>54</td>\n",
+       "      <td>2.226600</td>\n",
+       "      <td>3.012452</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>56</td>\n",
+       "      <td>1.500500</td>\n",
+       "      <td>2.992773</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>58</td>\n",
+       "      <td>0.795400</td>\n",
+       "      <td>2.976812</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>60</td>\n",
+       "      <td>0.092900</td>\n",
+       "      <td>2.965430</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>62</td>\n",
+       "      <td>0.597500</td>\n",
+       "      <td>2.950096</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>64</td>\n",
+       "      <td>1.287900</td>\n",
+       "      <td>2.940890</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>66</td>\n",
+       "      <td>1.970100</td>\n",
+       "      <td>2.930588</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>68</td>\n",
+       "      <td>2.240600</td>\n",
+       "      <td>2.923621</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>70</td>\n",
+       "      <td>1.825200</td>\n",
+       "      <td>2.917476</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>72</td>\n",
+       "      <td>1.137100</td>\n",
+       "      <td>2.911416</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>74</td>\n",
+       "      <td>0.455000</td>\n",
+       "      <td>2.906709</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>76</td>\n",
+       "      <td>0.225000</td>\n",
+       "      <td>2.902960</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>78</td>\n",
+       "      <td>0.907000</td>\n",
+       "      <td>2.898351</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>80</td>\n",
+       "      <td>1.582700</td>\n",
+       "      <td>2.894657</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>82</td>\n",
+       "      <td>2.208000</td>\n",
+       "      <td>2.891902</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>84</td>\n",
+       "      <td>2.161300</td>\n",
+       "      <td>2.889836</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>86</td>\n",
+       "      <td>1.485300</td>\n",
+       "      <td>2.886881</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>88</td>\n",
+       "      <td>0.808700</td>\n",
+       "      <td>2.888359</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>90</td>\n",
+       "      <td>0.134600</td>\n",
+       "      <td>2.898380</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>92</td>\n",
+       "      <td>0.540100</td>\n",
+       "      <td>2.904707</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>94</td>\n",
+       "      <td>1.212500</td>\n",
+       "      <td>2.884937</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>96</td>\n",
+       "      <td>1.882300</td>\n",
+       "      <td>2.886901</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>98</td>\n",
+       "      <td>2.190800</td>\n",
+       "      <td>2.881487</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>100</td>\n",
+       "      <td>1.831900</td>\n",
+       "      <td>2.881773</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>102</td>\n",
+       "      <td>1.164300</td>\n",
+       "      <td>2.879683</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>104</td>\n",
+       "      <td>0.490800</td>\n",
+       "      <td>2.878468</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>106</td>\n",
+       "      <td>0.178100</td>\n",
+       "      <td>2.877084</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>108</td>\n",
+       "      <td>0.841800</td>\n",
+       "      <td>2.875591</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>110</td>\n",
+       "      <td>1.500900</td>\n",
+       "      <td>2.874607</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>112</td>\n",
+       "      <td>2.174100</td>\n",
+       "      <td>2.873750</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>114</td>\n",
+       "      <td>0.664800</td>\n",
+       "      <td>2.875177</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>116</td>\n",
+       "      <td>1.330800</td>\n",
+       "      <td>2.873417</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>118</td>\n",
+       "      <td>1.988000</td>\n",
+       "      <td>2.872393</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>120</td>\n",
+       "      <td>2.162500</td>\n",
+       "      <td>2.873486</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>122</td>\n",
+       "      <td>1.675900</td>\n",
+       "      <td>2.873488</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>124</td>\n",
+       "      <td>1.014600</td>\n",
+       "      <td>2.874382</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>126</td>\n",
+       "      <td>0.350600</td>\n",
+       "      <td>2.878257</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>128</td>\n",
+       "      <td>0.307900</td>\n",
+       "      <td>2.882156</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>130</td>\n",
+       "      <td>0.963200</td>\n",
+       "      <td>2.874237</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>132</td>\n",
+       "      <td>1.624300</td>\n",
+       "      <td>2.872886</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>134</td>\n",
+       "      <td>2.146800</td>\n",
+       "      <td>2.873395</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>136</td>\n",
+       "      <td>2.014700</td>\n",
+       "      <td>2.875467</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>138</td>\n",
+       "      <td>1.351300</td>\n",
+       "      <td>2.876896</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>140</td>\n",
+       "      <td>0.699800</td>\n",
+       "      <td>2.879110</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>142</td>\n",
+       "      <td>0.044100</td>\n",
+       "      <td>2.881222</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>144</td>\n",
+       "      <td>0.610900</td>\n",
+       "      <td>2.880388</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>146</td>\n",
+       "      <td>1.260600</td>\n",
+       "      <td>2.882942</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>148</td>\n",
+       "      <td>1.908200</td>\n",
+       "      <td>2.885541</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>150</td>\n",
+       "      <td>2.125100</td>\n",
+       "      <td>2.886549</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>152</td>\n",
+       "      <td>1.685200</td>\n",
+       "      <td>2.889215</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>154</td>\n",
+       "      <td>1.042300</td>\n",
+       "      <td>2.893012</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>156</td>\n",
+       "      <td>0.386300</td>\n",
+       "      <td>2.897541</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>158</td>\n",
+       "      <td>0.258700</td>\n",
+       "      <td>2.897153</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>160</td>\n",
+       "      <td>0.901400</td>\n",
+       "      <td>2.905147</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>162</td>\n",
+       "      <td>1.545900</td>\n",
+       "      <td>2.902808</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>164</td>\n",
+       "      <td>2.106000</td>\n",
+       "      <td>2.907741</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>166</td>\n",
+       "      <td>2.025800</td>\n",
+       "      <td>2.897594</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>168</td>\n",
+       "      <td>1.384000</td>\n",
+       "      <td>2.900589</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>170</td>\n",
+       "      <td>0.730600</td>\n",
+       "      <td>2.906770</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>172</td>\n",
+       "      <td>0.087300</td>\n",
+       "      <td>2.923102</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>174</td>\n",
+       "      <td>0.556200</td>\n",
+       "      <td>2.896429</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>176</td>\n",
+       "      <td>1.199200</td>\n",
+       "      <td>2.916603</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>178</td>\n",
+       "      <td>1.837500</td>\n",
+       "      <td>2.915208</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>180</td>\n",
+       "      <td>2.091800</td>\n",
+       "      <td>2.918897</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>182</td>\n",
+       "      <td>1.706300</td>\n",
+       "      <td>2.923554</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>184</td>\n",
+       "      <td>1.060900</td>\n",
+       "      <td>2.924343</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>186</td>\n",
+       "      <td>0.426200</td>\n",
+       "      <td>2.937875</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>188</td>\n",
+       "      <td>0.213300</td>\n",
+       "      <td>2.928014</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>190</td>\n",
+       "      <td>0.846600</td>\n",
+       "      <td>2.946512</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>192</td>\n",
+       "      <td>1.483100</td>\n",
+       "      <td>2.939318</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>194</td>\n",
+       "      <td>2.072100</td>\n",
+       "      <td>2.945714</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>196</td>\n",
+       "      <td>2.029300</td>\n",
+       "      <td>2.949480</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>198</td>\n",
+       "      <td>1.394200</td>\n",
+       "      <td>2.952962</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>200</td>\n",
+       "      <td>0.761500</td>\n",
+       "      <td>2.967859</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>202</td>\n",
+       "      <td>0.123500</td>\n",
+       "      <td>2.950326</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>204</td>\n",
+       "      <td>0.505700</td>\n",
+       "      <td>2.971651</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>206</td>\n",
+       "      <td>1.131400</td>\n",
+       "      <td>2.973131</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>208</td>\n",
+       "      <td>1.762000</td>\n",
+       "      <td>2.972790</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>210</td>\n",
+       "      <td>2.054900</td>\n",
+       "      <td>2.977710</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>212</td>\n",
+       "      <td>1.715500</td>\n",
+       "      <td>2.984576</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>214</td>\n",
+       "      <td>1.087700</td>\n",
+       "      <td>2.982883</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>216</td>\n",
+       "      <td>0.456900</td>\n",
+       "      <td>2.988919</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>218</td>\n",
+       "      <td>0.166900</td>\n",
+       "      <td>2.996212</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>220</td>\n",
+       "      <td>0.794400</td>\n",
+       "      <td>2.994239</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>222</td>\n",
+       "      <td>1.415700</td>\n",
+       "      <td>3.001529</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>224</td>\n",
+       "      <td>2.044000</td>\n",
+       "      <td>3.003330</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>226</td>\n",
+       "      <td>0.622300</td>\n",
+       "      <td>3.006619</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>228</td>\n",
+       "      <td>1.249300</td>\n",
+       "      <td>3.011294</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>230</td>\n",
+       "      <td>1.873000</td>\n",
+       "      <td>3.013264</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>232</td>\n",
+       "      <td>2.038300</td>\n",
+       "      <td>3.014204</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>234</td>\n",
+       "      <td>1.581300</td>\n",
+       "      <td>3.021334</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>236</td>\n",
+       "      <td>0.957400</td>\n",
+       "      <td>3.019286</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>238</td>\n",
+       "      <td>0.334200</td>\n",
+       "      <td>3.023535</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>240</td>\n",
+       "      <td>0.290700</td>\n",
+       "      <td>3.026750</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>242</td>\n",
+       "      <td>0.915000</td>\n",
+       "      <td>3.029947</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>244</td>\n",
+       "      <td>1.532900</td>\n",
+       "      <td>3.030478</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>246</td>\n",
+       "      <td>2.030900</td>\n",
+       "      <td>3.033974</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>248</td>\n",
+       "      <td>1.905700</td>\n",
+       "      <td>3.042156</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>250</td>\n",
+       "      <td>1.282600</td>\n",
+       "      <td>3.039005</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>252</td>\n",
+       "      <td>0.661500</td>\n",
+       "      <td>3.036432</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>254</td>\n",
+       "      <td>0.041400</td>\n",
+       "      <td>3.039858</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>256</td>\n",
+       "      <td>0.580900</td>\n",
+       "      <td>3.042423</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>258</td>\n",
+       "      <td>1.199400</td>\n",
+       "      <td>3.042602</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>260</td>\n",
+       "      <td>1.818700</td>\n",
+       "      <td>3.046433</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>262</td>\n",
+       "      <td>2.023600</td>\n",
+       "      <td>3.047827</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>264</td>\n",
+       "      <td>1.608300</td>\n",
+       "      <td>3.050818</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>266</td>\n",
+       "      <td>0.993300</td>\n",
+       "      <td>3.050040</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>268</td>\n",
+       "      <td>0.371800</td>\n",
+       "      <td>3.053066</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>270</td>\n",
+       "      <td>0.247700</td>\n",
+       "      <td>3.051264</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>272</td>\n",
+       "      <td>0.863300</td>\n",
+       "      <td>3.050924</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>274</td>\n",
+       "      <td>1.487000</td>\n",
+       "      <td>3.053752</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>276</td>\n",
+       "      <td>2.020500</td>\n",
+       "      <td>3.056072</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>278</td>\n",
+       "      <td>1.937400</td>\n",
+       "      <td>3.054851</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>280</td>\n",
+       "      <td>1.321400</td>\n",
+       "      <td>3.053991</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>282</td>\n",
+       "      <td>0.704000</td>\n",
+       "      <td>3.053047</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>284</td>\n",
+       "      <td>0.082900</td>\n",
+       "      <td>3.054484</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>286</td>\n",
+       "      <td>0.536800</td>\n",
+       "      <td>3.054063</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>288</td>\n",
+       "      <td>1.155600</td>\n",
+       "      <td>3.050694</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>290</td>\n",
+       "      <td>1.777500</td>\n",
+       "      <td>3.054609</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import transformers\n",
+    "\n",
+    "# needed for gpt-neo-x tokenizer\n",
+    "tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "trainer = transformers.Trainer(\n",
+    "    model=model,\n",
+    "    train_dataset=train_dataset,\n",
+    "    eval_dataset=eval_dataset,\n",
+    "    args=transformers.TrainingArguments(\n",
+    "        evaluation_strategy=\"steps\",\n",
+    "        per_device_train_batch_size=64,\n",
+    "        per_device_eval_batch_size=16,\n",
+    "        gradient_accumulation_steps=64,\n",
+    "        warmup_steps=30,\n",
+    "        save_steps=100,\n",
+    "        eval_steps=2,\n",
+    "        num_train_epochs=300,\n",
+    "        learning_rate=3e-4,\n",
+    "        save_total_limit=2,\n",
+    "        load_best_model_at_end= True,\n",
+    "        fp16=True,\n",
+    "        logging_steps=1,\n",
+    "        lr_scheduler_type=\"cosine\",\n",
+    "        output_dir=\"/workspace\",\n",
+    "        optim=\"adamw_torch\",\n",
+    "    ),\n",
+    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
+    ")\n",
+    "model.config.use_cache = False  # silence the warnings. Please re-enable for inference!\n",
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "c6be457c-c54b-4191-885a-475745c75293",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#saving the adapter model\n",
+    "pt_save_directory = '/workspace'\n",
+    "tokenizer.save_pretrained(pt_save_directory)\n",
+    "model.save_pretrained(pt_save_directory)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7df8097d-40f9-4524-8103-161dad3be122",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 2048,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}