{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "FuXIFTFapAMI",
    "outputId": "c8ced1ad-c7b3-44ba-807b-26d7d13906bc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# You only need to run this once per machine\n",
    "!pip install -q -U bitsandbytes\n",
    "!pip install -q -U git+https://github.com/huggingface/transformers.git\n",
    "!pip install -q -U git+https://github.com/huggingface/peft.git\n",
    "!pip install -q -U git+https://github.com/huggingface/accelerate.git\n",
    "!pip install -q -U datasets scipy ipywidgets matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "s6f4z8EYmcJ6"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bafa7b9bdc443b2a435c6e25d892591",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1503c94c1cbe4e85a211815b7c92ded1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd4676787ee74624947224ea650d7f2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23f255f1b96c4065a8be912a80b5d5c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a558441ea6b426ab9aea6cf24f01645",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a811622aff9643acb65104ce39a48470",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "train_dataset = load_dataset('json', data_files='train.jsonl', split='train')\n",
    "eval_dataset = load_dataset('json', data_files='eval.jsonl', split='train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def formatting_func(example):\n",
    "    text = f\"### Question: {example['input']}\\n ### Answer: {example['output']}\"\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "shz8Xdv-yRgf"
   },
   "source": [
    "### 2. Load Base Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MJ-5idQwzvg-"
   },
   "source": [
    "Let's now load Mistral - mistralai/Mistral-7B-v0.1 - using 4-bit quantization!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "45524c98039a46d5b7745ad7cb638d2f"
     ]
    },
    "id": "E0Nl5mWL0k2T",
    "outputId": "47b6b01d-e9f2-4b70-919c-17ae64993843"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5ad88adfba64ed0912303fca597f219",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/656 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60abbc37ed924b59877dce200ca99814",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin.index.json:   0%|          | 0.00/23.9k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b5c20ffeab24da0aee4ff17676736b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec47922c62884700be5eca78f335be81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00001-of-00008.bin:   0%|          | 0.00/1.89G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cfbce9e1f304a0bb16cfc2cad392e19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00002-of-00008.bin:   0%|          | 0.00/1.95G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cdff563355ce4a2097fd6eb0af42091d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00003-of-00008.bin:   0%|          | 0.00/1.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9885a9fad4a84b9086a34370f28e724b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00004-of-00008.bin:   0%|          | 0.00/1.95G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "747f4a2cc31b4f04a296b454dbc5f4da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00005-of-00008.bin:   0%|          | 0.00/1.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64a7be0fd297453fb310662e65e7c7d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00006-of-00008.bin:   0%|          | 0.00/1.95G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f1d43fd50da408db7757d1b361400c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00007-of-00008.bin:   0%|          | 0.00/1.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3a1498559d34e48b12c3559d5ab0f18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model-00008-of-00008.bin:   0%|          | 0.00/816M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2704dea5d39e481f8c8b7bd13e9dd162",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0612fa01d0e419a9b24e6968a3e75a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/111 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
    "\n",
    "base_model_id = \"filipealmeida/Mistral-7B-v0.1-sharded\"\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_use_double_quant=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(base_model_id, quantization_config=bnb_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UjNdXolqyRgf"
   },
   "source": [
    "### 3. Tokenization\n",
    "\n",
    "Set up the tokenizer. Add padding on the left as it [makes training use less memory](https://ai.stackexchange.com/questions/41485/while-fine-tuning-a-decoder-only-llm-like-llama-on-chat-dataset-what-kind-of-pa).\n",
    "\n",
    "\n",
    "For `model_max_length`, it's helpful to get a distribution of your data lengths. Let's first tokenize without the truncation/padding, so we can get a length distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "haSUDD9HyRgf",
    "outputId": "22ee95db-2974-4ab0-e0c7-444d04d3e838"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "367b8791c4744eeea7ef725cf76466d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/963 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ce90f43a75844d3aedc11d993a0e24a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86a0ed3ec36342e9ba3f0afbcadccedf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d05abb36d24a43a48134c1cd222906a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "added_tokens.json:   0%|          | 0.00/42.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fa2eadd3958428c96fb4d75b36b9b0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/72.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    base_model_id,\n",
    "    padding_side=\"left\",\n",
    "    add_eos_token=True,\n",
    "    add_bos_token=True,\n",
    ")\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "def generate_and_tokenize_prompt(prompt):\n",
    "    return tokenizer(formatting_func(prompt))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WHnKLcq4yRgg"
   },
   "source": [
    "Reformat the prompt and tokenize each sample:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "S3iLAwLh3m19"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3c21de787474cad94006c8b3a56a4da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/800 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7952a2e6c4c4eea8ae39952b984cdcb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/200 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_train_dataset = train_dataset.map(generate_and_tokenize_prompt)\n",
    "tokenized_val_dataset = eval_dataset.map(generate_and_tokenize_prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O6ewk27p3m19"
   },
   "source": [
    "Let's get a distribution of our dataset lengths, so we can determine the appropriate `max_length` for our input tensors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "BA8M9yfC3m19",
    "outputId": "99c6d302-9bb6-47b1-cae9-a1cd870b4770"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAAIjCAYAAAAJLyrXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPP0lEQVR4nO3deVhV1f7H8c9hRhBwAkQRTUnFIc2RtFGUlKyulsOPDE3zVljOmY1qmWVlaoM2ipVlWVlpqeFcpqam5hSOiQODNwPEm6Cwfn/0cG5HUNkIHNT363nOcztrrbP3dy+24ufuvdexGWOMAAAAAADF5uLsAgAAAADgUkOQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAJwxRs3bpxsNlu57Oumm27STTfdZH+/cuVK2Ww2ff755+Wy//79+6tu3brlsq+Sys7O1qBBgxQcHCybzaZhw4Y5u6RSV94/9wtZvHixWrRoIS8vL9lsNmVkZBQ5LiEhQTabTb///nu51lcWrBxL3bp11b9//zKvCcClhSAF4LJS8I+jgpeXl5dCQkIUHR2t6dOn68SJE6Wyn6NHj2rcuHHasmVLqWyvNFXk2orj+eefV0JCgh588EF9+OGH6tev3znH1q1bV7fddls5VmfNxx9/rKlTpzq7jPP6448/1KtXL3l7e+uNN97Qhx9+KB8fH2eXVSw7d+7UuHHjLotgB+DS4+bsAgCgLEyYMEH16tXT6dOnlZqaqpUrV2rYsGGaMmWKvvnmGzVv3tw+9sknn9Rjjz1maftHjx7V+PHjVbduXbVo0aLYn/v+++8t7ackzlfbO++8o/z8/DKv4WIsX75c7du31zPPPOPsUi7axx9/rO3bt1foq2obNmzQiRMn9OyzzyoqKuq8Y/v166c+ffrI09OznKo7v507d2r8+PG66aabLF9prWjHAuDSQ5ACcFnq2rWrWrdubX8/duxYLV++XLfddptuv/127dq1S97e3pIkNzc3ubmV7V+H//3vf1WpUiV5eHiU6X4uxN3d3an7L4709HRFREQ4u4wrRnp6uiQpICDggmNdXV3l6upaxhWVj8vpWAA4B7f2Abhi3HLLLXrqqad08OBBffTRR/b2op6RSkxMVMeOHRUQECBfX181bNhQjz/+uKS/n29p06aNJGnAgAH22wgTEhIk/f0cVNOmTbVp0ybdcMMNqlSpkv2zZz8jVSAvL0+PP/64goOD5ePjo9tvv12HDh1yGHOu5zT+uc0L1VbUM1InT57UyJEjFRoaKk9PTzVs2FAvv/yyjDEO42w2m4YMGaKvvvpKTZs2laenp5o0aaLFixcXPeFnSU9P18CBAxUUFCQvLy9dc801mj17tr2/4LmhAwcO6Ntvv7XXXhq3bX300Udq1aqVvL29VbVqVfXp06fQ/Bb83Hbu3Kmbb75ZlSpVUq1atTR58uRC2zt48KBuv/12+fj4KDAwUMOHD9eSJUtks9m0cuVK+/a+/fZbHTx40H4sZ899fn6+Jk6cqNq1a8vLy0udOnXS3r17Hcbs2bNHPXv2VHBwsLy8vFS7dm316dNHmZmZFzzuefPm2Y+7evXquueee3TkyBGHY46Li5MktWnTRjab7bzPAhX1XFHB7ZU//vij2rZtKy8vL1111VX64IMPivzs6tWr9e9//1vVqlWTn5+f7r33Xv35558OY202m8aNG1do///8M5CQkKC7775bknTzzTfb57hg/i+kqGMxxui5555T7dq1ValSJd18883asWNHoc+ePn1a48ePV3h4uLy8vFStWjV17NhRiYmJxdo3gMsDV6QAXFH69eunxx9/XN9//73uv//+Isfs2LFDt912m5o3b64JEybI09NTe/fu1Zo1ayRJjRs31oQJE/T0009r8ODBuv766yVJ1113nX0bf/zxh7p27ao+ffronnvuUVBQ0Hnrmjhxomw2m8aMGaP09HRNnTpVUVFR2rJli/3KWXEUp7Z/Msbo9ttv14oVKzRw4EC1aNFCS5Ys0ejRo3XkyBG9+uqrDuN//PFHffnll3rooYdUuXJlTZ8+XT179lRycrKqVat2zrr++usv3XTTTdq7d6+GDBmievXqad68eerfv78yMjI0dOhQNW7cWB9++KGGDx+u2rVra+TIkZKkGjVqFPv4izJx4kQ99dRT6tWrlwYNGqRjx47ptdde0w033KDNmzc7XIn5888/deutt6pHjx7q1auXPv/8c40ZM0bNmjVT165dJf0dPG+55RalpKRo6NChCg4O1scff6wVK1Y47PeJJ55QZmamDh8+bJ9HX19fhzEvvPCCXFxcNGrUKGVmZmry5MmKjY3V+vXrJUm5ubmKjo5WTk6OHn74YQUHB+vIkSNauHChMjIy5O/vf87jTkhI0IABA9SmTRtNmjRJaWlpmjZtmtasWWM/7ieeeEINGzbU22+/bb8dtn79+pbneO/evbrrrrs0cOBAxcXF6f3331f//v3VqlUrNWnSxGHskCFDFBAQoHHjxikpKUkzZszQwYMH7UG6uG644QY98sgjmj59uh5//HE1btxYkuz/WxJPP/20nnvuOXXr1k3dunXTL7/8oi5duig3N9dh3Lhx4zRp0iQNGjRIbdu2VVZWljZu3KhffvlFnTt3LvH+AVxiDABcRmbNmmUkmQ0bNpxzjL+/v2nZsqX9/TPPPGP++dfhq6++aiSZY8eOnXMbGzZsMJLMrFmzCvXdeOONRpKZOXNmkX033nij/f2KFSuMJFOrVi2TlZVlb//ss8+MJDNt2jR7W1hYmImLi7vgNs9XW1xcnAkLC7O//+qrr4wk89xzzzmMu+uuu4zNZjN79+61t0kyHh4eDm1bt241ksxrr71WaF//NHXqVCPJfPTRR/a23NxcExkZaXx9fR2OPSwszMTExJx3e8Ud+/vvvxtXV1czceJEh/Zt27YZNzc3h/aCn9sHH3xgb8vJyTHBwcGmZ8+e9rZXXnnFSDJfffWVve2vv/4yjRo1MpLMihUr7O0xMTEO812g4OfeuHFjk5OTY2+fNm2akWS2bdtmjDFm8+bNRpKZN2/ehSfjH3Jzc01gYKBp2rSp+euvv+ztCxcuNJLM008/bW8rzp+Zs8ceOHDA3hYWFmYkmdWrV9vb0tPTjaenpxk5cmShz7Zq1crk5uba2ydPnmwkma+//treJsk888wzhfZ/9p+BefPmFZrz4jr7WNLT042Hh4eJiYkx+fn59nGPP/64keSw32uuuabY5yiAyxe39gG44vj6+p539b6CKxRff/11iRdm8PT01IABA4o9/t5771XlypXt7++66y7VrFlT3333XYn2X1zfffedXF1d9cgjjzi0jxw5UsYYLVq0yKE9KirK4YpF8+bN5efnp/37919wP8HBwerbt6+9zd3dXY888oiys7O1atWqUjiawr788kvl5+erV69e+s9//mN/BQcHKzw8vNBVJF9fX91zzz329x4eHmrbtq3D8S1evFi1atXS7bffbm/z8vI65xXO8xkwYIDDc3MFVxAL9ldwxWnJkiX673//W+ztbty4Uenp6XrooYfk5eVlb4+JiVGjRo307bffWq71fCIiIuy1S39fRWzYsGGR58XgwYMdntV78MEH5ebmVubn+oUsXbpUubm5evjhhx2ujBW1UEhAQIB27NihPXv2lGOFACoaghSAK052drZDaDlb79691aFDBw0aNEhBQUHq06ePPvvsM0uhqlatWpYWlggPD3d4b7PZ1KBBgzJf1vngwYMKCQkpNB8Ft0cdPHjQob1OnTqFtlGlSpVCz7gUtZ/w8HC5uDj+2jnXfkrLnj17ZIxReHi4atSo4fDatWuXfaGFArVr1y50e9nZx3fw4EHVr1+/0LgGDRpYru/s+axSpYok2fdXr149jRgxQu+++66qV6+u6OhovfHGGxd8PqpgPhs2bFior1GjRqU+31bOi7PPdV9fX9WsWdPpS5gXzMnZ9dWoUcP+cykwYcIEZWRk6Oqrr1azZs00evRo/frrr+VWK4CKgSAF4Ipy+PBhZWZmnvcfvd7e3lq9erWWLl2qfv366ddff1Xv3r3VuXNn5eXlFWs/Vp5rKq5zPT9S3JpKw7lWOTNnLUxRUeTn58tms2nx4sVKTEws9Hrrrbccxpf38RVnf6+88op+/fVXPf744/rrr7/0yCOPqEmTJjp8+HCZ1FQS5TVv5Xmun88NN9ygffv26f3331fTpk317rvv6tprr9W7777r7NIAlCOCFIAryocffihJio6OPu84FxcXderUSVOmTNHOnTs1ceJELV++3H4rmJWH4ovj7FuEjDHau3evwypvVapUUUZGRqHPnn11wUptYWFhOnr0aKFbHX/77Td7f2kICwvTnj17Cl3VK+39nK1+/foyxqhevXqKiooq9Grfvr3lbYaFhWnfvn2FQsLZq+1JpXeeNGvWTE8++aRWr16tH374QUeOHNHMmTPPW6MkJSUlFepLSkoqs/kujrPP9ezsbKWkpFzwXM/NzVVKSopDW2n+OSyYk7PrO3bsWJFX1qpWraoBAwbok08+0aFDh9S8efMiVxoEcPkiSAG4YixfvlzPPvus6tWrp9jY2HOOO378eKG2gi+2zcnJkST5+PhIUpHBpiQ++OADhzDz+eefKyUlxb5SnPR3KFi3bp3DCmILFy4stIy3ldq6deumvLw8vf766w7tr776qmw2m8P+L0a3bt2UmpqqTz/91N525swZvfbaa/L19dWNN95YKvs5W48ePeTq6qrx48cXCj7GGP3xxx+WtxkdHa0jR47om2++sbedOnVK77zzTqGxPj4+xVqm/FyysrJ05swZh7ZmzZrJxcXFfi4WpXXr1goMDNTMmTMdxi1atEi7du1STExMiWu6WG+//bZOnz5tfz9jxgydOXOm0Lm+evXqQp87+4pUaf45jIqKkru7u1577TWHc2Xq1KmFxp593vj6+qpBgwbn/ZkAuPyw/DmAy9KiRYv022+/6cyZM0pLS9Py5cuVmJiosLAwffPNNw4P4J9twoQJWr16tWJiYhQWFqb09HS9+eabql27tjp27Cjp73/oBQQEaObMmapcubJ8fHzUrl071atXr0T1Vq1aVR07dtSAAQOUlpamqVOnqkGDBg4LGAwaNEiff/65br31VvXq1Uv79u3TRx99VGi5aiu1de/eXTfffLOeeOIJ/f7777rmmmv0/fff6+uvv9awYcNKtBR2UQYPHqy33npL/fv316ZNm1S3bl19/vnnWrNmjaZOnXreZ9YuZO/evXruuecKtbds2VIxMTF67rnnNHbsWP3++++68847VblyZR04cEDz58/X4MGDNWrUKEv7+/e//63XX39dffv21dChQ1WzZk3NmTPHfk798ypJq1at9Omnn2rEiBFq06aNfH191b1792Lva/ny5RoyZIjuvvtuXX311Tpz5ow+/PBDubq6qmfPnuf8nLu7u1588UUNGDBAN954o/r27Wtf/rxu3boaPny4pWMuTbm5uerUqZN69eqlpKQkvfnmm+rYsaPD4h2DBg3SAw88oJ49e6pz587aunWrlixZourVqztsq0WLFnJ1ddWLL76ozMxMeXp66pZbblFgYKDlumrUqKFRo0Zp0qRJuu2229StWzdt3rxZixYtKrTfiIgI3XTTTWrVqpWqVq2qjRs36vPPP9eQIUNKNikALk3OWSwQAMpGwZLGBS8PDw8THBxsOnfubKZNm+awzHaBs5c/X7ZsmbnjjjtMSEiI8fDwMCEhIaZv375m9+7dDp/7+uuvTUREhHFzc3NYbvzGG280TZo0KbK+cy1//sknn5ixY8eawMBA4+3tbWJiYszBgwcLff6VV14xtWrVMp6enqZDhw5m48aNhbZ5vtrOXv7cGGNOnDhhhg8fbkJCQoy7u7sJDw83L730ksMS0Mb8vSR1fHx8oZrOtSz72dLS0syAAQNM9erVjYeHh2nWrFmRS7RbXf78nz/vf74GDhxoH/fFF1+Yjh07Gh8fH+Pj42MaNWpk4uPjTVJSkn3MuX5uRc3Z/v37TUxMjPH29jY1atQwI0eONF988YWRZNatW2cfl52dbf7v//7PBAQEGEn27RT83M9e1vzAgQMOP6/9+/eb++67z9SvX994eXmZqlWrmptvvtksXbq0WPPz6aefmpYtWxpPT09TtWpVExsbaw4fPuwwpjSWPy/q53X2eVnw2VWrVpnBgwebKlWqGF9fXxMbG2v++OMPh8/m5eWZMWPGmOrVq5tKlSqZ6Ohos3fv3iLPtXfeecdcddVVxtXV1dJS6EUdS15enhk/frypWbOm8fb2NjfddJPZvn17of0+99xzpm3btiYgIMB4e3ubRo0amYkTJzos6w7g8mczpoI+IQwAwCVk6tSpGj58uA4fPqxatWo5u5wKp+ALgjds2KDWrVs7uxwAuGg8IwUAgEV//fWXw/tTp07prbfeUnh4OCEKAK4QPCMFAIBFPXr0UJ06ddSiRQtlZmbqo48+0m+//aY5c+Y4u7QrXnZ2trKzs887pkaNGudcsh0AiosgBQCARdHR0Xr33Xc1Z84c5eXlKSIiQnPnzlXv3r2dXdoV7+WXX9b48ePPO+bAgQMOy60DQEnwjBQAALhs7N+/X/v37z/vmI4dO5535U4AKA6CFAAAAABYxGITAAAAAGARz0hJys/P19GjR1W5cmWHL1IEAAAAcGUxxujEiRMKCQmRi8u5rzsRpCQdPXpUoaGhzi4DAAAAQAVx6NAh1a5d+5z9BClJlStXlvT3ZPn5+Tm5GgAAAADOkpWVpdDQUHtGOBeClGS/nc/Pz48gBQAAAOCCj/yw2AQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgkZuzC0Bh3bs7u4L/WbDA2RUAAAAAFQ9XpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsMjpQerIkSO65557VK1aNXl7e6tZs2bauHGjvd8Yo6efflo1a9aUt7e3oqKitGfPHodtHD9+XLGxsfLz81NAQIAGDhyo7Ozs8j4UAAAAAFcIpwapP//8Ux06dJC7u7sWLVqknTt36pVXXlGVKlXsYyZPnqzp06dr5syZWr9+vXx8fBQdHa1Tp07Zx8TGxmrHjh1KTEzUwoULtXr1ag0ePNgZhwQAAADgCmAzxhhn7fyxxx7TmjVr9MMPPxTZb4xRSEiIRo4cqVGjRkmSMjMzFRQUpISEBPXp00e7du1SRESENmzYoNatW0uSFi9erG7duunw4cMKCQkptN2cnBzl5OTY32dlZSk0NFSZmZny8/MrgyO1pnt3Z1fwPwsWOLsCAAAAoPxkZWXJ39//gtnAqVekvvnmG7Vu3Vp33323AgMD1bJlS73zzjv2/gMHDig1NVVRUVH2Nn9/f7Vr105r166VJK1du1YBAQH2ECVJUVFRcnFx0fr164vc76RJk+Tv729/hYaGltERAgAAALgcOTVI7d+/XzNmzFB4eLiWLFmiBx98UI888ohmz54tSUpNTZUkBQUFOXwuKCjI3peamqrAwECHfjc3N1WtWtU+5mxjx45VZmam/XXo0KHSPjQAAAAAlzE3Z+48Pz9frVu31vPPPy9JatmypbZv366ZM2cqLi6uzPbr6ekpT0/PMts+AAAAgMubU69I1axZUxEREQ5tjRs3VnJysiQpODhYkpSWluYwJi0tzd4XHBys9PR0h/4zZ87o+PHj9jEAAAAAUJqcGqQ6dOigpKQkh7bdu3crLCxMklSvXj0FBwdr2bJl9v6srCytX79ekZGRkqTIyEhlZGRo06ZN9jHLly9Xfn6+2rVrVw5HAQAAAOBK49Rb+4YPH67rrrtOzz//vHr16qWff/5Zb7/9tt5++21Jks1m07Bhw/Tcc88pPDxc9erV01NPPaWQkBDdeeedkv6+gnXrrbfq/vvv18yZM3X69GkNGTJEffr0KXLFPgAAAAC4WE4NUm3atNH8+fM1duxYTZgwQfXq1dPUqVMVGxtrH/Poo4/q5MmTGjx4sDIyMtSxY0ctXrxYXl5e9jFz5szRkCFD1KlTJ7m4uKhnz56aPn26Mw4JAAAAwBXAqd8jVVEUd6348sL3SAEAAADOcUl8jxQAAAAAXIoIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWOTUIDVu3DjZbDaHV6NGjez9p06dUnx8vKpVqyZfX1/17NlTaWlpDttITk5WTEyMKlWqpMDAQI0ePVpnzpwp70MBAAAAcAVxc3YBTZo00dKlS+3v3dz+V9Lw4cP17bffat68efL399eQIUPUo0cPrVmzRpKUl5enmJgYBQcH66efflJKSoruvfdeubu76/nnny/3YwEAAABwZXB6kHJzc1NwcHCh9szMTL333nv6+OOPdcstt0iSZs2apcaNG2vdunVq3769vv/+e+3cuVNLly5VUFCQWrRooWeffVZjxozRuHHj5OHhUd6HAwAAAOAK4PRnpPbs2aOQkBBdddVVio2NVXJysiRp06ZNOn36tKKiouxjGzVqpDp16mjt2rWSpLVr16pZs2YKCgqyj4mOjlZWVpZ27Nhxzn3m5OQoKyvL4QUAAAAAxeXUINWuXTslJCRo8eLFmjFjhg4cOKDrr79eJ06cUGpqqjw8PBQQEODwmaCgIKWmpkqSUlNTHUJUQX9B37lMmjRJ/v7+9ldoaGjpHhgAAACAy5pTb+3r2rWr/b+bN2+udu3aKSwsTJ999pm8vb3LbL9jx47ViBEj7O+zsrIIUwAAAACKzem39v1TQECArr76au3du1fBwcHKzc1VRkaGw5i0tDT7M1XBwcGFVvEreF/Uc1cFPD095efn5/ACAAAAgOKqUEEqOztb+/btU82aNdWqVSu5u7tr2bJl9v6kpCQlJycrMjJSkhQZGalt27YpPT3dPiYxMVF+fn6KiIgo9/oBAAAAXBmcemvfqFGj1L17d4WFheno0aN65pln5Orqqr59+8rf318DBw7UiBEjVLVqVfn5+enhhx9WZGSk2rdvL0nq0qWLIiIi1K9fP02ePFmpqal68sknFR8fL09PT2ceGgAAAIDLmFOD1OHDh9W3b1/98ccfqlGjhjp27Kh169apRo0akqRXX31VLi4u6tmzp3JychQdHa0333zT/nlXV1ctXLhQDz74oCIjI+Xj46O4uDhNmDDBWYcEAAAA4ApgM8YYZxfhbFlZWfL391dmZmaFeF6qe3dnV/A/CxY4uwIAAACg/BQ3G1SoZ6QAAAAA4FJAkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwKIKE6ReeOEF2Ww2DRs2zN526tQpxcfHq1q1avL19VXPnj2Vlpbm8Lnk5GTFxMSoUqVKCgwM1OjRo3XmzJlyrh4AAADAlaRCBKkNGzborbfeUvPmzR3ahw8frgULFmjevHlatWqVjh49qh49etj78/LyFBMTo9zcXP3000+aPXu2EhIS9PTTT5f3IQAAAAC4gjg9SGVnZys2NlbvvPOOqlSpYm/PzMzUe++9pylTpuiWW25Rq1atNGvWLP30009at26dJOn777/Xzp079dFHH6lFixbq2rWrnn32Wb3xxhvKzc111iEBAAAAuMw5PUjFx8crJiZGUVFRDu2bNm3S6dOnHdobNWqkOnXqaO3atZKktWvXqlmzZgoKCrKPiY6OVlZWlnbs2HHOfebk5CgrK8vhBQAAAADF5ebMnc+dO1e//PKLNmzYUKgvNTVVHh4eCggIcGgPCgpSamqqfcw/Q1RBf0HfuUyaNEnjx4+/yOoBAAAAXKmcdkXq0KFDGjp0qObMmSMvL69y3ffYsWOVmZlpfx06dKhc9w8AAADg0ua0ILVp0yalp6fr2muvlZubm9zc3LRq1SpNnz5dbm5uCgoKUm5urjIyMhw+l5aWpuDgYElScHBwoVX8Ct4XjCmKp6en/Pz8HF4AAAAAUFxOC1KdOnXStm3btGXLFvurdevWio2Ntf+3u7u7li1bZv9MUlKSkpOTFRkZKUmKjIzUtm3blJ6ebh+TmJgoPz8/RURElPsxAQAAALgyOO0ZqcqVK6tp06YObT4+PqpWrZq9feDAgRoxYoSqVq0qPz8/Pfzww4qMjFT79u0lSV26dFFERIT69eunyZMnKzU1VU8++aTi4+Pl6elZ7scEAAAA4Mrg1MUmLuTVV1+Vi4uLevbsqZycHEVHR+vNN9+097u6umrhwoV68MEHFRkZKR8fH8XFxWnChAlOrBoAAADA5c5mjDFWP7R//35dddVVZVGPU2RlZcnf31+ZmZkV4nmp7t2dXcH/LFjg7AoAAACA8lPcbFCiZ6QaNGigm2++WR999JFOnTpV4iIBAAAA4FJUoiD1yy+/qHnz5hoxYoSCg4P173//Wz///HNp1wYAAAAAFVKJglSLFi00bdo0HT16VO+//75SUlLUsWNHNW3aVFOmTNGxY8dKu04AAAAAqDAuavlzNzc39ejRQ/PmzdOLL76ovXv3atSoUQoNDdW9996rlJSU0qoTAAAAACqMiwpSGzdu1EMPPaSaNWtqypQpGjVqlPbt26fExEQdPXpUd9xxR2nVCQAAAAAVRomWP58yZYpmzZqlpKQkdevWTR988IG6desmF5e/c1m9evWUkJCgunXrlmatAAAAAFAhlChIzZgxQ/fdd5/69++vmjVrFjkmMDBQ77333kUVBwAAAAAVUYmC1J49ey44xsPDQ3FxcSXZPAAAAABUaCV6RmrWrFmaN29eofZ58+Zp9uzZF10UAAAAAFRkJQpSkyZNUvXq1Qu1BwYG6vnnn7/oogAAAACgIitRkEpOTla9evUKtYeFhSk5OfmiiwIAAACAiqxEQSowMFC//vprofatW7eqWrVqF10UAAAAAFRkJQpSffv21SOPPKIVK1YoLy9PeXl5Wr58uYYOHao+ffqUdo0AAAAAUKGUaNW+Z599Vr///rs6deokN7e/N5Gfn697772XZ6QAAAAAXPZKFKQ8PDz06aef6tlnn9XWrVvl7e2tZs2aKSwsrLTrAwAAAIAKp0RBqsDVV1+tq6++urRqAQAAAIBLQomCVF5enhISErRs2TKlp6crPz/foX/58uWlUhwAAAAAVEQlClJDhw5VQkKCYmJi1LRpU9lsttKuCwAAAAAqrBIFqblz5+qzzz5Tt27dSrseAAAAAKjwSrT8uYeHhxo0aFDatQAAAADAJaFEQWrkyJGaNm2ajDGlXQ8AAAAAVHglurXvxx9/1IoVK7Ro0SI1adJE7u7uDv1ffvllqRQHAAAAABVRiYJUQECA/vWvf5V2LQAAAABwSShRkJo1a1Zp1wEAAAAAl4wSPSMlSWfOnNHSpUv11ltv6cSJE5Kko0ePKjs7u9SKAwAAAICKqERXpA4ePKhbb71VycnJysnJUefOnVW5cmW9+OKLysnJ0cyZM0u7TgAAAACoMEp0RWro0KFq3bq1/vzzT3l7e9vb//Wvf2nZsmWlVhwAAAAAVEQluiL1ww8/6KeffpKHh4dDe926dXXkyJFSKQwAAAAAKqoSXZHKz89XXl5eofbDhw+rcuXKF10UAAAAAFRkJQpSXbp00dSpU+3vbTabsrOz9cwzz6hbt26lVRsAAAAAVEglurXvlVdeUXR0tCIiInTq1Cn93//9n/bs2aPq1avrk08+Ke0aAQAAAKBCKVGQql27trZu3aq5c+fq119/VXZ2tgYOHKjY2FiHxScAAAAA4HJUoiAlSW5ubrrnnntKsxYAAAAAuCSUKEh98MEH5+2/9957S1QMAAAAAFwKShSkhg4d6vD+9OnT+u9//ysPDw9VqlSJIAUAAADgslaiVfv+/PNPh1d2draSkpLUsWNHFpsAAAAAcNkrUZAqSnh4uF544YVCV6sAAAAA4HJTakFK+nsBiqNHj5bmJgEAAACgwinRM1LffPONw3tjjFJSUvT666+rQ4cOpVIYAAAAAFRUJQpSd955p8N7m82mGjVq6JZbbtErr7xSGnUBAAAAQIVVoiCVn59f2nUAAAAAwCWjVJ+RAgAAAIArQYmuSI0YMaLYY6dMmVKSXQAAAABAhVWiILV582Zt3rxZp0+fVsOGDSVJu3fvlqurq6699lr7OJvNVjpVAgAAAEAFUqIg1b17d1WuXFmzZ89WlSpVJP39Jb0DBgzQ9ddfr5EjR5ZqkQAAAABQkdiMMcbqh2rVqqXvv/9eTZo0cWjfvn27unTpcsl9l1RWVpb8/f2VmZkpPz8/Z5ej7t2dXcH/LFjg7AoAAACA8lPcbFCixSaysrJ07NixQu3Hjh3TiRMnSrJJAAAAALhklChI/etf/9KAAQP05Zdf6vDhwzp8+LC++OILDRw4UD169CjtGgEAAACgQinRM1IzZ87UqFGj9H//9386ffr03xtyc9PAgQP10ksvlWqBAAAAAFDRlOgZqQInT57Uvn37JEn169eXj49PqRVWnnhG6tx4RgoAAABXkjJ9RqpASkqKUlJSFB4eLh8fH11EJgMAAACAS0aJgtQff/yhTp066eqrr1a3bt2UkpIiSRo4cCBLnwMAAAC47JUoSA0fPlzu7u5KTk5WpUqV7O29e/fW4sWLS604AAAAAKiISrTYxPfff68lS5aodu3aDu3h4eE6ePBgqRQGAAAAABVVia5InTx50uFKVIHjx4/L09PzoosCAAAAgIqsREHq+uuv1wcffGB/b7PZlJ+fr8mTJ+vmm28uteIAAAAAoCIq0a19kydPVqdOnbRx40bl5ubq0Ucf1Y4dO3T8+HGtWbOmtGsEAAAAgAqlRFekmjZtqt27d6tjx4664447dPLkSfXo0UObN29W/fr1i72dGTNmqHnz5vLz85Ofn58iIyO1aNEie/+pU6cUHx+vatWqydfXVz179lRaWprDNpKTkxUTE6NKlSopMDBQo0eP1pkzZ0pyWAAAAABQLJavSJ0+fVq33nqrZs6cqSeeeOKidl67dm298MILCg8PlzFGs2fP1h133KHNmzerSZMmGj58uL799lvNmzdP/v7+GjJkiHr06GG/6pWXl6eYmBgFBwfrp59+UkpKiu699165u7vr+eefv6jaAAAAAOBcbKYE36Jbo0YN/fTTTwoPDy/1gqpWraqXXnpJd911l2rUqKGPP/5Yd911lyTpt99+U+PGjbV27Vq1b99eixYt0m233aajR48qKChIkjRz5kyNGTNGx44dk4eHR5H7yMnJUU5Ojv19VlaWQkNDL/jtxeWle3dnV/A/CxY4uwIAAACg/GRlZcnf3/+C2aBEt/bdc889eu+990pcXFHy8vI0d+5cnTx5UpGRkdq0aZNOnz6tqKgo+5hGjRqpTp06Wrt2rSRp7dq1atasmT1ESVJ0dLSysrK0Y8eOc+5r0qRJ8vf3t79CQ0NL9VgAAAAAXN5KtNjEmTNn9P7772vp0qVq1aqVfHx8HPqnTJlS7G1t27ZNkZGROnXqlHx9fTV//nxFRERoy5Yt8vDwUEBAgMP4oKAgpaamSpJSU1MdQlRBf0HfuYwdO1YjRoywvy+4IgUAAAAAxWEpSO3fv19169bV9u3bde2110qSdu/e7TDGZrNZKqBhw4basmWLMjMz9fnnnysuLk6rVq2ytA2rPD09+b4rAAAAACVmKUiFh4crJSVFK1askCT17t1b06dPL3RVyAoPDw81aNBAktSqVStt2LBB06ZNU+/evZWbm6uMjAyHq1JpaWkKDg6WJAUHB+vnn3922F7Bqn4FYwAAAACgtFl6RursdSkWLVqkkydPlmpB+fn5ysnJUatWreTu7q5ly5bZ+5KSkpScnKzIyEhJUmRkpLZt26b09HT7mMTERPn5+SkiIqJU6wIAAACAAiV6RqpACRb8czB27Fh17dpVderU0YkTJ/Txxx9r5cqVWrJkifz9/TVw4ECNGDFCVatWlZ+fnx5++GFFRkaqffv2kqQuXbooIiJC/fr10+TJk5Wamqonn3xS8fHx3LoHAAAAoMxYClI2m63QM1BWn4n6p/T0dN17771KSUmRv7+/mjdvriVLlqhz586SpFdffVUuLi7q2bOncnJyFB0drTfffNP+eVdXVy1cuFAPPvigIiMj5ePjo7i4OE2YMKHENQEAAADAhVj6HikXFxd17drVfrVnwYIFuuWWWwqt2vfll1+WbpVlrLhrxZcXvkcKAAAAcI7iZgNLV6Ti4uIc3t9zzz0lqw4AAAAALmGWgtSsWbPKqg4AAAAAuGRYWrUPAAAAAECQAgAAAADLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFbs4uABVb9+7OruB/FixwdgUAAADA37giBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwyKlBatKkSWrTpo0qV66swMBA3XnnnUpKSnIYc+rUKcXHx6tatWry9fVVz549lZaW5jAmOTlZMTExqlSpkgIDAzV69GidOXOmPA8FAAAAwBXEqUFq1apVio+P17p165SYmKjTp0+rS5cuOnnypH3M8OHDtWDBAs2bN0+rVq3S0aNH1aNHD3t/Xl6eYmJilJubq59++kmzZ89WQkKCnn76aWccEgAAAIArgM0YY5xdRIFjx44pMDBQq1at0g033KDMzEzVqFFDH3/8se666y5J0m+//abGjRtr7dq1at++vRYtWqTbbrtNR48eVVBQkCRp5syZGjNmjI4dOyYPD48L7jcrK0v+/v7KzMyUn59fmR5jcXTv7uwKKqYFC5xdAQAAAC53xc0GFeoZqczMTElS1apVJUmbNm3S6dOnFRUVZR/TqFEj1alTR2vXrpUkrV27Vs2aNbOHKEmKjo5WVlaWduzYUeR+cnJylJWV5fACAAAAgOKqMEEqPz9fw4YNU4cOHdS0aVNJUmpqqjw8PBQQEOAwNigoSKmpqfYx/wxRBf0FfUWZNGmS/P397a/Q0NBSPhoAAAAAl7MKE6Ti4+O1fft2zZ07t8z3NXbsWGVmZtpfhw4dKvN9AgAAALh8uDm7AEkaMmSIFi5cqNWrV6t27dr29uDgYOXm5iojI8PhqlRaWpqCg4PtY37++WeH7RWs6lcw5myenp7y9PQs5aMAAAAAcKVw6hUpY4yGDBmi+fPna/ny5apXr55Df6tWreTu7q5ly5bZ25KSkpScnKzIyEhJUmRkpLZt26b09HT7mMTERPn5+SkiIqJ8DgQAAADAFcWpV6Ti4+P18ccf6+uvv1blypXtzzT5+/vL29tb/v7+GjhwoEaMGKGqVavKz89PDz/8sCIjI9W+fXtJUpcuXRQREaF+/fpp8uTJSk1N1ZNPPqn4+HiuOgEAAAAoE04NUjNmzJAk3XTTTQ7ts2bNUv/+/SVJr776qlxcXNSzZ0/l5OQoOjpab775pn2sq6urFi5cqAcffFCRkZHy8fFRXFycJkyYUF6HAQAAAOAKU6G+R8pZ+B6pSwPfIwUAAICydkl+jxQAAAAAXAoIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWOTUILV69Wp1795dISEhstls+uqrrxz6jTF6+umnVbNmTXl7eysqKkp79uxxGHP8+HHFxsbKz89PAQEBGjhwoLKzs8vxKAAAAABcaZwapE6ePKlrrrlGb7zxRpH9kydP1vTp0zVz5kytX79ePj4+io6O1qlTp+xjYmNjtWPHDiUmJmrhwoVavXq1Bg8eXF6HAAAAAOAKZDPGGGcXIUk2m03z58/XnXfeKenvq1EhISEaOXKkRo0aJUnKzMxUUFCQEhIS1KdPH+3atUsRERHasGGDWrduLUlavHixunXrpsOHDyskJKRY+87KypK/v78yMzPl5+dXJsdnRffuzq6gYlqwwNkVAAAA4HJX3GxQYZ+ROnDggFJTUxUVFWVv8/f3V7t27bR27VpJ0tq1axUQEGAPUZIUFRUlFxcXrV+//pzbzsnJUVZWlsMLAAAAAIqrwgap1NRUSVJQUJBDe1BQkL0vNTVVgYGBDv1ubm6qWrWqfUxRJk2aJH9/f/srNDS0lKsHAAAAcDmrsEGqLI0dO1aZmZn216FDh5xdEgAAAIBLSIUNUsHBwZKktLQ0h/a0tDR7X3BwsNLT0x36z5w5o+PHj9vHFMXT01N+fn4OLwAAAAAorgobpOrVq6fg4GAtW7bM3paVlaX169crMjJSkhQZGamMjAxt2rTJPmb58uXKz89Xu3btyr1mAAAAAFcGN2fuPDs7W3v37rW/P3DggLZs2aKqVauqTp06GjZsmJ577jmFh4erXr16euqppxQSEmJf2a9x48a69dZbdf/992vmzJk6ffq0hgwZoj59+hR7xT4AAAAAsMqpQWrjxo26+eab7e9HjBghSYqLi1NCQoIeffRRnTx5UoMHD1ZGRoY6duyoxYsXy8vLy/6ZOXPmaMiQIerUqZNcXFzUs2dPTZ8+vdyPBQAAAMCVo8J8j5Qz8T1Slwa+RwoAAABl7ZL/HikAAAAAqKgIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIjdnFwAUV/fuzq7A0YIFzq4AAAAAzsIVKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGCRm7MLAC5V3bs7u4L/WbDA2RUAAABcWbgiBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAiN2cXUFreeOMNvfTSS0pNTdU111yj1157TW3btnV2WUC56N7d2RX8z4IFzq4AAACg7F0WQerTTz/ViBEjNHPmTLVr105Tp05VdHS0kpKSFBgY6OzyADhJRQqYFQ2BFwCAi2MzxhhnF3Gx2rVrpzZt2uj111+XJOXn5ys0NFQPP/ywHnvssQt+PisrS/7+/srMzJSfn19Zl3tB/OMPwJWEUHdpqEi/mzhnYBXnb9GYl6IVNxtc8lekcnNztWnTJo0dO9be5uLioqioKK1du7bIz+Tk5CgnJ8f+PjMzU9Lfk1YRnD7t7AoAoPzcequzK6iYPvvM2RU4qki/myrIr2tJUq9ezq7gfyraOVORcP4WjXkpWkEmuND1pks+SP3nP/9RXl6egoKCHNqDgoL022+/FfmZSZMmafz48YXaQ0NDy6RGAACs8vd3dgUVF3NTNObl0sDPqWgVcV5OnDgh//MUdskHqZIYO3asRowYYX+fn5+v48ePq1q1arLZbOVWR1ZWlkJDQ3Xo0KEKcUvh5Yg5LnvMcdljjssW81v2mOOyxxyXPea4bFWk+TXG6MSJEwoJCTnvuEs+SFWvXl2urq5KS0tzaE9LS1NwcHCRn/H09JSnp6dDW0BAQFmVeEF+fn5OP2Eud8xx2WOOyx5zXLaY37LHHJc95rjsMcdlq6LM7/muRBW45L9HysPDQ61atdKyZcvsbfn5+Vq2bJkiIyOdWBkAAACAy9Ulf0VKkkaMGKG4uDi1bt1abdu21dSpU3Xy5EkNGDDA2aUBAAAAuAxdFkGqd+/eOnbsmJ5++mmlpqaqRYsWWrx4caEFKCoaT09PPfPMM4VuM0TpYY7LHnNc9pjjssX8lj3muOwxx2WPOS5bl+L8XhbfIwUAAAAA5emSf0YKAAAAAMobQQoAAAAALCJIAQAAAIBFBCkAAAAAsIgg5URvvPGG6tatKy8vL7Vr104///yzs0uqkFavXq3u3bsrJCRENptNX331lUO/MUZPP/20atasKW9vb0VFRWnPnj0OY44fP67Y2Fj5+fkpICBAAwcOVHZ2tsOYX3/9Vddff728vLwUGhqqyZMnl/WhVQiTJk1SmzZtVLlyZQUGBurOO+9UUlKSw5hTp04pPj5e1apVk6+vr3r27FnoS7CTk5MVExOjSpUqKTAwUKNHj9aZM2ccxqxcuVLXXnutPD091aBBAyUkJJT14VUIM2bMUPPmze1fMhgZGalFixbZ+5nf0vfCCy/IZrNp2LBh9jbm+eKMGzdONpvN4dWoUSN7P/N78Y4cOaJ77rlH1apVk7e3t5o1a6aNGzfa+/l9d3Hq1q1b6By22WyKj4+XxDlcGvLy8vTUU0+pXr168vb2Vv369fXss8/qn2vbXVbnsYFTzJ0713h4eJj333/f7Nixw9x///0mICDApKWlObu0Cue7774zTzzxhPnyyy+NJDN//nyH/hdeeMH4+/ubr776ymzdutXcfvvtpl69euavv/6yj7n11lvNNddcY9atW2d++OEH06BBA9O3b197f2ZmpgkKCjKxsbFm+/bt5pNPPjHe3t7mrbfeKq/DdJro6Ggza9Yss337drNlyxbTrVs3U6dOHZOdnW0f88ADD5jQ0FCzbNkys3HjRtO+fXtz3XXX2fvPnDljmjZtaqKioszmzZvNd999Z6pXr27Gjh1rH7N//35TqVIlM2LECLNz507z2muvGVdXV7N48eJyPV5n+Oabb8y3335rdu/ebZKSkszjjz9u3N3dzfbt240xzG9p+/nnn03dunVN8+bNzdChQ+3tzPPFeeaZZ0yTJk1MSkqK/XXs2DF7P/N7cY4fP27CwsJM//79zfr1683+/fvNkiVLzN69e+1j+H13cdLT0x3O38TERCPJrFixwhjDOVwaJk6caKpVq2YWLlxoDhw4YObNm2d8fX3NtGnT7GMup/OYIOUkbdu2NfHx8fb3eXl5JiQkxEyaNMmJVVV8Zwep/Px8ExwcbF566SV7W0ZGhvH09DSffPKJMcaYnTt3Gklmw4YN9jGLFi0yNpvNHDlyxBhjzJtvvmmqVKlicnJy7GPGjBljGjZsWMZHVPGkp6cbSWbVqlXGmL/n093d3cybN88+ZteuXUaSWbt2rTHm77Dr4uJiUlNT7WNmzJhh/Pz87HP66KOPmiZNmjjsq3fv3iY6OrqsD6lCqlKlinn33XeZ31J24sQJEx4ebhITE82NN95oD1LM88V75plnzDXXXFNkH/N78caMGWM6dux4zn5+35W+oUOHmvr165v8/HzO4VISExNj7rvvPoe2Hj16mNjYWGPM5Xcec2ufE+Tm5mrTpk2Kioqyt7m4uCgqKkpr1651YmWXngMHDig1NdVhLv39/dWuXTv7XK5du1YBAQFq3bq1fUxUVJRcXFy0fv16+5gbbrhBHh4e9jHR0dFKSkrSn3/+WU5HUzFkZmZKkqpWrSpJ2rRpk06fPu0wx40aNVKdOnUc5rhZs2YOX4IdHR2trKws7dixwz7mn9soGHOlnfN5eXmaO3euTp48qcjISOa3lMXHxysmJqbQXDDPpWPPnj0KCQnRVVddpdjYWCUnJ0tifkvDN998o9atW+vuu+9WYGCgWrZsqXfeecfez++70pWbm6uPPvpI9913n2w2G+dwKbnuuuu0bNky7d69W5K0detW/fjjj+ratauky+88Jkg5wX/+8x/l5eU5/EGUpKCgIKWmpjqpqktTwXydby5TU1MVGBjo0O/m5qaqVas6jClqG//cx5UgPz9fw4YNU4cOHdS0aVNJfx+/h4eHAgICHMaePccXmr9zjcnKytJff/1VFodToWzbtk2+vr7y9PTUAw88oPnz5ysiIoL5LUVz587VL7/8okmTJhXqY54vXrt27ZSQkKDFixdrxowZOnDggK6//nqdOHGC+S0F+/fv14wZMxQeHq4lS5bowQcf1COPPKLZs2dL4vddafvqq6+UkZGh/v37S+LviNLy2GOPqU+fPmrUqJHc3d3VsmVLDRs2TLGxsZIuv/PYrdz2BKDCi4+P1/bt2/Xjjz86u5TLTsOGDbVlyxZlZmbq888/V1xcnFatWuXssi4bhw4d0tChQ5WYmCgvLy9nl3NZKvh/lCWpefPmateuncLCwvTZZ5/J29vbiZVdHvLz89W6dWs9//zzkqSWLVtq+/btmjlzpuLi4pxc3eXnvffeU9euXRUSEuLsUi4rn332mebMmaOPP/5YTZo00ZYtWzRs2DCFhIRclucxV6ScoHr16nJ1dS20EkxaWpqCg4OdVNWlqWC+zjeXwcHBSk9Pd+g/c+aMjh8/7jCmqG38cx+XuyFDhmjhwoVasWKFateubW8PDg5Wbm6uMjIyHMafPccXmr9zjfHz87si/hHm4eGhBg0aqFWrVpo0aZKuueYaTZs2jfktJZs2bVJ6erquvfZaubm5yc3NTatWrdL06dPl5uamoKAg5rmUBQQE6Oqrr9bevXs5j0tBzZo1FRER4dDWuHFj++2T/L4rPQcPHtTSpUs1aNAgexvncOkYPXq0/apUs2bN1K9fPw0fPtx+p8Dldh4TpJzAw8NDrVq10rJly+xt+fn5WrZsmSIjI51Y2aWnXr16Cg4OdpjLrKwsrV+/3j6XkZGRysjI0KZNm+xjli9frvz8fLVr184+ZvXq1Tp9+rR9TGJioho2bKgqVaqU09E4hzFGQ4YM0fz587V8+XLVq1fPob9Vq1Zyd3d3mOOkpCQlJyc7zPG2bdsc/uJLTEyUn5+f/R8GkZGRDtsoGHOlnvP5+fnKyclhfktJp06dtG3bNm3ZssX+at26tWJjY+3/zTyXruzsbO3bt081a9bkPC4FHTp0KPTVE7t371ZYWJgkft+VplmzZikwMFAxMTH2Ns7h0vHf//5XLi6O8cLV1VX5+fmSLsPzuFyXtoDd3Llzjaenp0lISDA7d+40gwcPNgEBAQ4rweBvJ06cMJs3bzabN282ksyUKVPM5s2bzcGDB40xfy+jGRAQYL7++mvz66+/mjvuuKPIZTRbtmxp1q9fb3788UcTHh7usIxmRkaGCQoKMv369TPbt283c+fONZUqVboiloN98MEHjb+/v1m5cqXDsrD//e9/7WMeeOABU6dOHbN8+XKzceNGExkZaSIjI+39BUvCdunSxWzZssUsXrzY1KhRo8glYUePHm127dpl3njjjStmSdjHHnvMrFq1yhw4cMD8+uuv5rHHHjM2m818//33xhjmt6z8c9U+Y5jnizVy5EizcuVKc+DAAbNmzRoTFRVlqlevbtLT040xzO/F+vnnn42bm5uZOHGi2bNnj5kzZ46pVKmS+eijj+xj+H138fLy8kydOnXMmDFjCvVxDl+8uLg4U6tWLfvy519++aWpXr26efTRR+1jLqfzmCDlRK+99pqpU6eO8fDwMG3btjXr1q1zdkkV0ooVK4ykQq+4uDhjzN9LaT711FMmKCjIeHp6mk6dOpmkpCSHbfzxxx+mb9++xtfX1/j5+ZkBAwaYEydOOIzZunWr6dixo/H09DS1atUyL7zwQnkdolMVNbeSzKxZs+xj/vrrL/PQQw+ZKlWqmEqVKpl//etfJiUlxWE7v//+u+natavx9vY21atXNyNHjjSnT592GLNixQrTokUL4+HhYa666iqHfVzO7rvvPhMWFmY8PDxMjRo1TKdOnewhyhjmt6ycHaSY54vTu3dvU7NmTePh4WFq1aplevfu7fAdR8zvxVuwYIFp2rSp8fT0NI0aNTJvv/22Qz+/7y7ekiVLjKRC82YM53BpyMrKMkOHDjV16tQxXl5e5qqrrjJPPPGEwzLll9N5bDPmH181DAAAAAC4IJ6RAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAVHj9+/fXnXfeWerbTU1NVefOneXj46OAgIBy3XdZqFu3rqZOnXreMTabTV999VW51AMAlzOCFABAUsUIDL///rtsNpu2bNlSLvt79dVXlZKSoi1btmj37t1Fjpk2bZoSEhLKpZ5/SkhIOGe4O5cNGzZo8ODBZVMQAMCBm7MLAADAWfbt26dWrVopPDz8nGP8/f3LsaKLU6NGDWeXAABXDK5IAQCKZfv27eratat8fX0VFBSkfv366T//+Y+9/6abbtIjjzyiRx99VFWrVlVwcLDGjRvnsI3ffvtNHTt2lJeXlyIiIrR06VKHW83q1asnSWrZsqVsNptuuukmh8+//PLLqlmzpqpVq6b4+HidPn36vDXPmDFD9evXl4eHhxo2bKgPP/zQ3le3bl198cUX+uCDD2Sz2dS/f/8it3H2lbriHKfNZtOMGTPUtWtXeXt766qrrtLnn39u71+5cqVsNpsyMjLsbVu2bJHNZtPvv/+ulStXasCAAcrMzJTNZpPNZiu0j6KcfWvfnj17dMMNN9jnOzEx0WF8bm6uhgwZopo1a8rLy0thYWGaNGnSBfcDACBIAQCKISMjQ7fccotatmypjRs3avHixUpLS1OvXr0cxs2ePVs+Pj5av369Jk+erAkTJtj/8Z6Xl6c777xTlSpV0vr16/X222/riSeecPj8zz//LElaunSpUlJS9OWXX9r7VqxYoX379mnFihWaPXu2EhISznvL3fz58zV06FCNHDlS27dv17///W8NGDBAK1askPT3bXC33nqrevXqpZSUFE2bNq3Y83G+4yzw1FNPqWfPntq6datiY2PVp08f7dq1q1jbv+666zR16lT5+fkpJSVFKSkpGjVqVLHrk6T8/Hz16NFDHh4eWr9+vWbOnKkxY8Y4jJk+fbq++eYbffbZZ0pKStKcOXNUt25dS/sBgCsVt/YBAC7o9ddfV8uWLfX888/b295//32FhoZq9+7duvrqqyVJzZs31zPPPCNJCg8P1+uvv65ly5apc+fOSkxM1L59+7Ry5UoFBwdLkiZOnKjOnTvbt1lwa1q1atXsYwpUqVJFr7/+ulxdXdWoUSPFxMRo2bJluv/++4us+eWXX1b//v310EMPSZJGjBihdevW6eWXX9bNN9+sGjVqyNPTU97e3oX2dSHnO84Cd999twYNGiRJevbZZ5WYmKjXXntNb7755gW37+HhIX9/f9lsNsu1FVi6dKl+++03LVmyRCEhIZKk559/Xl27drWPSU5OVnh4uDp27CibzaawsLAS7QsArkRckQIAXNDWrVu1YsUK+fr62l+NGjWS9PdzRgWaN2/u8LmaNWsqPT1dkpSUlKTQ0FCHYNC2bdti19CkSRO5uroWue2i7Nq1Sx06dHBo69ChQ7GvCp3P+Y6zQGRkZKH3pbHv4tq1a5dCQ0PtIaqomvr3768tW7aoYcOGeuSRR/T999+XW30AcKnjihQA4IKys7PVvXt3vfjii4X6atasaf9vd3d3hz6bzab8/PxSqaEst13etbi4/P3/Yxpj7G0Xet6rLFx77bU6cOCAFi1apKVLl6pXr16KiopyeJ4LAFA0rkgBAC7o2muv1Y4dO1S3bl01aNDA4eXj41OsbTRs2FCHDh1SWlqavW3Dhg0OYzw8PCT9/TzVxWrcuLHWrFnj0LZmzRpFRERc9LaLY926dYXeN27cWNL/bmFMSUmx95+95LuHh8dFzUPjxo116NAhh32cXZMk+fn5qXfv3nrnnXf06aef6osvvtDx48dLvF8AuFJwRQoAYJeZmVnoH/QFK+S988476tu3r321ur1792ru3Ll69913HW65O5fOnTurfv36iouL0+TJk3XixAk9+eSTkv6+oiNJgYGB8vb21uLFi1W7dm15eXmVePnx0aNHq1evXmrZsqWioqK0YMECffnll1q6dGmJtmfVvHnz1Lp1a3Xs2FFz5szRzz//rPfee0+S1KBBA4WGhmrcuHGaOHGidu/erVdeecXh83Xr1lV2draWLVuma665RpUqVVKlSpWKvf+oqChdffXViouL00svvaSsrKxCi3tMmTJFNWvWVMuWLeXi4qJ58+YpODjY8vdXAcCViCtSAAC7lStXqmXLlg6v8ePHKyQkRGvWrFFeXp66dOmiZs2aadiwYQoICLDfpnYhrq6u+uqrr5Sdna02bdpo0KBB9n/Ye3l5SZLc3Nw0ffp0vfXWWwoJCdEdd9xR4mO58847NW3aNL388stq0qSJ3nrrLc2aNavQkuplZfz48Zo7d66aN2+uDz74QJ988on9api7u7s++eQT/fbbb2revLlefPFFPffccw6fv+666/TAAw+od+/eqlGjhiZPnmxp/y4uLpo/f77++usvtW3bVoMGDdLEiRMdxlSuXFmTJ09W69at1aZNG/3+++/67rvviv0zBYArmc388wZtAADK0Zo1a9SxY0ft3btX9evXd3Y5pcZms2n+/PkO3z8FALi8cGsfAKDczJ8/X76+vgoPD9fevXs1dOhQdejQ4bIKUQCAKwNBCgBQbk6cOKExY8YoOTlZ1atXV1RUVKFng1C0H374weE7oM6WnZ1djtUAALi1DwCAS8Bff/2lI0eOnLO/QYMG5VgNAIAgBQAAAAAWsSwPAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAW/T9Lv+UGP7daqQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_data_lengths(tokenize_train_dataset, tokenized_val_dataset):\n",
    "    lengths = [len(x['input_ids']) for x in tokenized_train_dataset]\n",
    "    lengths += [len(x['input_ids']) for x in tokenized_val_dataset]\n",
    "    print(len(lengths))\n",
    "\n",
    "    # Plotting the histogram\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.hist(lengths, bins=20, alpha=0.7, color='blue')\n",
    "    plt.xlabel('Length of input_ids')\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.title('Distribution of Lengths of input_ids')\n",
    "    plt.show()\n",
    "\n",
    "plot_data_lengths(tokenized_train_dataset, tokenized_val_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nBk4Qp_vyRgh"
   },
   "source": [
    "From here, you can choose where you'd like to set the `max_length` to be. You can truncate and pad training examples to fit them to your chosen size. Be aware that choosing a larger `max_length` has its compute tradeoffs.\n",
    "\n",
    "I'm using my personal notes to train the model, and they vary greatly in length. I spent some time cleaning the dataset so the samples were about the same length, cutting up individual notes if needed, but being sure to not cut in the middle of a word or sentence."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bMlw8h743m19"
   },
   "source": [
    "Now let's tokenize again with padding and truncation, and set up the tokenize function to make labels and input_ids the same. This is basically what [self-supervised fine-tuning is](https://neptune.ai/blog/self-supervised-learning)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "BA8M9yfC3m19",
    "outputId": "99c6d302-9bb6-47b1-cae9-a1cd870b4770"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "981\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAAIjCAYAAAAJLyrXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHyUlEQVR4nO3deVxVdf7H8fcFZBcQlU0JTXFfc4vRTBNFJcrRGZexUkdzKizX8mebWpmTlakt2tQk2Z6VlU5auE+OmppmmpGYSyaIowFiCgjf3x89uNMVVA4i9wKv5+NxH3m+53vO+ZzLYXn3Ped7bcYYIwAAAABAqbk5uwAAAAAAqGwIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAqr0ZM2bIZrNVyLF69OihHj162JfXr18vm82mDz74oEKOP3LkSDVo0KBCjlVWOTk5GjNmjMLCwmSz2TRhwgRnl1TuKvrrfjmrVq1Su3bt5O3tLZvNpszMzBL7JSUlyWaz6dChQxVa39Vg5VwaNGigkSNHXvWaAFQuBCkAVUrRH0dFL29vb0VERCguLk4LFizQ6dOny+U4x44d04wZM7Rr165y2V95cuXaSuPJJ59UUlKS7r77br3xxhu6/fbbL9q3QYMGuvnmmyuwOmvefvttzZs3z9llXNLJkyc1ePBg+fj46MUXX9Qbb7whPz8/Z5dVKt99951mzJhRJYIdgMrHw9kFAMDV8Nhjj6lhw4bKz89Xenq61q9frwkTJmju3Ln69NNP1aZNG3vfhx9+WP/3f/9naf/Hjh3TzJkz1aBBA7Vr167U233xxReWjlMWl6rtlVdeUWFh4VWv4UqsXbtW119/vaZPn+7sUq7Y22+/rT179rj0qNq2bdt0+vRpPf7444qNjb1k39tvv11Dhw6Vl5dXBVV3ad99951mzpypHj16WB5pdbVzAVD5EKQAVEn9+vVTx44d7cvTpk3T2rVrdfPNN+uWW27Rvn375OPjI0ny8PCQh8fV/XH466+/ytfXV56enlf1OJdTo0YNpx6/NDIyMtSiRQtnl1FtZGRkSJKCgoIu29fd3V3u7u5XuaKKUZXOBYBzcGsfgGrjpptu0iOPPKLDhw/rzTfftLeX9IxUcnKyunXrpqCgIPn7+6tp06Z68MEHJf32fEunTp0kSaNGjbLfRpiUlCTpt+egWrVqpR07dqh79+7y9fW1b3vhM1JFCgoK9OCDDyosLEx+fn665ZZb9NNPPzn0udhzGr/f5+VqK+kZqTNnzmjy5MmKjIyUl5eXmjZtqmeeeUbGGId+NptN48aN08cff6xWrVrJy8tLLVu21KpVq0p+wy+QkZGh0aNHKzQ0VN7e3mrbtq1ef/11+/qi54YOHjyof/3rX/bay+O2rTfffFMdOnSQj4+PgoODNXTo0GLvb9HX7bvvvlPPnj3l6+urevXqac6cOcX2d/jwYd1yyy3y8/NTSEiIJk6cqM8//1w2m03r16+37+9f//qXDh8+bD+XC9/7wsJCzZo1S/Xr15e3t7d69eql1NRUhz779+/XoEGDFBYWJm9vb9WvX19Dhw5VVlbWZc976dKl9vOuU6eObrvtNv38888O5zxixAhJUqdOnWSz2S75LFBJzxUV3V755ZdfqnPnzvL29ta1116rJUuWlLjtxo0b9be//U21a9dWQECA7rjjDv3yyy8OfW02m2bMmFHs+L//HkhKStKf//xnSVLPnj3t73HR+385JZ2LMUZPPPGE6tevL19fX/Xs2VN79+4ttm1+fr5mzpyp6OhoeXt7q3bt2urWrZuSk5NLdWwAVQMjUgCqldtvv10PPvigvvjiC915550l9tm7d69uvvlmtWnTRo899pi8vLyUmpqqTZs2SZKaN2+uxx57TI8++qjGjh2rG264QZL0hz/8wb6PkydPql+/fho6dKhuu+02hYaGXrKuWbNmyWazaerUqcrIyNC8efMUGxurXbt22UfOSqM0tf2eMUa33HKL1q1bp9GjR6tdu3b6/PPPdf/99+vnn3/Wc88959D/yy+/1EcffaR77rlHNWvW1IIFCzRo0CAdOXJEtWvXvmhdZ8+eVY8ePZSamqpx48apYcOGWrp0qUaOHKnMzEyNHz9ezZs31xtvvKGJEyeqfv36mjx5siSpbt26pT7/ksyaNUuPPPKIBg8erDFjxujEiRN6/vnn1b17d+3cudNhJOaXX35R3759NXDgQA0ePFgffPCBpk6dqtatW6tfv36SfgueN910k9LS0jR+/HiFhYXp7bff1rp16xyO+9BDDykrK0tHjx61v4/+/v4Off7+97/Lzc1NU6ZMUVZWlubMmaPhw4dr69atkqS8vDzFxcUpNzdX9957r8LCwvTzzz9rxYoVyszMVGBg4EXPOykpSaNGjVKnTp00e/ZsHT9+XPPnz9emTZvs5/3QQw+padOm+sc//mG/HbZRo0aW3+PU1FT96U9/0ujRozVixAi99tprGjlypDp06KCWLVs69B03bpyCgoI0Y8YMpaSkaOHChTp8+LA9SJdW9+7ddd9992nBggV68MEH1bx5c0my/7csHn30UT3xxBPq37+/+vfvr6+//lp9+vRRXl6eQ78ZM2Zo9uzZGjNmjDp37qzs7Gxt375dX3/9tXr37l3m4wOoZAwAVCGLFy82ksy2bdsu2icwMNC0b9/evjx9+nTz+x+Hzz33nJFkTpw4cdF9bNu2zUgyixcvLrbuxhtvNJLMokWLSlx344032pfXrVtnJJl69eqZ7Oxse/v7779vJJn58+fb26KiosyIESMuu89L1TZixAgTFRVlX/7444+NJPPEE0849PvTn/5kbDabSU1NtbdJMp6eng5t33zzjZFknn/++WLH+r158+YZSebNN9+0t+Xl5ZmYmBjj7+/vcO5RUVEmPj7+kvsrbd9Dhw4Zd3d3M2vWLIf2b7/91nh4eDi0F33dlixZYm/Lzc01YWFhZtCgQfa2Z5991kgyH3/8sb3t7NmzplmzZkaSWbdunb09Pj7e4f0uUvR1b968ucnNzbW3z58/30gy3377rTHGmJ07dxpJZunSpZd/M34nLy/PhISEmFatWpmzZ8/a21esWGEkmUcffdTeVprvmQv7Hjx40N4WFRVlJJmNGzfa2zIyMoyXl5eZPHlysW07dOhg8vLy7O1z5swxkswnn3xib5Nkpk+fXuz4F34PLF26tNh7XloXnktGRobx9PQ08fHxprCw0N7vwQcfNJIcjtu2bdtSX6MAqi5u7QNQ7fj7+19y9r6iEYpPPvmkzBMzeHl5adSoUaXuf8cdd6hmzZr25T/96U8KDw/XZ599Vqbjl9Znn30md3d33XfffQ7tkydPljFGK1eudGiPjY11GLFo06aNAgIC9OOPP172OGFhYRo2bJi9rUaNGrrvvvuUk5OjDRs2lMPZFPfRRx+psLBQgwcP1n//+1/7KywsTNHR0cVGkfz9/XXbbbfZlz09PdW5c2eH81u1apXq1aunW265xd7m7e190RHOSxk1apTDc3NFI4hFxysacfr888/166+/lnq/27dvV0ZGhu655x55e3vb2+Pj49WsWTP961//slzrpbRo0cJeu/TbKGLTpk1LvC7Gjh3r8Kze3XffLQ8Pj6t+rV/O6tWrlZeXp3vvvddhZKykiUKCgoK0d+9e7d+/vwIrBOBqCFIAqp2cnByH0HKhIUOGqGvXrhozZoxCQ0M1dOhQvf/++5ZCVb169SxNLBEdHe2wbLPZ1Lhx46s+rfPhw4cVERFR7P0ouj3q8OHDDu3XXHNNsX3UqlWr2DMuJR0nOjpabm6Ov3Yudpzysn//fhljFB0drbp16zq89u3bZ59ooUj9+vWL3V524fkdPnxYjRo1KtavcePGluu78P2sVauWJNmP17BhQ02aNEmvvvqq6tSpo7i4OL344ouXfT6q6P1s2rRpsXXNmjUr9/fbynVx4bXu7++v8PBwp09hXvSeXFhf3bp17V+XIo899pgyMzPVpEkTtW7dWvfff792795dYbUCcA0EKQDVytGjR5WVlXXJP3p9fHy0ceNGrV69Wrfffrt2796tIUOGqHfv3iooKCjVcaw811RaF3t+pLQ1lYeLzXJmLpiYwlUUFhbKZrNp1apVSk5OLvZ6+eWXHfpX9PmV5njPPvusdu/erQcffFBnz57Vfffdp5YtW+ro0aNXpaayqKj3rSKv9Uvp3r27Dhw4oNdee02tWrXSq6++quuuu06vvvqqs0sDUIEIUgCqlTfeeEOSFBcXd8l+bm5u6tWrl+bOnavvvvtOs2bN0tq1a+23gll5KL40LrxFyBij1NRUh1neatWqpczMzGLbXji6YKW2qKgoHTt2rNitjt9//719fXmIiorS/v37i43qlfdxLtSoUSMZY9SwYUPFxsYWe11//fWW9xkVFaUDBw4UCwkXzrYnld910rp1az388MPauHGj/v3vf+vnn3/WokWLLlmjJKWkpBRbl5KSctXe79K48FrPyclRWlraZa/1vLw8paWlObSV5/dh0XtyYX0nTpwocWQtODhYo0aN0jvvvKOffvpJbdq0KXGmQQBVF0EKQLWxdu1aPf7442rYsKGGDx9+0X6nTp0q1lb0wba5ubmSJD8/P0kqMdiUxZIlSxzCzAcffKC0tDT7THHSb6Fgy5YtDjOIrVixotg03lZq69+/vwoKCvTCCy84tD/33HOy2WwOx78S/fv3V3p6ut577z172/nz5/X888/L399fN954Y7kc50IDBw6Uu7u7Zs6cWSz4GGN08uRJy/uMi4vTzz//rE8//dTedu7cOb3yyivF+vr5+ZVqmvKLyc7O1vnz5x3aWrduLTc3N/u1WJKOHTsqJCREixYtcui3cuVK7du3T/Hx8WWu6Ur94x//UH5+vn154cKFOn/+fLFrfePGjcW2u3BEqjy/D2NjY1WjRg09//zzDtfKvHnzivW98Lrx9/dX48aNL/k1AVD1MP05gCpp5cqV+v7773X+/HkdP35ca9euVXJysqKiovTpp586PIB/occee0wbN25UfHy8oqKilJGRoZdeekn169dXt27dJP32h15QUJAWLVqkmjVrys/PT126dFHDhg3LVG9wcLC6deumUaNG6fjx45o3b54aN27sMIHBmDFj9MEHH6hv374aPHiwDhw4oDfffLPYdNVWaktISFDPnj310EMP6dChQ2rbtq2++OILffLJJ5owYUKZpsIuydixY/Xyyy9r5MiR2rFjhxo0aKAPPvhAmzZt0rx58y75zNrlpKam6oknnijW3r59e8XHx+uJJ57QtGnTdOjQIQ0YMEA1a9bUwYMHtWzZMo0dO1ZTpkyxdLy//e1veuGFFzRs2DCNHz9e4eHheuutt+zX1O9HSTp06KD33ntPkyZNUqdOneTv76+EhIRSH2vt2rUaN26c/vznP6tJkyY6f/683njjDbm7u2vQoEEX3a5GjRp66qmnNGrUKN14440aNmyYffrzBg0aaOLEiZbOuTzl5eWpV69eGjx4sFJSUvTSSy+pW7duDpN3jBkzRnfddZcGDRqk3r1765tvvtHnn3+uOnXqOOyrXbt2cnd311NPPaWsrCx5eXnppptuUkhIiOW66tatqylTpmj27Nm6+eab1b9/f+3cuVMrV64sdtwWLVqoR48e6tChg4KDg7V9+3Z98MEHGjduXNneFACVk3MmCwSAq6NoSuOil6enpwkLCzO9e/c28+fPd5hmu8iF05+vWbPG3HrrrSYiIsJ4enqaiIgIM2zYMPPDDz84bPfJJ5+YFi1aGA8PD4fpxm+88UbTsmXLEuu72PTn77zzjpk2bZoJCQkxPj4+Jj4+3hw+fLjY9s8++6ypV6+e8fLyMl27djXbt28vts9L1Xbh9OfGGHP69GkzceJEExERYWrUqGGio6PN008/7TAFtDG/TUmdmJhYrKaLTct+oePHj5tRo0aZOnXqGE9PT9O6desSp2i3Ov3577/ev3+NHj3a3u/DDz803bp1M35+fsbPz880a9bMJCYmmpSUFHufi33dSnrPfvzxRxMfH298fHxM3bp1zeTJk82HH35oJJktW7bY++Xk5Ji//OUvJigoyEiy76fo637htOYHDx50+Hr9+OOP5q9//atp1KiR8fb2NsHBwaZnz55m9erVpXp/3nvvPdO+fXvj5eVlgoODzfDhw83Ro0cd+pTH9Oclfb0uvC6Ltt2wYYMZO3asqVWrlvH39zfDhw83J0+edNi2oKDATJ061dSpU8f4+vqauLg4k5qaWuK19sorr5hrr73WuLu7W5oKvaRzKSgoMDNnzjTh4eHGx8fH9OjRw+zZs6fYcZ944gnTuXNnExQUZHx8fEyzZs3MrFmzHKZ1B1D12Yxx0SeEAQCoRObNm6eJEyfq6NGjqlevnrPLcTlFHxC8bds2dezY0dnlAMAV4xkpAAAsOnv2rMPyuXPn9PLLLys6OpoQBQDVBM9IAQBg0cCBA3XNNdeoXbt2ysrK0ptvvqnvv/9eb731lrNLq/ZycnKUk5NzyT5169a96JTtAFBaBCkAACyKi4vTq6++qrfeeksFBQVq0aKF3n33XQ0ZMsTZpVV7zzzzjGbOnHnJPgcPHnSYbh0AyoJnpAAAQJXx448/6scff7xkn27dul1y5k4AKA2CFAAAAABYxGQTAAAAAGARz0hJKiws1LFjx1SzZk2HD1IEAAAAUL0YY3T69GlFRETIze3i404EKUnHjh1TZGSks8sAAAAA4CJ++ukn1a9f/6LrCVKSatasKem3NysgIMDJ1QAAAABwluzsbEVGRtozwsUQpCT77XwBAQEEKQAAAACXfeSHySYAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAizycXQBcW0KCsyv4n+XLnV0BAAAA8BtGpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAipwap2bNnq1OnTqpZs6ZCQkI0YMAApaSkOPTp0aOHbDabw+uuu+5y6HPkyBHFx8fL19dXISEhuv/++3X+/PmKPBUAAAAA1YiHMw++YcMGJSYmqlOnTjp//rwefPBB9enTR9999538/Pzs/e6880499thj9mVfX1/7vwsKChQfH6+wsDD95z//UVpamu644w7VqFFDTz75ZIWeDwAAAIDqwalBatWqVQ7LSUlJCgkJ0Y4dO9S9e3d7u6+vr8LCwkrcxxdffKHvvvtOq1evVmhoqNq1a6fHH39cU6dO1YwZM+Tp6Vlsm9zcXOXm5tqXs7Ozy+mMAAAAAFQHLvWMVFZWliQpODjYof2tt95SnTp11KpVK02bNk2//vqrfd3mzZvVunVrhYaG2tvi4uKUnZ2tvXv3lnic2bNnKzAw0P6KjIy8CmcDAAAAoKpy6ojU7xUWFmrChAnq2rWrWrVqZW//y1/+oqioKEVERGj37t2aOnWqUlJS9NFHH0mS0tPTHUKUJPtyenp6iceaNm2aJk2aZF/Ozs4mTAEAAAAoNZcJUomJidqzZ4++/PJLh/axY8fa/926dWuFh4erV69eOnDggBo1alSmY3l5ecnLy+uK6gUAAABQfbnErX3jxo3TihUrtG7dOtWvX/+Sfbt06SJJSk1NlSSFhYXp+PHjDn2Kli/2XBUAAAAAXAmnBiljjMaNG6dly5Zp7dq1atiw4WW32bVrlyQpPDxckhQTE6Nvv/1WGRkZ9j7JyckKCAhQixYtrkrdAAAAAKo3p97al5iYqLfffluffPKJatasaX+mKTAwUD4+Pjpw4IDefvtt9e/fX7Vr19bu3bs1ceJEde/eXW3atJEk9enTRy1atNDtt9+uOXPmKD09XQ8//LASExO5fQ8AAADAVeHUEamFCxcqKytLPXr0UHh4uP313nvvSZI8PT21evVq9enTR82aNdPkyZM1aNAgLV++3L4Pd3d3rVixQu7u7oqJidFtt92mO+64w+FzpwAAAACgPDl1RMoYc8n1kZGR2rBhw2X3ExUVpc8++6y8ygIAAACAS3KJySYAAAAAoDIhSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgkVOD1OzZs9WpUyfVrFlTISEhGjBggFJSUhz6nDt3TomJiapdu7b8/f01aNAgHT9+3KHPkSNHFB8fL19fX4WEhOj+++/X+fPnK/JUAAAAAFQjTg1SGzZsUGJiorZs2aLk5GTl5+erT58+OnPmjL3PxIkTtXz5ci1dulQbNmzQsWPHNHDgQPv6goICxcfHKy8vT//5z3/0+uuvKykpSY8++qgzTgkAAABANWAzxhhnF1HkxIkTCgkJ0YYNG9S9e3dlZWWpbt26evvtt/WnP/1JkvT999+refPm2rx5s66//nqtXLlSN998s44dO6bQ0FBJ0qJFizR16lSdOHFCnp6elz1udna2AgMDlZWVpYCAgKt6jpVNQoKzK/if5cudXQEAAACqutJmA5d6RiorK0uSFBwcLEnasWOH8vPzFRsba+/TrFkzXXPNNdq8ebMkafPmzWrdurU9RElSXFycsrOztXfv3hKPk5ubq+zsbIcXAAAAAJSWywSpwsJCTZgwQV27dlWrVq0kSenp6fL09FRQUJBD39DQUKWnp9v7/D5EFa0vWleS2bNnKzAw0P6KjIws57MBAAAAUJW5TJBKTEzUnj179O677171Y02bNk1ZWVn2108//XTVjwkAAACg6vBwdgGSNG7cOK1YsUIbN25U/fr17e1hYWHKy8tTZmamw6jU8ePHFRYWZu/z1VdfOeyvaFa/oj4X8vLykpeXVzmfBQAAAIDqwqkjUsYYjRs3TsuWLdPatWvVsGFDh/UdOnRQjRo1tGbNGntbSkqKjhw5opiYGElSTEyMvv32W2VkZNj7JCcnKyAgQC1atKiYEwEAAABQrTh1RCoxMVFvv/22PvnkE9WsWdP+TFNgYKB8fHwUGBio0aNHa9KkSQoODlZAQIDuvfdexcTE6Prrr5ck9enTRy1atNDtt9+uOXPmKD09XQ8//LASExMZdQIAAABwVTg1SC1cuFCS1KNHD4f2xYsXa+TIkZKk5557Tm5ubho0aJByc3MVFxenl156yd7X3d1dK1as0N13362YmBj5+flpxIgReuyxxyrqNAAAAABUMy71OVLOwudIXRyfIwUAAIDqpFJ+jhQAAAAAVAYEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYJGHswsASishwdkVOFq+3NkVAAAAwFkYkQIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWFSmIPXjjz+Wdx0AAAAAUGmUKUg1btxYPXv21Jtvvqlz586Vd00AAAAA4NLKFKS+/vprtWnTRpMmTVJYWJj+9re/6auvvirv2gAAAADAJZUpSLVr107z58/XsWPH9NprryktLU3dunVTq1atNHfuXJ04caK86wQAAAAAl3FFk014eHho4MCBWrp0qZ566imlpqZqypQpioyM1B133KG0tLTyqhMAAAAAXMYVBant27frnnvuUXh4uObOnaspU6bowIEDSk5O1rFjx3TrrbeWV50AAAAA4DI8yrLR3LlztXjxYqWkpKh///5asmSJ+vfvLze333JZw4YNlZSUpAYNGpRnrQAAAADgEsoUpBYuXKi//vWvGjlypMLDw0vsExISon/+859XVBwAAAAAuKIyBan9+/dfto+np6dGjBhRlt0DAAAAgEsr0zNSixcv1tKlS4u1L126VK+//nqp97Nx40YlJCQoIiJCNptNH3/8scP6kSNHymazObz69u3r0OfUqVMaPny4AgICFBQUpNGjRysnJ6cspwUAAAAApVKmIDV79mzVqVOnWHtISIiefPLJUu/nzJkzatu2rV588cWL9unbt6/S0tLsr3feecdh/fDhw7V3714lJydrxYoV2rhxo8aOHVv6kwEAAAAAi8p0a9+RI0fUsGHDYu1RUVE6cuRIqffTr18/9evX75J9vLy8FBYWVuK6ffv2adWqVdq2bZs6duwoSXr++efVv39/PfPMM4qIiCh1LQAAAABQWmUakQoJCdHu3buLtX/zzTeqXbv2FRf1e+vXr1dISIiaNm2qu+++WydPnrSv27x5s4KCguwhSpJiY2Pl5uamrVu3XnSfubm5ys7OdngBAAAAQGmVKUgNGzZM9913n9atW6eCggIVFBRo7dq1Gj9+vIYOHVpuxfXt21dLlizRmjVr9NRTT2nDhg3q16+fCgoKJEnp6ekKCQlx2MbDw0PBwcFKT0+/6H5nz56twMBA+ysyMrLcagYAAABQ9ZXp1r7HH39chw4dUq9eveTh8dsuCgsLdccdd1h6Rupyfh/KWrdurTZt2qhRo0Zav369evXqVeb9Tps2TZMmTbIvZ2dnE6YAAAAAlFqZgpSnp6fee+89Pf744/rmm2/k4+Oj1q1bKyoqqrzrc3DttdeqTp06Sk1NVa9evRQWFqaMjAyHPufPn9epU6cu+lyV9NtzV15eXle1VgAAAABVV5mCVJEmTZqoSZMm5VXLZR09elQnT560fwhwTEyMMjMztWPHDnXo0EGStHbtWhUWFqpLly4VVhcAAACA6qVMQaqgoEBJSUlas2aNMjIyVFhY6LB+7dq1pdpPTk6OUlNT7csHDx7Url27FBwcrODgYM2cOVODBg1SWFiYDhw4oAceeECNGzdWXFycJKl58+bq27ev7rzzTi1atEj5+fkaN26chg4dyox9AAAAAK6aMgWp8ePHKykpSfHx8WrVqpVsNluZDr59+3b17NnTvlz03NKIESO0cOFC7d69W6+//royMzMVERGhPn366PHHH3e4Le+tt97SuHHj1KtXL7m5uWnQoEFasGBBmeoBAAAAgNKwGWOM1Y3q1KmjJUuWqH///lejpgqXnZ2twMBAZWVlKSAgwNnluJSEBGdX4LqWL3d2BQAAAChvpc0GZZr+3NPTU40bNy5zcQAAAABQmZUpSE2ePFnz589XGQazAAAAAKDSK9MzUl9++aXWrVunlStXqmXLlqpRo4bD+o8++qhcigMAAAAAV1SmIBUUFKQ//vGP5V0LAAAAAFQKZQpSixcvLu86AAAAAKDSKNMzUpJ0/vx5rV69Wi+//LJOnz4tSTp27JhycnLKrTgAAAAAcEVlGpE6fPiw+vbtqyNHjig3N1e9e/dWzZo19dRTTyk3N1eLFi0q7zoBAAAAwGWUaURq/Pjx6tixo3755Rf5+PjY2//4xz9qzZo15VYcAAAAALiiMo1I/fvf/9Z//vMfeXp6OrQ3aNBAP//8c7kUBgAAAACuqkwjUoWFhSooKCjWfvToUdWsWfOKiwIAAAAAV1amINWnTx/NmzfPvmyz2ZSTk6Pp06erf//+5VUbAAAAALikMt3a9+yzzyouLk4tWrTQuXPn9Je//EX79+9XnTp19M4775R3jQAAAADgUsoUpOrXr69vvvlG7777rnbv3q2cnByNHj1aw4cPd5h8AgAAAACqojIFKUny8PDQbbfdVp61AAAAAEClUKYgtWTJkkuuv+OOO8pUDAAAAABUBmUKUuPHj3dYzs/P16+//ipPT0/5+voSpAAAAABUaWUKUr/88kuxtv379+vuu+/W/ffff8VFAZVBQoKzK/if5cudXQEAAED1Uqbpz0sSHR2tv//978VGqwAAAACgqim3ICX9NgHFsWPHynOXAAAAAOByynRr36effuqwbIxRWlqaXnjhBXXt2rVcCgMAAAAAV1WmIDVgwACHZZvNprp16+qmm27Ss88+Wx51AQAAAIDLKlOQKiwsLO86AAAAAKDSKNdnpAAAAACgOijTiNSkSZNK3Xfu3LllOQQAAAAAuKwyBamdO3dq586dys/PV9OmTSVJP/zwg9zd3XXdddfZ+9lstvKpEgAAAABcSJmCVEJCgmrWrKnXX39dtWrVkvTbh/SOGjVKN9xwgyZPnlyuRQIAAACAK7EZY4zVjerVq6cvvvhCLVu2dGjfs2eP+vTpU+k+Syo7O1uBgYHKyspSQECAs8txKQkJzq4ApbF8ubMrAAAAqBpKmw3KNNlEdna2Tpw4Uaz9xIkTOn36dFl2CQAAAACVRpmC1B//+EeNGjVKH330kY4ePaqjR4/qww8/1OjRozVw4MDyrhEAAAAAXEqZnpFatGiRpkyZor/85S/Kz8//bUceHho9erSefvrpci0QAAAAAFxNmZ6RKnLmzBkdOHBAktSoUSP5+fmVW2EViWekLo5npCoHnpECAAAoH1f1GakiaWlpSktLU3R0tPz8/HQFmQwAAAAAKo0yBamTJ0+qV69eatKkifr376+0tDRJ0ujRo5n6HAAAAECVV6YgNXHiRNWoUUNHjhyRr6+vvX3IkCFatWpVuRUHAAAAAK6oTJNNfPHFF/r8889Vv359h/bo6GgdPny4XAoDAAAAAFdVphGpM2fOOIxEFTl16pS8vLyuuCgAAAAAcGVlClI33HCDlixZYl+22WwqLCzUnDlz1LNnz3IrDgAAAABcUZlu7ZszZ4569eql7du3Ky8vTw888ID27t2rU6dOadOmTeVdIwAAAAC4lDKNSLVq1Uo//PCDunXrpltvvVVnzpzRwIEDtXPnTjVq1Ki8awQAAAAAl2J5RCo/P199+/bVokWL9NBDD12NmgAAAADApVkekapRo4Z27959NWoBAAAAgEqhTLf23XbbbfrnP/9Z3rUAAAAAQKVQpskmzp8/r9dee02rV69Whw4d5Ofn57B+7ty55VIcAAAAALgiS0Hqxx9/VIMGDbRnzx5dd911kqQffvjBoY/NZiu/6gAAAADABVkKUtHR0UpLS9O6deskSUOGDNGCBQsUGhp6VYoDAAAAAFdk6RkpY4zD8sqVK3XmzJlyLQgAAAAAXF2ZnpEqcmGwQvlISHB2BQAAAAAuxdKIlM1mK/YMFM9EAQAAAKhuLI1IGWM0cuRIeXl5SZLOnTunu+66q9isfR999FH5VQgAAAAALsZSkBoxYoTD8m233VauxQAAAABAZWApSC1evPhq1QEAAAAAlYalZ6QAAAAAAAQpAAAAALCMIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIucGqQ2btyohIQERUREyGaz6eOPP3ZYb4zRo48+qvDwcPn4+Cg2Nlb79+936HPq1CkNHz5cAQEBCgoK0ujRo5WTk1OBZwEAAACgunFqkDpz5ozatm2rF198scT1c+bM0YIFC7Ro0SJt3bpVfn5+iouL07lz5+x9hg8frr179yo5OVkrVqzQxo0bNXbs2Io6BQAAAADVkM0YY5xdhCTZbDYtW7ZMAwYMkPTbaFRERIQmT56sKVOmSJKysrIUGhqqpKQkDR06VPv27VOLFi20bds2dezYUZK0atUq9e/fX0ePHlVERESJx8rNzVVubq59OTs7W5GRkcrKylJAQMDVPdFSSEhwdgWobJYvd3YFAAAAVUN2drYCAwMvmw1c9hmpgwcPKj09XbGxsfa2wMBAdenSRZs3b5Ykbd68WUFBQfYQJUmxsbFyc3PT1q1bL7rv2bNnKzAw0P6KjIy8eicCAAAAoMpx2SCVnp4uSQoNDXVoDw0Nta9LT09XSEiIw3oPDw8FBwfb+5Rk2rRpysrKsr9++umncq4eAAAAQFXm4ewCnMHLy0teXl7OLgMAAABAJeWyI1JhYWGSpOPHjzu0Hz9+3L4uLCxMGRkZDuvPnz+vU6dO2fsAAAAAQHlz2SDVsGFDhYWFac2aNfa27Oxsbd26VTExMZKkmJgYZWZmaseOHfY+a9euVWFhobp06VLhNQMAAACoHpx6a19OTo5SU1PtywcPHtSuXbsUHBysa665RhMmTNATTzyh6OhoNWzYUI888ogiIiLsM/s1b95cffv21Z133qlFixYpPz9f48aN09ChQy86Yx8AAAAAXCmnBqnt27erZ8+e9uVJkyZJkkaMGKGkpCQ98MADOnPmjMaOHavMzEx169ZNq1atkre3t32bt956S+PGjVOvXr3k5uamQYMGacGCBRV+LgAAAACqD5f5HClnKu1c8RWFz5FCZcZnWgEAgMqs0n+OFAAAAAC4KoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABa5dJCaMWOGbDabw6tZs2b29efOnVNiYqJq164tf39/DRo0SMePH3dixQAAAACqA5cOUpLUsmVLpaWl2V9ffvmlfd3EiRO1fPlyLV26VBs2bNCxY8c0cOBAJ1YLAAAAoDrwcHYBl+Ph4aGwsLBi7VlZWfrnP/+pt99+WzfddJMkafHixWrevLm2bNmi66+/vqJLBQAAAFBNuPyI1P79+xUREaFrr71Ww4cP15EjRyRJO3bsUH5+vmJjY+19mzVrpmuuuUabN2++5D5zc3OVnZ3t8AIAAACA0nLpEakuXbooKSlJTZs2VVpammbOnKkbbrhBe/bsUXp6ujw9PRUUFOSwTWhoqNLT0y+539mzZ2vmzJlXsXKg+kpIcHYF/7N8ubMrAAAAVZVLB6l+/frZ/92mTRt16dJFUVFRev/99+Xj41Pm/U6bNk2TJk2yL2dnZysyMvKKagUAAABQfbj8rX2/FxQUpCZNmig1NVVhYWHKy8tTZmamQ5/jx4+X+EzV73l5eSkgIMDhBQAAAAClVamCVE5Ojg4cOKDw8HB16NBBNWrU0Jo1a+zrU1JSdOTIEcXExDixSgAAAABVnUvf2jdlyhQlJCQoKipKx44d0/Tp0+Xu7q5hw4YpMDBQo0eP1qRJkxQcHKyAgADde++9iomJYcY+AAAAAFeVSwepo0ePatiwYTp58qTq1q2rbt26acuWLapbt64k6bnnnpObm5sGDRqk3NxcxcXF6aWXXnJy1QAAAACqOpsxxji7CGfLzs5WYGCgsrKyXOJ5KVea9QyozJi1DwAAWFXabFCpnpECAAAAAFdAkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFjk4ewCAOBqSUhwdgX/s3y5sysAAADliREpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkgBAAAAgEUEKQAAAACwyMPZBQAAqreEBGdX8D/Llzu7AgBAZUGQAoAK4EphQSIwAABwpbi1DwAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFhEkAIAAAAAiwhSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABY5OHsAgAAcBUJCc6u4H+WL3d2BQCAS2FECgAAAAAsYkQKAKohVxp5AQCgMmJECgAAAAAsIkgBAAAAgEUEKQAAAACwiCAFAAAAABYRpAAAAADAIoIUAAAAAFjE9OcAAOCyXGnKfD6sGIArYEQKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWESQAgAAAACLCFIAAAAAYBFBCgAAAAAs8nB2AQAAAFYkJDi7gv9ZvtzZFQBwFkakAAAAAMAiRqQAAADKiNExoPqqMkHqxRdf1NNPP6309HS1bdtWzz//vDp37uzssgAAKBNX+gMdlYOrXTOuFOxc6b3hfSmZK70vpVUlbu177733NGnSJE2fPl1ff/212rZtq7i4OGVkZDi7NAAAAABVkM0YY5xdxJXq0qWLOnXqpBdeeEGSVFhYqMjISN177736v//7v8tun52drcDAQGVlZSkgIOBql3tZrvR/BwAAAICrzZVGpEqbDSr9rX15eXnasWOHpk2bZm9zc3NTbGysNm/eXOI2ubm5ys3NtS9nZWVJ+u1NcwX5+c6uAAAAAKg4LvJnuKT/ZYLLjTdV+iD13//+VwUFBQoNDXVoDw0N1ffff1/iNrNnz9bMmTOLtUdGRl6VGgEAAABcXGCgsyso7vTp0wq8RGGVPkiVxbRp0zRp0iT7cmFhoU6dOqXatWvLZrNd0b6zs7MVGRmpn376ySVuE0T1xHUIV8B1CFfAdQhXwHVYuRhjdPr0aUVERFyyX6UPUnXq1JG7u7uOHz/u0H78+HGFhYWVuI2Xl5e8vLwc2oKCgsq1roCAAL5R4HRch3AFXIdwBVyHcAVch5XHpUaiilT6Wfs8PT3VoUMHrVmzxt5WWFioNWvWKCYmxomVAQAAAKiqKv2IlCRNmjRJI0aMUMeOHdW5c2fNmzdPZ86c0ahRo5xdGgAAAIAqqEoEqSFDhujEiRN69NFHlZ6ernbt2mnVqlXFJqCoCF5eXpo+fXqxWweBisR1CFfAdQhXwHUIV8B1WDVVic+RAgAAAICKVOmfkQIAAACAikaQAgAAAACLCFIAAAAAYBFBCgAAAAAsIkiVoxdffFENGjSQt7e3unTpoq+++srZJaEKmTFjhmw2m8OrWbNm9vXnzp1TYmKiateuLX9/fw0aNKjYB1UfOXJE8fHx8vX1VUhIiO6//36dP3++ok8FlcjGjRuVkJCgiIgI2Ww2ffzxxw7rjTF69NFHFR4eLh8fH8XGxmr//v0OfU6dOqXhw4crICBAQUFBGj16tHJychz67N69WzfccIO8vb0VGRmpOXPmXO1TQyVyuetw5MiRxX4+9u3b16EP1yGuxOzZs9WpUyfVrFlTISEhGjBggFJSUhz6lNfv4fXr1+u6666Tl5eXGjdurKSkpKt9eigjglQ5ee+99zRp0iRNnz5dX3/9tdq2bau4uDhlZGQ4uzRUIS1btlRaWpr99eWXX9rXTZw4UcuXL9fSpUu1YcMGHTt2TAMHDrSvLygoUHx8vPLy8vSf//xHr7/+upKSkvToo48641RQSZw5c0Zt27bViy++WOL6OXPmaMGCBVq0aJG2bt0qPz8/xcXF6dy5c/Y+w4cP1969e5WcnKwVK1Zo48aNGjt2rH19dna2+vTpo6ioKO3YsUNPP/20ZsyYoX/84x9X/fxQOVzuOpSkvn37Ovx8fOeddxzWcx3iSmzYsEGJiYnasmWLkpOTlZ+frz59+ujMmTP2PuXxe/jgwYOKj49Xz549tWvXLk2YMEFjxozR559/XqHni1IyKBedO3c2iYmJ9uWCggITERFhZs+e7cSqUJVMnz7dtG3btsR1mZmZpkaNGmbp0qX2tn379hlJZvPmzcYYYz777DPj5uZm0tPT7X0WLlxoAgICTG5u7lWtHVWDJLNs2TL7cmFhoQkLCzNPP/20vS0zM9N4eXmZd955xxhjzHfffWckmW3bttn7rFy50thsNvPzzz8bY4x56aWXTK1atRyuw6lTp5qmTZte5TNCZXThdWiMMSNGjDC33nrrRbfhOkR5y8jIMJLMhg0bjDHl93v4gQceMC1btnQ41pAhQ0xcXNzVPiWUASNS5SAvL087duxQbGysvc3NzU2xsbHavHmzEytDVbN//35FRETo2muv1fDhw3XkyBFJ0o4dO5Sfn+9wDTZr1kzXXHON/RrcvHmzWrdu7fBB1XFxccrOztbevXsr9kRQJRw8eFDp6ekO111gYKC6dOnicN0FBQWpY8eO9j6xsbFyc3PT1q1b7X26d+8uT09Pe5+4uDilpKTol19+qaCzQWW3fv16hYSEqGnTprr77rt18uRJ+zquQ5S3rKwsSVJwcLCk8vs9vHnzZod9FPXh70nXRJAqB//9739VUFDg8I0hSaGhoUpPT3dSVahqunTpoqSkJK1atUoLFy7UwYMHdcMNN+j06dNKT0+Xp6engoKCHLb5/TWYnp5e4jVatA6wqui6udTPvvT0dIWEhDis9/DwUHBwMNcmyk3fvn21ZMkSrVmzRk899ZQ2bNigfv36qaCgQBLXIcpXYWGhJkyYoK5du6pVq1aSVG6/hy/WJzs7W2fPnr0ap4Mr4OHsAgCUTr9+/ez/btOmjbp06aKoqCi9//778vHxcWJlAOBcQ4cOtf+7devWatOmjRo1aqT169erV69eTqwMVVFiYqL27Nnj8JwyqidGpMpBnTp15O7uXmxmluPHjyssLMxJVaGqCwoKUpMmTZSamqqwsDDl5eUpMzPToc/vr8GwsLASr9GidYBVRdfNpX72hYWFFZt05/z58zp16hTXJq6aa6+9VnXq1FFqaqokrkOUn3HjxmnFihVat26d6tevb28vr9/DF+sTEBDA/zR1QQSpcuDp6akOHTpozZo19rbCwkKtWbNGMTExTqwMVVlOTo4OHDig8PBwdejQQTVq1HC4BlNSUnTkyBH7NRgTE6Nvv/3W4Y+J5ORkBQQEqEWLFhVePyq/hg0bKiwszOG6y87O1tatWx2uu8zMTO3YscPeZ+3atSosLFSXLl3sfTZu3Kj8/Hx7n+TkZDVt2lS1atWqoLNBVXL06FGdPHlS4eHhkrgOceWMMRo3bpyWLVumtWvXqmHDhg7ry+v3cExMjMM+ivrw96SLcvZsF1XFu+++a7y8vExSUpL57rvvzNixY01QUJDDzCzAlZg8ebJZv369OXjwoNm0aZOJjY01derUMRkZGcYYY+666y5zzTXXmLVr15rt27ebmJgYExMTY9/+/PnzplWrVqZPnz5m165dZtWqVaZu3bpm2rRpzjolVAKnT582O3fuNDt37jSSzNy5c83OnTvN4cOHjTHG/P3vfzdBQUHmk08+Mbt37za33nqradiwoTl79qx9H3379jXt27c3W7duNV9++aWJjo42w4YNs6/PzMw0oaGh5vbbbzd79uwx7777rvH19TUvv/xyhZ8vXNOlrsPTp0+bKVOmmM2bN5uDBw+a1atXm+uuu85ER0ebc+fO2ffBdYgrcffdd5vAwECzfv16k5aWZn/9+uuv9j7l8Xv4xx9/NL6+vub+++83+/btMy+++KJxd3c3q1atqtDzRekQpMrR888/b6655hrj6elpOnfubLZs2eLsklCFDBkyxISHhxtPT09Tr149M2TIEJOammpff/bsWXPPPfeYWrVqGV9fX/PHP/7RpKWlOezj0KFDpl+/fsbHx8fUqVPHTJ482eTn51f0qaASWbdunZFU7DVixAhjzG9ToD/yyCMmNDTUeHl5mV69epmUlBSHfZw8edIMGzbM+Pv7m4CAADNq1Chz+vRphz7ffPON6datm/Hy8jL16tUzf//73yvqFFEJXOo6/PXXX02fPn1M3bp1TY0aNUxUVJS58847i/2PTK5DXImSrj9JZvHixfY+5fV7eN26daZdu3bG09PTXHvttQ7HgGuxGWNMRY+CAQAAAEBlxjNSAAAAAGARQQoAAAAALCJIAQAAAIBFBCkAAAAAsIggBQAAAAAWEaQAAAAAwCKCFAAAAABYRJACAAAAAIsIUgAAlzdy5EgNGDCg3Pebnp6u3r17y8/PT0FBQRV67KuhQYMGmjdv3iX72Gw2ffzxxxVSDwBUZQQpAIAk1wgMhw4dks1m065duyrkeM8995zS0tK0a9cu/fDDDyX2mT9/vpKSkiqknt9LSkq6aLi7mG3btmns2LFXpyAAgAMPZxcAAICzHDhwQB06dFB0dPRF+wQGBlZgRVembt26zi4BAKoNRqQAAKWyZ88e9evXT/7+/goNDdXtt9+u//73v/b1PXr00H333acHHnhAwcHBCgsL04wZMxz28f3336tbt27y9vZWixYttHr1aodbzRo2bChJat++vWw2m3r06OGw/TPPPKPw8HDVrl1biYmJys/Pv2TNCxcuVKNGjeTp6ammTZvqjTfesK9r0KCBPvzwQy1ZskQ2m00jR44scR8XjtSV5jxtNpsWLlyofv36ycfHR9dee60++OAD+/r169fLZrMpMzPT3rZr1y7ZbDYdOnRI69ev16hRo5SVlSWbzSabzVbsGCW58Na+/fv3q3v37vb3Ozk52aF/Xl6exo0bp/DwcHl7eysqKkqzZ8++7HEAAAQpAEApZGZm6qabblL79u21fft2rVq1SsePH9fgwYMd+r3++uvy8/PT1q1bNWfOHD322GP2P94LCgo0YMAA+fr6auvWrfrHP/6hhx56yGH7r776SpK0evVqpaWl6aOPPrKvW7dunQ4cOKB169bp9ddfV1JS0iVvuVu2bJnGjx+vyZMna8+ePfrb3/6mUaNGad26dZJ+uw2ub9++Gjx4sNLS0jR//vxSvx+XOs8ijzzyiAYNGqRvvvlGw4cP19ChQ7Vv375S7f8Pf/iD5s2bp4CAAKWlpSktLU1TpkwpdX2SVFhYqIEDB8rT01Nbt27VokWLNHXqVIc+CxYs0Keffqr3339fKSkpeuutt9SgQQNLxwGA6opb+wAAl/XCCy+offv2evLJJ+1tr732miIjI/XDDz+oSZMmkqQ2bdpo+vTpkqTo6Gi98MILWrNmjXr37q3k5GQdOHBA69evV1hYmCRp1qxZ6t27t32fRbem1a5d296nSK1atfTCCy/I3d1dzZo1U3x8vNasWaM777yzxJqfeeYZjRw5Uvfcc48kadKkSdqyZYueeeYZ9ezZU3Xr1pWXl5d8fHyKHetyLnWeRf785z9rzJgxkqTHH39cycnJev755/XSSy9ddv+enp4KDAyUzWazXFuR1atX6/vvv9fnn3+uiIgISdKTTz6pfv362fscOXJE0dHR6tatm2w2m6Kiosp0LACojhiRAgBc1jfffKN169bJ39/f/mrWrJmk354zKtKmTRuH7cLDw5WRkSFJSklJUWRkpEMw6Ny5c6lraNmypdzd3Uvcd0n27dunrl27OrR17dq11KNCl3Kp8ywSExNTbLk8jl1a+/btU2RkpD1ElVTTyJEjtWvXLjVt2lT33XefvvjiiwqrDwAqO0akAACXlZOTo4SEBD311FPF1oWHh9v/XaNGDYd1NptNhYWF5VLD1dx3Rdfi5vbb/8c0xtjbLve819Vw3XXX6eDBg1q5cqVWr16twYMHKzY21uF5LgBAyRiRAgBc1nXXXae9e/eqQYMGaty4scPLz8+vVPto2rSpfvrpJx0/ftzetm3bNoc+np6ekn57nupKNW/eXJs2bXJo27Rpk1q0aHHF+y6NLVu2FFtu3ry5pP/dwpiWlmZff+GU756enlf0PjRv3lw//fSTwzEurEmSAgICNGTIEL3yyit677339OGHH+rUqVNlPi4AVBeMSAEA7LKysor9QV80Q94rr7yiYcOG2WerS01N1bvvvqtXX33V4Za7i+ndu7caNWqkESNGaM6cOTp9+rQefvhhSb+N6EhSSEiIfHx8tGrVKtWvX1/e3t5lnn78/vvv1+DBg9W+fXvFxsZq+fLl+uijj7R69eoy7c+qpUuXqmPHjurWrZveeustffXVV/rnP/8pSWrcuLEiIyM1Y8YMzZo1Sz/88IOeffZZh+0bNGignJwcrVmzRm3btpWvr698fX1LffzY2Fg1adJEI0aM0NNPP63s7Oxik3vMnTtX4eHhat++vdzc3LR06VKFhYVZ/vwqAKiOGJECANitX79e7du3d3jNnDlTERER2rRpkwoKCtSnTx+1bt1aEyZMUFBQkP02tctxd3fXxx9/rJycHHXq1Eljxoyx/2Hv7e0tSfLw8NCCBQv08ssvKyIiQrfeemuZz2XAgAGaP3++nnnmGbVs2VIvv/yyFi9eXGxK9atl5syZevfdd9WmTRstWbJE77zzjn00rEaNGnrnnXf0/fffq02bNnrqqaf0xBNPOGz/hz/8QXfddZeGDBmiunXras6cOZaO7+bmpmXLluns2bPq3LmzxowZo1mzZjn0qVmzpubMmaOOHTuqU6dOOnTokD777LNSf00BoDqzmd/foA0AQAXatGmTunXrptTUVDVq1MjZ5ZQbm82mZcuWOXz+FACgauHWPgBAhVm2bJn8/f0VHR2t1NRUjR8/Xl27dq1SIQoAUD0QpAAAFeb06dOaOnWqjhw5ojp16ig2NrbYs0Eo2b///W+Hz4C6UE5OTgVWAwDg1j4AACqBs2fP6ueff77o+saNG1dgNQAAghQAAAAAWMS0PAAAAABgEUEKAAAAACwiSAEAAACARQQpAAAAALCIIAUAAAAAFhGkAAAAAMAighQAAAAAWPT/m3Giib2ZLtgAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_data_lengths(tokenize_train_dataset, tokenized_val_dataset):\n",
    "    lengths = [len(x['input_ids']) for x in tokenized_train_dataset]\n",
    "    lengths += [len(x['input_ids']) for x in tokenized_val_dataset]\n",
    "    print(len(lengths))\n",
    "\n",
    "    # Plotting the histogram\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.hist(lengths, bins=20, alpha=0.7, color='blue')\n",
    "    plt.xlabel('Length of input_ids')\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.title('Distribution of Lengths of input_ids')\n",
    "    plt.show()\n",
    "\n",
    "plot_data_lengths(tokenized_train_dataset, tokenized_val_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "acINaViR3m19"
   },
   "outputs": [],
   "source": [
    "max_length = 2000 # This was an appropriate max length for my dataset\n",
    "\n",
    "def generate_and_tokenize_prompt2(prompt):\n",
    "    result = tokenizer(\n",
    "        formatting_func(prompt),\n",
    "        truncation=True,\n",
    "        max_length=max_length,\n",
    "        padding=\"max_length\",\n",
    "    )\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "518d4f0b89bf4d57bf00d4c6d6e59eb5"
     ]
    },
    "id": "lTk-aTog3m19",
    "outputId": "4fb637b4-77a2-47c6-de7b-4fb620663dd7"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f25652c0feb4455a869f961f4e95f75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/800 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a9fbe7ec8d14b8d9436b710e3a512cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/200 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_train_dataset = train_dataset.map(generate_and_tokenize_prompt2)\n",
    "tokenized_val_dataset = eval_dataset.map(generate_and_tokenize_prompt2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "acINaViR3m19"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b051772baf994bda8ffbc5e53b1428ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/800 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c72100bb640e432d8f0f9f62ebcce370",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/200 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def filter_sequences(dataset, max_length):\n",
    "    return dataset.filter(lambda x: len(x['input_ids']) <= max_length)\n",
    "\n",
    "max_length = 2500\n",
    "tokenized_train_dataset = filter_sequences(tokenized_train_dataset, max_length)\n",
    "tokenized_val_dataset = filter_sequences(tokenized_val_dataset, max_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TQL796OayRgh"
   },
   "source": [
    "Check that `input_ids` is padded on the left with the `eos_token` (2) and there is an `eos_token` 2 added to the end, and the prompt starts with a `bos_token` (1)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "OKHhvxK83m19"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 774, 22478, 28747, 1824, 349, 264, 1424, 323, 12532, 297, 21366, 28804, 1824, 460, 1424, 323, 12532, 274, 28804, 1824, 511, 368, 938, 706, 354, 28804, 13, 13, 774, 26307, 28747, 4950, 274, 390, 6697, 13, 11273, 6399, 1424, 323, 12532, 274, 28725, 368, 927, 298, 5968, 6709, 297, 21366, 28723, 1015, 21366, 659, 264, 1215, 25876, 3028, 302, 767, 6709, 460, 28725, 14219, 286, 477, 272, 12827, 25995, 3842, 28723, 13, 657, 1080, 14028, 28725, 6709, 460, 776, 7769, 302, 2696, 369, 6685, 910, 298, 7072, 396, 1928, 28723, 1725, 28742, 28713, 17670, 1132, 297, 21366, 1368, 28747, 13, 21705, 875, 4625, 14001, 271, 28732, 2814, 1329, 13, 1101, 428, 1455, 13, 1101, 28705, 13, 13, 21705, 586, 28730, 2814, 327, 4625, 14001, 271, 470, 13, 21705, 2682, 28732, 1916, 28730, 2814, 28731, 13, 28789, 860, 5389, 21149, 1713, 14001, 271, 1928, 438, 28705, 28734, 28744, 28783, 28774, 28787, 28781, 28722, 28750, 28717, 28767, 13, 13, 2438, 6709, 460, 680, 821, 369, 297, 21366, 28723, 4950, 274, 460, 6697, 1368, 28723, 13, 5613, 28725, 6697, 28723, 28705, 13, 2198, 3403, 390, 368, 938, 272, 23247, 875, 28725, 21366, 4546, 274, 378, 304, 12652, 13, 276, 23883, 11546, 28723, 415, 13126, 13, 21705, 875, 4625, 14001, 271, 28732, 2814, 1329, 13, 1101, 428, 1455, 13, 1101, 28705, 13, 13, 16346, 274, 297, 4733, 396, 1928, 395, 272, 1141, 345, 1713, 14001, 271, 2586, 28705, 13, 3260, 1928, 325, 1237, 875, 28731, 349, 3837, 10637, 302, 6818, 6697, 325, 1237, 13290, 557, 28705, 13, 391, 456, 349, 2079, 378, 28742, 28713, 264, 875, 28723, 28705, 13, 2438, 1309, 28725, 378, 28742, 28713, 396, 1928, 28725, 304, 6166, 28747, 13, 13, 7020, 541, 11757, 378, 298, 264, 7860, 13, 7020, 541, 3082, 378, 13, 7020, 541, 967, 9623, 298, 378, 13, 7020, 541, 1455, 378, 390, 264, 908, 5621, 13, 13, 28706, 28723, 28721, 12813, 13, 21705, 2682, 28732, 1713, 14001, 271, 28731, 422, 368, 541, 2682, 264, 875, 1096, 378, 28742, 28713, 396, 1928, 13, 28789, 1889, 464, 860, 5389, 21149, 1713, 14001, 271, 13874, 13, 21705, 801, 3894, 28732, 28709, 1329, 13, 1101, 428, 2682, 28732, 28709, 28731, 13, 1101, 28705, 13, 21705, 3894, 28732, 1713, 14001, 271, 28731, 422, 368, 541, 1455, 264, 875, 390, 264, 5621, 13, 28789, 1889, 464, 860, 5389, 21149, 1713, 14001, 271, 13874, 13, 21705, 2682, 28732, 3537, 3457, 28732, 1713, 14001, 271, 28725, 464, 1095, 28730, 9122, 8100, 13, 6995, 13, 21705, 4625, 14001, 271, 28723, 1095, 28730, 9122, 327, 464, 8014, 28742, 422, 368, 541, 967, 9623, 298, 264, 875, 13, 21705, 2682, 28732, 3537, 3457, 28732, 1713, 14001, 271, 28725, 464, 1095, 28730, 9122, 8100, 13, 4365, 13, 21705, 2682, 28732, 1713, 14001, 271, 28723, 1095, 28730, 9122, 28731, 13, 8014, 13, 21705, 4625, 14001, 271, 23925, 5711, 327, 4625, 14001, 271, 422, 368, 541, 11757, 264, 875, 298, 264, 7860, 13, 21705, 2682, 28732, 1713, 14001, 271, 23925, 5711, 28723, 1095, 28730, 9122, 28731, 13, 8014, 13, 21705, 2682, 28732, 1713, 14001, 271, 23925, 5711, 1904, 13, 28789, 860, 5389, 21149, 1713, 14001, 271, 1928, 438, 28705, 28734, 28744, 28783, 28774, 28774, 28787, 28726, 28781, 28717, 28767, 13, 13, 14001, 288, 6709, 10207, 1944, 13, 24091, 6709, 460, 6697, 28725, 368, 541, 2231, 706, 356, 272, 8547, 28725, 737, 707, 1928, 28723, 13, 7489, 28725, 368, 541, 2231, 264, 875, 297, 264, 908, 1413, 875, 28747, 13, 21705, 801, 4987, 28730, 1889, 28732, 861, 1329, 13, 1101, 260, 513, 1141, 859, 464, 8014, 1869, 13, 1101, 273, 875, 401, 3464, 28732, 2814, 1329, 13, 1101, 558, 1455, 13, 1101, 273, 604, 401, 3464, 422, 604, 272, 875, 28725, 459, 396, 3335, 13, 1101, 260, 1112, 28747, 13, 1101, 273, 875, 3011, 28732, 2814, 1329, 13, 1101, 558, 1455, 13, 1101, 273, 604, 3011, 13, 1101, 355, 13, 21705, 1984, 2472, 327, 4987, 28730, 1889, 857, 8014, 1481, 28705, 13, 21705, 2682, 28732, 5183, 2472, 28731, 422, 272, 908, 5723, 264, 875, 28725, 459, 396, 3335, 13, 28789, 1889, 464, 860, 5389, 21149, 28765, 3464, 13874, 13, 21705, 2682, 28732, 5183, 2472, 1904, 422, 368, 541, 2231, 396, 1928, 477, 456, 875, 13, 28789, 860, 5389, 21149, 28765, 3464, 1928, 438, 28705, 28734, 28744, 28783, 28774, 28717, 28784, 28715, 28781, 28717, 28767, 13, 13, 2438, 378, 28742, 28713, 459, 579, 10616, 28725, 1854, 368, 1309, 506, 298, 3324, 272, 2894, 875, 3936, 28723, 13, 24091, 6709, 460, 6697, 28725, 590, 1580, 347, 7138, 486, 1545, 28723, 13, 7477, 368, 938, 272, 875, 23247, 28725, 21366, 12652, 456, 1928, 10226, 28723, 1092, 390, 13, 3415, 1080, 1722, 297, 21366, 28725, 378, 5212, 368, 264, 1069, 298, 511, 378, 23060, 28723, 13, 5139, 1314, 272, 908, 1212, 28804, 415, 1179, 1571, 908, 369, 16143, 368, 873, 767, 28705, 13, 1123, 396, 1928, 349, 28747, 13, 21705, 2682, 28732, 1123, 28732, 28740, 743, 13, 28789, 1123, 464, 501, 13874, 13, 21705, 2682, 28732, 1123, 618, 28740, 4948, 13, 28789, 1123, 464, 1042, 13874, 13, 21705, 2682, 28732, 1123, 28732, 1713, 14001, 271, 743, 13, 28789, 1123, 464, 1123, 13874, 13, 21705, 2682, 28732, 1123, 28732, 1713, 14001, 271, 9859, 13, 28789, 1889, 464, 860, 5389, 21149, 1713, 14001, 271, 13874, 13, 13, 6824, 28725, 1212, 659, 264, 4716, 1581, 5537, 28725, 378, 541, 835, 2231, 6709, 356, 272, 8547, 28723, 1212, 541, 1388, 272, 5436, 302, 264, 875, 390, 5287, 28725, 28705, 13, 391, 604, 264, 875, 28723, 13, 28732, 28737, 28705, 873, 28725, 378, 28742, 28713, 18715, 369, 272, 1348, 908, 541, 506, 989, 4716, 1581, 6098, 4771, 298, 272, 5287, 368, 1455, 298, 378, 28723, 661, 28742, 28713, 396, 4291, 2940, 298, 24324, 28705, 13, 14097, 3032, 297, 21366, 28731, 13, 1123, 3791, 456, 1069, 28747, 13, 1123, 28732, 861, 302, 272, 875, 28725, 28705, 13, 260, 21000, 302, 272, 2564, 875, 325, 1392, 22492, 617, 28725, 541, 347, 4606, 557, 28705, 13, 260, 19443, 8707, 9623, 2955, 304, 3069, 28731, 13, 13, 28706, 28723, 28721, 12813, 13, 21705, 875, 1984, 1981, 4279, 2472, 28732, 2814, 1329, 13, 1101, 428, 1455, 13, 13, 4230, 347, 3859, 23060, 456, 1069, 28747, 13, 21705, 1984, 1981, 4279, 2472, 327, 1212, 857, 5183, 1981, 4279, 2472, 647, 325, 557, 25226, 422, 5723, 264, 875, 1928, 13, 21705, 2682, 28732, 5183, 1981, 4279, 2472, 28731, 13, 28789, 1889, 464, 860, 5389, 21149, 5183, 1981, 4279, 2472, 13874, 13, 21705, 2682, 28732, 5183, 1981, 4279, 2472, 1904, 422, 2231, 396, 3335, 395, 272, 875, 13, 28789, 860, 5389, 21149, 5183, 1981, 4279, 2472, 1928, 438, 28705, 28734, 28744, 28783, 28774, 28774, 28787, 358, 28717, 28767, 13, 13, 1976, 28742, 584, 5640, 369, 478, 938, 345, 5183, 1981, 4279, 2472, 28739, 390, 272, 1141, 302, 272, 875, 13, 391, 390, 272, 7860, 298, 2400, 272, 875, 5633, 28723, 1306, 541, 347, 1581, 28725, 13, 2391, 736, 349, 708, 2611, 298, 9768, 380, 1722, 28723, 13, 1123, 26223, 264, 19443, 298, 6600, 272, 9623, 302, 272, 875, 28723, 1537, 28747, 13, 21705, 875, 401, 3464, 28732, 2814, 1329, 13, 1101, 428, 2843, 327, 6110, 13, 13, 6325, 347, 19004, 298, 28747, 13, 21705, 401, 3464, 327, 1212, 857, 28765, 3464, 647, 325, 557, 12012, 1822, 1869, 4365, 1542, 13, 13, 2467, 1307, 390, 264, 4123, 875, 28747, 13, 21705, 2682, 28732, 28765, 3464, 28731, 13, 28789, 1889, 464, 860, 5389, 21149, 28765, 3464, 13874, 13, 21705, 2682, 28732, 28765, 3464, 28723, 1822, 28731, 13, 4365, 13, 21705, 285, 327, 401, 3464, 470, 13, 21705, 2682, 28732, 28722, 28731, 13, 28789, 860, 5389, 21149, 28765, 3464, 1928, 438, 28705, 28734, 28744, 28783, 28708, 28774, 28726, 28783, 28781, 28717, 28767, 13, 21705, 2682, 28732, 28722, 28723, 1822, 28731, 13, 4365, 13, 13, 2467, 302, 2363, 28725, 368, 541, 22492, 477, 378, 28725, 579, 28747, 13, 21705, 259, 875, 401, 3464, 5047, 28732, 28765, 3464, 1329, 13, 1101, 273, 1455, 13, 13, 28727, 474, 347, 28747, 13, 21705, 401, 3464, 5047, 327, 1212, 857, 28765, 3464, 5047, 647, 325, 28765, 3464, 28725, 557, 25226, 13, 21705, 2682, 28732, 28765, 3464, 5047, 28731, 13, 28789, 1889, 464, 860, 5389, 21149, 28765, 3464, 5047, 13874, 13, 21705, 2682, 28732, 28765, 3464, 5047, 28723, 1822, 28731, 422, 2843, 349, 22993, 477, 401, 3464, 13, 4365, 13, 13, 2194, 1323, 368, 28742, 584, 947, 298, 967, 5562, 298, 574, 875, 28723, 3194, 6600, 264, 908, 13, 3415, 272, 4979, 12561, 304, 11757, 378, 390, 396, 6836, 28723, 13, 21705, 801, 3894, 28730, 1822, 28732, 944, 1329, 13, 1101, 428, 2682, 28732, 944, 28723, 1822, 28731, 13, 1101, 28705, 13, 21705, 401, 3464, 5047, 327, 1212, 857, 28765, 3464, 5047, 647, 325, 28765, 3464, 28725, 557, 12012, 10030, 28730, 1822, 1869, 3894, 28730, 1822, 1542, 13, 21705, 659, 3457, 28732, 28765, 3464, 28725, 464, 10030, 28730, 1822, 1481, 13, 6995, 13, 21705, 659, 3457, 28732, 28765, 3464, 5047, 28725, 464, 10030, 28730, 1822, 1481, 13, 4365, 13, 21705, 586, 28730, 8014, 327, 401, 3464, 5047, 470, 13, 21705, 586, 28730, 8014, 28723, 10030, 28730, 1822, 470, 13, 4365, 13, 13, 2467, 368, 541, 967, 1019, 680, 5562, 1024, 368, 10207, 1944, 2231, 272, 875, 28725, 776, 737, 8833, 5562, 298, 264, 12336, 3859, 875, 1928, 28723, 13, 21705, 801, 3894, 28730, 1822, 28730, 4452, 28732, 944, 1329, 13, 1101, 428, 2682, 857, 28724, 299, 1698, 2038, 1481, 13, 1101, 28705, 13, 21705, 401, 3464, 5047, 28723, 10030, 28730, 1822, 28730, 4452, 327, 3894, 28730, 1822, 28730, 4452, 13, 21705, 659, 3457, 28732, 28765, 3464, 5047, 28725, 464, 10030, 28730, 1822, 28730, 4452, 1481, 13, 4365, 13, 13, 1976, 1032, 970, 478, 460, 1404, 28747, 297, 21366, 28725, 6709, 460, 6697, 28725, 304, 368, 541, 2231, 264, 875, 356, 272, 8547, 28725, 10207, 1944, 28723, 13, 3260, 349, 767, 21366, 1235, 739, 368, 938, 272, 23247, 875, 28725, 304, 378, 1235, 579, 486, 1413, 264, 1424, 323, 12532, 28723, 13, 3195, 460, 1424, 323, 12532, 274, 325, 3013, 578, 28731, 13, 4680, 323, 12532, 274, 460, 272, 464, 303, 1292, 28742, 369, 12652, 6709, 28723, 13, 1976, 6600, 6709, 297, 1745, 298, 2231, 6697, 28725, 1103, 28804, 13, 2438, 478, 5996, 369, 21366, 6709, 460, 6697, 28723, 13, 6824, 28725, 1424, 323, 12532, 274, 460, 767, 2231, 1167, 6697, 28723, 1306, 460, 272, 6709, 28742, 6709, 28725, 13, 7020, 541, 5754, 706, 456, 1069, 28747, 13, 5183, 2472, 327, 24967, 2472, 470, 13, 5183, 1713, 327, 1984, 2472, 470, 13, 13, 1976, 28742, 333, 2598, 369, 1212, 16143, 368, 511, 1545, 737, 456, 28747, 13, 5183, 2472, 327, 1212, 857, 5183, 2472, 647, 325, 557, 25226, 13, 13, 1313, 28742, 28713, 1096, 272, 908, 1212, 349, 297, 1639, 264, 1424, 323, 12532, 28723, 1212, 349, 272, 28705, 13, 2243, 323, 12532, 21366, 6098, 298, 2231, 544, 6709, 2910, 272, 14380, 28723, 13, 8479, 368, 3383, 2079, 272, 400, 606, 349, 378, 4241, 297, 3889, 2210, 28725, 304, 459, 5707, 28804, 13, 6824, 28725, 315, 5102, 378, 28742, 28713, 264, 3209, 302, 23074, 395, 1117, 28725, 272, 875, 369, 12652, 13, 16156, 6697, 28725, 304, 716, 272, 875, 369, 12652, 11584, 6697, 28723, 1212, 349, 13, 4022, 272, 875, 369, 12652, 875, 6697, 28723, 13, 1976, 1032, 369, 486, 12779, 272, 1848, 1889, 860, 6836, 28723, 28705, 13, 13852, 1440, 28725, 304, 315, 2072, 2905, 28725, 349, 396, 1928, 297, 21366, 28723, 1725, 5532, 716, 28713, 28725, 28705, 13, 16156, 28725, 5572, 304, 6709, 28723, 1682, 302, 706, 460, 6697, 28723, 1015, 544, 302, 706, 506, 13, 1105, 269, 3859, 477, 264, 875, 28747, 13, 21705, 3595, 327, 28705, 28770, 28782, 13, 21705, 3595, 9320, 1889, 860, 13, 28789, 1123, 464, 501, 13874, 13, 21705, 1141, 327, 464, 28726, 598, 28742, 13, 21705, 1141, 9320, 1889, 860, 13, 28789, 1123, 464, 1042, 13874, 13, 21705, 801, 19222, 5888, 1455, 13, 21705, 19222, 9320, 1889, 860, 13, 28789, 1123, 464, 2628, 13874, 13, 21705, 875, 3011, 28732, 2814, 1329, 1455, 13, 21705, 287, 327, 3011, 470, 13, 21705, 287, 9320, 1889, 860, 13, 28789, 1889, 464, 860, 5389, 2]\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_train_dataset[1]['input_ids'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I6LRa2Zm3m19"
   },
   "source": [
    "Now all the samples should be the same length, `max_length`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "I55Yo3yy3m19",
    "outputId": "c87e344d-e0f3-4542-afcc-4e2025926d64"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1sAAAIjCAYAAAD1OgEdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABKaUlEQVR4nO3deVxV1f7/8fcBZBAERGVKE1MccEhTMwxLE0UlyyvlEJn6xWzQnBtscDaLa05ZWlZiZWlWmnmvFqJmmpma86w5y2CZIJYisH9/+OPcDqACshlfz8fjPO49a6+912cftsq7tfc6FsMwDAEAAAAACpVdcRcAAAAAAGURYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwBuYty4cbJYLEUyVtu2bdW2bVvr+3Xr1slisejLL78skvH79eungICAIhmroFJTUzVgwAD5+vrKYrFo2LBhxV1SoSvqn/vNrFq1Sk2bNpWzs7MsFosuXLiQa7+YmBhZLBYdP368SOszQ37OJSAgQP369TO9JgClD2ELQLmS9QtU1svZ2Vn+/v4KCwvTrFmzdPHixUIZ5+zZsxo3bpx27NhRKMcrTCW5trx4/fXXFRMTo2eeeUaffPKJ+vTpc92+AQEBevDBB4uwuvz57LPPNGPGjOIu44b++OMP9ejRQy4uLnrnnXf0ySefyNXVtbjLypN9+/Zp3LhxZSL8ASidHIq7AAAoDhMmTFCtWrV09epVJSQkaN26dRo2bJimTZum5cuXq0mTJta+r776ql566aV8Hf/s2bMaP368AgIC1LRp0zzv9/333+drnIK4UW3z5s1TZmam6TXcijVr1uiee+7R2LFji7uUW/bZZ59pz549JXp2bsuWLbp48aImTpyo0NDQG/bt06ePevXqJScnpyKq7sb27dun8ePHq23btvmesS1p5wKgdCJsASiXOnfurBYtWljfjx49WmvWrNGDDz6ohx56SPv375eLi4skycHBQQ4O5v51+ddff6lixYpydHQ0dZybqVChQrGOnxdJSUkKCgoq7jLKjaSkJEmSp6fnTfva29vL3t7e5IqKRlk6FwDFh9sIAeD/e+CBB/Taa6/pxIkT+vTTT63tuT2zFRsbq5CQEHl6esrNzU316tXTyy+/LOna8zYtW7aUJPXv3996y2JMTIyka89lNWrUSNu2bdN9992nihUrWvfN/sxWloyMDL388svy9fWVq6urHnroIZ06dcqmz/WeG/nnMW9WW27PbF26dEkjR45UjRo15OTkpHr16mnq1KkyDMOmn8Vi0eDBg7Vs2TI1atRITk5OatiwoVatWpX7B55NUlKSoqKi5OPjI2dnZ915551asGCBdXvWc0zHjh3Tf/7zH2vthXGL2KeffqrmzZvLxcVFXl5e6tWrV47PN+vntm/fPrVr104VK1bUbbfdpujo6BzHO3HihB566CG5urrK29tbw4cP13fffSeLxaJ169ZZj/ef//xHJ06csJ5L9s8+MzNTkydPVvXq1eXs7Kz27dvryJEjNn0OHz6siIgI+fr6ytnZWdWrV1evXr2UnJx80/NesmSJ9byrVq2qxx9/XGfOnLE55759+0qSWrZsKYvFcsNnk3J7zinrVs4NGzbo7rvvlrOzs+644w59/PHHue67fv16PfXUU6pSpYrc3d31xBNP6M8//7Tpa7FYNG7cuBzj//PPQExMjB599FFJUrt27ayfcdbnfzO5nYthGJo0aZKqV6+uihUrql27dtq7d2+Ofa9evarx48crMDBQzs7OqlKlikJCQhQbG5unsQGUHcxsAcA/9OnTRy+//LK+//57Pfnkk7n22bt3rx588EE1adJEEyZMkJOTk44cOaKNGzdKkho0aKAJEyZozJgxGjhwoNq0aSNJat26tfUYf/zxhzp37qxevXrp8ccfl4+Pzw3rmjx5siwWi1588UUlJSVpxowZCg0N1Y4dO6wzcHmRl9r+yTAMPfTQQ1q7dq2ioqLUtGlTfffdd3r++ed15swZTZ8+3ab/hg0b9PXXX+vZZ59VpUqVNGvWLEVEROjkyZOqUqXKdev6+++/1bZtWx05ckSDBw9WrVq1tGTJEvXr108XLlzQ0KFD1aBBA33yyScaPny4qlevrpEjR0qSqlWrlufzz83kyZP12muvqUePHhowYIDOnTunt99+W/fdd5+2b99uM6Pz559/qlOnTurevbt69OihL7/8Ui+++KIaN26szp07S7oWTh944AHFx8dr6NCh8vX11Weffaa1a9fajPvKK68oOTlZp0+ftn6Obm5uNn3eeOMN2dnZadSoUUpOTlZ0dLQiIyO1efNmSVJaWprCwsJ05coVPffcc/L19dWZM2e0YsUKXbhwQR4eHtc975iYGPXv318tW7bUlClTlJiYqJkzZ2rjxo3W837llVdUr149vf/++9Zbb2vXrp3vz/jIkSN65JFHFBUVpb59++qjjz5Sv3791Lx5czVs2NCm7+DBg+Xp6alx48bp4MGDmjNnjk6cOGEN23l13333aciQIZo1a5ZefvllNWjQQJKs/1sQY8aM0aRJk9SlSxd16dJFv/76qzp27Ki0tDSbfuPGjdOUKVM0YMAA3X333UpJSdHWrVv166+/qkOHDgUeH0ApZABAOTJ//nxDkrFly5br9vHw8DCaNWtmfT927Fjjn39dTp8+3ZBknDt37rrH2LJliyHJmD9/fo5t999/vyHJmDt3bq7b7r//fuv7tWvXGpKM2267zUhJSbG2f/HFF4YkY+bMmda2mjVrGn379r3pMW9UW9++fY2aNWta3y9btsyQZEyaNMmm3yOPPGJYLBbjyJEj1jZJhqOjo03bzp07DUnG22+/nWOsf5oxY4Yhyfj000+tbWlpaUZwcLDh5uZmc+41a9Y0wsPDb3i8vPY9fvy4YW9vb0yePNmmfffu3YaDg4NNe9bP7eOPP7a2XblyxfD19TUiIiKsbW+99ZYhyVi2bJm17e+//zbq169vSDLWrl1rbQ8PD7f5vLNk/dwbNGhgXLlyxdo+c+ZMQ5Kxe/duwzAMY/v27YYkY8mSJTf/MP4hLS3N8Pb2Nho1amT8/fff1vYVK1YYkowxY8ZY2/LyZyZ732PHjlnbatasaUgy1q9fb21LSkoynJycjJEjR+bYt3nz5kZaWpq1PTo62pBkfPPNN9Y2ScbYsWNzjJ/9z8CSJUtyfOZ5lf1ckpKSDEdHRyM8PNzIzMy09nv55ZcNSTbj3nnnnXm+RgGUbdxGCADZuLm53XBVwqyZjm+++abAi0k4OTmpf//+ee7/xBNPqFKlStb3jzzyiPz8/PTf//63QOPn1X//+1/Z29tryJAhNu0jR46UYRhauXKlTXtoaKjNzEeTJk3k7u6u33777abj+Pr6qnfv3ta2ChUqaMiQIUpNTdUPP/xQCGeT09dff63MzEz16NFDv//+u/Xl6+urwMDAHLNRbm5uevzxx63vHR0ddffdd9uc36pVq3TbbbfpoYcesrY5Oztfd6b0Rvr372/zHF/WTGTWeFkzV999953++uuvPB9369atSkpK0rPPPitnZ2dre3h4uOrXr6///Oc/+a71RoKCgqy1S9dmI+vVq5frdTFw4ECbZwefeeYZOTg4mH6t38zq1auVlpam5557zmaGLbfFTTw9PbV3714dPny4CCsEUBIRtgAgm9TUVJtgk13Pnj117733asCAAfLx8VGvXr30xRdf5Ct43XbbbflaDCMwMNDmvcViUZ06dUxf0vrEiRPy9/fP8Xlk3Yp14sQJm/bbb789xzEqV66c45mb3MYJDAyUnZ3tP0vXG6ewHD58WIZhKDAwUNWqVbN57d+/37o4RJbq1avnuJUt+/mdOHFCtWvXztGvTp06+a4v++dZuXJlSbKOV6tWLY0YMUIffPCBqlatqrCwML3zzjs3fV4r6/OsV69ejm3169cv9M87P9dF9mvdzc1Nfn5+xb58e9Znkr2+atWqWX8uWSZMmKALFy6obt26aty4sZ5//nnt2rWryGoFUHIQtgDgH06fPq3k5OQb/mLs4uKi9evXa/Xq1erTp4927dqlnj17qkOHDsrIyMjTOPl5ziqvrvc8S15rKgzXW73NyLaYRkmRmZkpi8WiVatWKTY2Nsfrvffes+lf1OeXl/Heeust7dq1Sy+//LL+/vtvDRkyRA0bNtTp06dNqakgiupzK8pr/Ubuu+8+HT16VB999JEaNWqkDz74QHfddZc++OCD4i4NQBEjbAHAP3zyySeSpLCwsBv2s7OzU/v27TVt2jTt27dPkydP1po1a6y3neXnQf68yH47kmEYOnLkiM3qdZUrV9aFCxdy7Jt9liI/tdWsWVNnz57NcVvlgQMHrNsLQ82aNXX48OEcs4OFPU52tWvXlmEYqlWrlkJDQ3O87rnnnnwfs2bNmjp69GiOIJF9FUGp8K6Txo0b69VXX9X69ev1448/6syZM5o7d+4Na5SkgwcP5th28OBB0z7vvMh+raempio+Pv6m13paWpri4+Nt2grzz2HWZ5K9vnPnzuU6Q+fl5aX+/fvr888/16lTp9SkSZNcV1AEULYRtgDg/1uzZo0mTpyoWrVqKTIy8rr9zp8/n6Mt68uBr1y5IklydXWVpFzDT0F8/PHHNoHnyy+/VHx8vHUFPOlacPj5559tVkZbsWJFjiXM81Nbly5dlJGRodmzZ9u0T58+XRaLxWb8W9GlSxclJCRo8eLF1rb09HS9/fbbcnNz0/33318o42TXvXt32dvba/z48TnCkWEY+uOPP/J9zLCwMJ05c0bLly+3tl2+fFnz5s3L0dfV1TVPS7RfT0pKitLT023aGjduLDs7O+u1mJsWLVrI29tbc+fOtem3cuVK7d+/X+Hh4QWu6Va9//77unr1qvX9nDlzlJ6enuNaX79+fY79ss9sFeafw9DQUFWoUEFvv/22zbUyY8aMHH2zXzdubm6qU6fODX8mAMomln4HUC6tXLlSBw4cUHp6uhITE7VmzRrFxsaqZs2aWr58uc2iAdlNmDBB69evV3h4uGrWrKmkpCS9++67ql69ukJCQiRd+2XQ09NTc+fOVaVKleTq6qpWrVqpVq1aBarXy8tLISEh6t+/vxITEzVjxgzVqVPHZtGFAQMG6Msvv1SnTp3Uo0cPHT16VJ9++mmOpbrzU1vXrl3Vrl07vfLKKzp+/LjuvPNOff/99/rmm280bNiwAi0DnpuBAwfqvffeU79+/bRt2zYFBAToyy+/1MaNGzVjxowbPkN3M0eOHNGkSZNytDdr1kzh4eGaNGmSRo8erePHj6tbt26qVKmSjh07pqVLl2rgwIEaNWpUvsZ76qmnNHv2bPXu3VtDhw6Vn5+fFi5caL2m/jnb0rx5cy1evFgjRoxQy5Yt5ebmpq5du+Z5rDVr1mjw4MF69NFHVbduXaWnp+uTTz6Rvb29IiIirrtfhQoV9Oabb6p///66//771bt3b+vS7wEBARo+fHi+zrkwpaWlqX379urRo4cOHjyod999VyEhITYLjgwYMEBPP/20IiIi1KFDB+3cuVPfffedqlatanOspk2byt7eXm+++aaSk5Pl5OSkBx54QN7e3vmuq1q1aho1apSmTJmiBx98UF26dNH27du1cuXKHOMGBQWpbdu2at68uby8vLR161Z9+eWXGjx4cME+FAClV/EsgggAxSNrOeesl6Ojo+Hr62t06NDBmDlzps0S41myL/0eFxdnPPzww4a/v7/h6Oho+Pv7G7179zYOHTpks98333xjBAUFGQ4ODjZLrd9///1Gw4YNc63veku/f/7558bo0aMNb29vw8XFxQgPDzdOnDiRY/+33nrLuO222wwnJyfj3nvvNbZu3ZrjmDeqLfvS74ZhGBcvXjSGDx9u+Pv7GxUqVDACAwONf//73zbLXxvGteW4Bw0alKOm6y1Jn11iYqLRv39/o2rVqoajo6PRuHHjXJenz+/S7//8ef/zFRUVZe331VdfGSEhIYarq6vh6upq1K9f3xg0aJBx8OBBa5/r/dxy+8x+++03Izw83HBxcTGqVatmjBw50vjqq68MScbPP/9s7Zeammo89thjhqenpyHJepysn3v2Jd2PHTtm8/P67bffjP/7v/8zateubTg7OxteXl5Gu3btjNWrV+fp81m8eLHRrFkzw8nJyfDy8jIiIyON06dP2/QpjKXfc/t5Zb8us/b94YcfjIEDBxqVK1c23NzcjMjISOOPP/6w2TcjI8N48cUXjapVqxoVK1Y0wsLCjCNHjuR6rc2bN8+44447DHt7+3wtA5/buWRkZBjjx483/Pz8DBcXF6Nt27bGnj17cow7adIk4+677zY8PT0NFxcXo379+sbkyZNtlrQHUD5YDKOEPrUMAEAZMmPGDA0fPlynT5/WbbfdVtzllDhZX7K8ZcsWtWjRorjLAYBCwTNbAAAUsr///tvm/eXLl/Xee+8pMDCQoAUA5QjPbAEAUMi6d++u22+/XU2bNlVycrI+/fRTHThwQAsXLizu0sq91NRUpaam3rBPtWrVrrtcPQDkB2ELAIBCFhYWpg8++EALFy5URkaGgoKCtGjRIvXs2bO4Syv3pk6dqvHjx9+wz7Fjx2yWmgeAguKZLQAAUG789ttv+u23327YJyQk5IYrkgJAXhG2AAAAAMAELJABAAAAACbgma08yMzM1NmzZ1WpUiWbL6MEAAAAUL4YhqGLFy/K399fdnY3nrsibOXB2bNnVaNGjeIuAwAAAEAJcerUKVWvXv2GfQhbeVCpUiVJ1z5Qd3f3Yq4GAAAAQHFJSUlRjRo1rBnhRghbeZB166C7uzthCwAAAECeHi9igQwAAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMUa9hav369unbtKn9/f1ksFi1btsxmu2EYGjNmjPz8/OTi4qLQ0FAdPnzYps/58+cVGRkpd3d3eXp6KioqSqmpqTZ9du3apTZt2sjZ2Vk1atRQdHS02acGAAAAoJwr1rB16dIl3XnnnXrnnXdy3R4dHa1Zs2Zp7ty52rx5s1xdXRUWFqbLly9b+0RGRmrv3r2KjY3VihUrtH79eg0cONC6PSUlRR07dlTNmjW1bds2/fvf/9a4ceP0/vvvm35+AAAAAMovi2EYRnEXIUkWi0VLly5Vt27dJF2b1fL399fIkSM1atQoSVJycrJ8fHwUExOjXr16af/+/QoKCtKWLVvUokULSdKqVavUpUsXnT59Wv7+/pozZ45eeeUVJSQkyNHRUZL00ksvadmyZTpw4ECeaktJSZGHh4eSk5Pl7u5e+CcPAAAAoFTITzYosc9sHTt2TAkJCQoNDbW2eXh4qFWrVtq0aZMkadOmTfL09LQGLUkKDQ2VnZ2dNm/ebO1z3333WYOWJIWFhengwYP6888/cx37ypUrSklJsXkBAAAAQH44FHcB15OQkCBJ8vHxsWn38fGxbktISJC3t7fNdgcHB3l5edn0qVWrVo5jZG2rXLlyjrGnTJmi8ePHF86JAADKlK5di7uC//n22+KuAABwIyV2Zqs4jR49WsnJydbXqVOnirskAAAAAKVMiQ1bvr6+kqTExESb9sTEROs2X19fJSUl2WxPT0/X+fPnbfrkdox/jpGdk5OT3N3dbV4AAAAAkB8lNmzVqlVLvr6+iouLs7alpKRo8+bNCg4OliQFBwfrwoUL2rZtm7XPmjVrlJmZqVatWln7rF+/XlevXrX2iY2NVb169XK9hRAAAAAACkOxhq3U1FTt2LFDO3bskHRtUYwdO3bo5MmTslgsGjZsmCZNmqTly5dr9+7deuKJJ+Tv729dsbBBgwbq1KmTnnzySf3yyy/auHGjBg8erF69esnf31+S9Nhjj8nR0VFRUVHau3evFi9erJkzZ2rEiBHFdNYAAAAAyoNiXSBj69atateunfV9VgDq27evYmJi9MILL+jSpUsaOHCgLly4oJCQEK1atUrOzs7WfRYuXKjBgwerffv2srOzU0REhGbNmmXd7uHhoe+//16DBg1S8+bNVbVqVY0ZM8bmu7gAAAAAoLCVmO/ZKsn4ni0AQBZWIwSA8q1MfM8WAAAAAJRmhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABCU6bGVkZOi1115TrVq15OLiotq1a2vixIkyDMPaxzAMjRkzRn5+fnJxcVFoaKgOHz5sc5zz588rMjJS7u7u8vT0VFRUlFJTU4v6dAAAAACUIyU6bL355puaM2eOZs+erf379+vNN99UdHS03n77bWuf6OhozZo1S3PnztXmzZvl6uqqsLAwXb582donMjJSe/fuVWxsrFasWKH169dr4MCBxXFKAAAAAMoJi/HPaaIS5sEHH5SPj48+/PBDa1tERIRcXFz06aefyjAM+fv7a+TIkRo1apQkKTk5WT4+PoqJiVGvXr20f/9+BQUFacuWLWrRooUkadWqVerSpYtOnz4tf3//m9aRkpIiDw8PJScny93d3ZyTBQCUCl27FncF//Ptt8VdAQCUP/nJBiV6Zqt169aKi4vToUOHJEk7d+7Uhg0b1LlzZ0nSsWPHlJCQoNDQUOs+Hh4eatWqlTZt2iRJ2rRpkzw9Pa1BS5JCQ0NlZ2enzZs35zrulStXlJKSYvMCAAAAgPxwKO4CbuSll15SSkqK6tevL3t7e2VkZGjy5MmKjIyUJCUkJEiSfHx8bPbz8fGxbktISJC3t7fNdgcHB3l5eVn7ZDdlyhSNHz++sE8HAAAAQDlSome2vvjiCy1cuFCfffaZfv31Vy1YsEBTp07VggULTB139OjRSk5Otr5OnTpl6ngAAAAAyp4SPbP1/PPP66WXXlKvXr0kSY0bN9aJEyc0ZcoU9e3bV76+vpKkxMRE+fn5WfdLTExU06ZNJUm+vr5KSkqyOW56errOnz9v3T87JycnOTk5mXBGAAAAAMqLEj2z9ddff8nOzrZEe3t7ZWZmSpJq1aolX19fxcXFWbenpKRo8+bNCg4OliQFBwfrwoUL2rZtm7XPmjVrlJmZqVatWhXBWQAAAAAoj0r0zFbXrl01efJk3X777WrYsKG2b9+uadOm6f/+7/8kSRaLRcOGDdOkSZMUGBioWrVq6bXXXpO/v7+6desmSWrQoIE6deqkJ598UnPnztXVq1c1ePBg9erVK08rEQIAAABAQZTosPX222/rtdde07PPPqukpCT5+/vrqaee0pgxY6x9XnjhBV26dEkDBw7UhQsXFBISolWrVsnZ2dnaZ+HChRo8eLDat28vOzs7RUREaNasWcVxSgAAAADKiRL9PVslBd+zBQDIwvdsAUD5Vma+ZwsAAAAASivCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGCCEh+2zpw5o8cff1xVqlSRi4uLGjdurK1bt1q3G4ahMWPGyM/PTy4uLgoNDdXhw4dtjnH+/HlFRkbK3d1dnp6eioqKUmpqalGfCgAAAIBypESHrT///FP33nuvKlSooJUrV2rfvn166623VLlyZWuf6OhozZo1S3PnztXmzZvl6uqqsLAwXb582donMjJSe/fuVWxsrFasWKH169dr4MCBxXFKAAAAAMoJi2EYRnEXcT0vvfSSNm7cqB9//DHX7YZhyN/fXyNHjtSoUaMkScnJyfLx8VFMTIx69eql/fv3KygoSFu2bFGLFi0kSatWrVKXLl10+vRp+fv737SOlJQUeXh4KDk5We7u7oV3ggCAUqdr1+Ku4H++/ba4KwCA8ic/2aBEz2wtX75cLVq00KOPPipvb281a9ZM8+bNs24/duyYEhISFBoaam3z8PBQq1attGnTJknSpk2b5OnpaQ1akhQaGio7Oztt3rw513GvXLmilJQUmxcAAAAA5EeBwtZvv/1W2HVcd5w5c+YoMDBQ3333nZ555hkNGTJECxYskCQlJCRIknx8fGz28/HxsW5LSEiQt7e3zXYHBwd5eXlZ+2Q3ZcoUeXh4WF81atQo7FMDAAAAUMYVKGzVqVNH7dq106effmrzbFRhy8zM1F133aXXX39dzZo108CBA/Xkk09q7ty5po0pSaNHj1ZycrL1derUKVPHAwAAAFD2FChs/frrr2rSpIlGjBghX19fPfXUU/rll18Kuzb5+fkpKCjIpq1BgwY6efKkJMnX11eSlJiYaNMnMTHRus3X11dJSUk229PT03X+/Hlrn+ycnJzk7u5u8wIAAACA/ChQ2GratKlmzpyps2fP6qOPPlJ8fLxCQkLUqFEjTZs2TefOnSuU4u69914dPHjQpu3QoUOqWbOmJKlWrVry9fVVXFycdXtKSoo2b96s4OBgSVJwcLAuXLigbdu2WfusWbNGmZmZatWqVaHUCQAAAADZ3dICGQ4ODurevbuWLFmiN998U0eOHNGoUaNUo0YNPfHEE4qPj7+l4oYPH66ff/5Zr7/+uo4cOaLPPvtM77//vgYNGiRJslgsGjZsmCZNmqTly5dr9+7deuKJJ+Tv769u3bpJujYT1qlTJz355JP65ZdftHHjRg0ePFi9evXK00qEAAAAAFAQtxS2tm7dqmeffVZ+fn6aNm2aRo0apaNHjyo2NlZnz57Vww8/fEvFtWzZUkuXLtXnn3+uRo0aaeLEiZoxY4YiIyOtfV544QU999xzGjhwoFq2bKnU1FStWrVKzs7O1j4LFy5U/fr11b59e3Xp0kUhISF6//33b6k2AAAAALiRAn3P1rRp0zR//nwdPHhQXbp00YABA9SlSxfZ2f0vu50+fVoBAQFKT08v1IKLA9+zBQDIwvdsAUD5lp9s4FCQAebMmaP/+7//U79+/eTn55drH29vb3344YcFOTwAAAAAlHoFCluHDx++aR9HR0f17du3IIcHAAAAgFKvQM9szZ8/X0uWLMnRvmTJEusXDgMAAABAeVagsDVlyhRVrVo1R7u3t7def/31Wy4KAAAAAEq7AoWtkydPqlatWjnaa9asaf3CYQAAAAAozwoUtry9vbVr164c7Tt37lSVKlVuuSgAAAAAKO0KFLZ69+6tIUOGaO3atcrIyFBGRobWrFmjoUOHqlevXoVdIwAAAACUOgVajXDixIk6fvy42rdvLweHa4fIzMzUE088wTNbAAAAAKAChi1HR0ctXrxYEydO1M6dO+Xi4qLGjRurZs2ahV0fAAAAAJRKBQpbWerWrau6desWVi0AAAAAUGYUKGxlZGQoJiZGcXFxSkpKUmZmps32NWvWFEpxAAAAAFBaFShsDR06VDExMQoPD1ejRo1ksVgKuy4AAAAAKNUKFLYWLVqkL774Ql26dCnsegAAAACgTCjQ0u+Ojo6qU6dOYdcCAAAAAGVGgcLWyJEjNXPmTBmGUdj1AAAAAECZUKDbCDds2KC1a9dq5cqVatiwoSpUqGCz/euvvy6U4gAAAACgtCpQ2PL09NS//vWvwq4FAAAAAMqMAoWt+fPnF3YdAAAAAFCmFOiZLUlKT0/X6tWr9d577+nixYuSpLNnzyo1NbXQigMAAACA0qpAM1snTpxQp06ddPLkSV25ckUdOnRQpUqV9Oabb+rKlSuaO3duYdcJAAAAAKVKgWa2hg4dqhYtWujPP/+Ui4uLtf1f//qX4uLiCq04AAAAACitCjSz9eOPP+qnn36So6OjTXtAQIDOnDlTKIUBAAAAQGlWoJmtzMxMZWRk5Gg/ffq0KlWqdMtFAQAAAEBpV6Cw1bFjR82YMcP63mKxKDU1VWPHjlWXLl0KqzYAAAAAKLUKdBvhW2+9pbCwMAUFBeny5ct67LHHdPjwYVWtWlWff/55YdcIAAAAAKVOgcJW9erVtXPnTi1atEi7du1SamqqoqKiFBkZabNgBgAAAACUVwUKW5Lk4OCgxx9/vDBrAQAAAIAyo0Bh6+OPP77h9ieeeKJAxQAAAABAWVGgsDV06FCb91evXtVff/0lR0dHVaxYkbAFAAAAoNwr0GqEf/75p80rNTVVBw8eVEhICAtkAAAAAIAKGLZyExgYqDfeeCPHrBcAAAAAlEeFFraka4tmnD17tjAPCQAAAAClUoGe2Vq+fLnNe8MwFB8fr9mzZ+vee+8tlMIAAAAAoDQrUNjq1q2bzXuLxaJq1arpgQce0FtvvVUYdQEAAABAqVagsJWZmVnYdQAAAABAmVKoz2wBAAAAAK4p0MzWiBEj8tx32rRpBRkCAAAAAEq1AoWt7du3a/v27bp69arq1asnSTp06JDs7e111113WftZLJbCqRIAAAAASpkCha2uXbuqUqVKWrBggSpXrizp2hcd9+/fX23atNHIkSMLtUgAAAAAKG0shmEY+d3ptttu0/fff6+GDRvatO/Zs0cdO3Ysc9+1lZKSIg8PDyUnJ8vd3b24ywEAFKOuXYu7gv/59tvirgAAyp/8ZIMCLZCRkpKic+fO5Wg/d+6cLl68WJBDAgAAAECZUqCw9a9//Uv9+/fX119/rdOnT+v06dP66quvFBUVpe7duxd2jQAAAABQ6hToma25c+dq1KhReuyxx3T16tVrB3JwUFRUlP79738XaoEAAAAAUBoV6JmtLJcuXdLRo0clSbVr15arq2uhFVaS8MwWACALz2wBQPlm+jNbWeLj4xUfH6/AwEC5urrqFnIbAAAAAJQpBQpbf/zxh9q3b6+6deuqS5cuio+PlyRFRUWx7DsAAAAAqIBha/jw4apQoYJOnjypihUrWtt79uypVatWFVpxAAAAAFBaFWiBjO+//17fffedqlevbtMeGBioEydOFEphAAAAAFCaFWhm69KlSzYzWlnOnz8vJyenWy4KAAAAAEq7AoWtNm3a6OOPP7a+t1gsyszMVHR0tNq1a1doxQEAAABAaVWg2wijo6PVvn17bd26VWlpaXrhhRe0d+9enT9/Xhs3bizsGgEAAACg1CnQzFajRo106NAhhYSE6OGHH9alS5fUvXt3bd++XbVr1y7sGgEAAACg1Mn3zNbVq1fVqVMnzZ07V6+88ooZNQEAAABAqZfvma0KFSpo165dZtQCAAAAAGVGgW4jfPzxx/Xhhx8Wdi0AAAAAUGYUaIGM9PR0ffTRR1q9erWaN28uV1dXm+3Tpk0rlOIAAAAAoLTKV9j67bffFBAQoD179uiuu+6SJB06dMimj8ViKbzqAAAAAKCUylfYCgwMVHx8vNauXStJ6tmzp2bNmiUfHx9TigMAAACA0ipfz2wZhmHzfuXKlbp06VKhFgQAAAAAZUGBFsjIkj18AQAAAACuyVfYslgsOZ7J4hktAAAAAMgpX89sGYahfv36ycnJSZJ0+fJlPf300zlWI/z6668Lr0IAAAAAKIXyFbb69u1r8/7xxx8v1GIAAAAAoKzIV9iaP3++WXUAAAAAQJlySwtkAAAAAAByR9gCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMUKrC1htvvCGLxaJhw4ZZ2y5fvqxBgwapSpUqcnNzU0REhBITE232O3nypMLDw1WxYkV5e3vr+eefV3p6ehFXDwAAAKA8KTVha8uWLXrvvffUpEkTm/bhw4fr22+/1ZIlS/TDDz/o7Nmz6t69u3V7RkaGwsPDlZaWpp9++kkLFixQTEyMxowZU9SnAAAAAKAcKRVhKzU1VZGRkZo3b54qV65sbU9OTtaHH36oadOm6YEHHlDz5s01f/58/fTTT/r5558lSd9//7327dunTz/9VE2bNlXnzp01ceJEvfPOO0pLSyuuUwIAAABQxpWKsDVo0CCFh4crNDTUpn3btm26evWqTXv9+vV1++23a9OmTZKkTZs2qXHjxvLx8bH2CQsLU0pKivbu3ZvreFeuXFFKSorNCwAAAADyw6G4C7iZRYsW6ddff9WWLVtybEtISJCjo6M8PT1t2n18fJSQkGDt88+glbU9a1tupkyZovHjxxdC9QAAAADKqxI9s3Xq1CkNHTpUCxculLOzc5GNO3r0aCUnJ1tfp06dKrKxAQAAAJQNJTpsbdu2TUlJSbrrrrvk4OAgBwcH/fDDD5o1a5YcHBzk4+OjtLQ0XbhwwWa/xMRE+fr6SpJ8fX1zrE6Y9T6rT3ZOTk5yd3e3eQEAAABAfpTosNW+fXvt3r1bO3bssL5atGihyMhI6/+vUKGC4uLirPscPHhQJ0+eVHBwsCQpODhYu3fvVlJSkrVPbGys3N3dFRQUVOTnBAAAAKB8KNHPbFWqVEmNGjWyaXN1dVWVKlWs7VFRURoxYoS8vLzk7u6u5557TsHBwbrnnnskSR07dlRQUJD69Omj6OhoJSQk6NVXX9WgQYPk5ORU5OcEAAAAoHwo0WErL6ZPny47OztFREToypUrCgsL07vvvmvdbm9vrxUrVuiZZ55RcHCwXF1d1bdvX02YMKEYqwYAAABQ1lkMwzCKu4iSLiUlRR4eHkpOTub5LQAo57p2Le4K/ufbb4u7AgAof/KTDUr0M1sAAAAAUFoRtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAAAAExC2AAAAAMAEhC0AAAAAMAFhCwAAAABMQNgCAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATlOiwNWXKFLVs2VKVKlWSt7e3unXrpoMHD9r0uXz5sgYNGqQqVarIzc1NERERSkxMtOlz8uRJhYeHq2LFivL29tbzzz+v9PT0ojwVAAAAAOVMiQ5bP/zwgwYNGqSff/5ZsbGxunr1qjp27KhLly5Z+wwfPlzffvutlixZoh9++EFnz55V9+7drdszMjIUHh6utLQ0/fTTT1qwYIFiYmI0ZsyY4jglAAAAAOWExTAMo7iLyKtz587J29tbP/zwg+677z4lJyerWrVq+uyzz/TII49Ikg4cOKAGDRpo06ZNuueee7Ry5Uo9+OCDOnv2rHx8fCRJc+fO1Ysvvqhz587J0dHxpuOmpKTIw8NDycnJcnd3N/UcAQAlW9euxV3B/3z7bXFXAADlT36yQYme2couOTlZkuTl5SVJ2rZtm65evarQ0FBrn/r16+v222/Xpk2bJEmbNm1S48aNrUFLksLCwpSSkqK9e/fmOs6VK1eUkpJi8wIAAACA/Cg1YSszM1PDhg3Tvffeq0aNGkmSEhIS5OjoKE9PT5u+Pj4+SkhIsPb5Z9DK2p61LTdTpkyRh4eH9VWjRo1CPhsAAAAAZV2pCVuDBg3Snj17tGjRItPHGj16tJKTk62vU6dOmT4mAAAAgLLFobgLyIvBgwdrxYoVWr9+vapXr25t9/X1VVpami5cuGAzu5WYmChfX19rn19++cXmeFmrFWb1yc7JyUlOTk6FfBYAAAAAypMSPbNlGIYGDx6spUuXas2aNapVq5bN9ubNm6tChQqKi4uzth08eFAnT55UcHCwJCk4OFi7d+9WUlKStU9sbKzc3d0VFBRUNCcCAAAAoNwp0TNbgwYN0meffaZvvvlGlSpVsj5j5eHhIRcXF3l4eCgqKkojRoyQl5eX3N3d9dxzzyk4OFj33HOPJKljx44KCgpSnz59FB0drYSEBL366qsaNGgQs1cAAAAATFOiw9acOXMkSW3btrVpnz9/vvr16ydJmj59uuzs7BQREaErV64oLCxM7777rrWvvb29VqxYoWeeeUbBwcFydXVV3759NWHChKI6DQAAAADlUKn6nq3iwvdsAQCy8D1bAFC+ldnv2QIAAACA0oKwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJiAsAUAAAAAJiBsAQAAAIAJCFsAAAAAYALCFgAAAACYgLAFAAAAACYgbAEAAACACQhbAAAAAGACwhYAAAAAmICwBQAAAAAmIGwBAAAAgAkIWwAAAABgAsIWAAAAAJigXIWtd955RwEBAXJ2dlarVq30yy+/FHdJAAAAAMqochO2Fi9erBEjRmjs2LH69ddfdeeddyosLExJSUnFXRoAAACAMqjchK1p06bpySefVP/+/RUUFKS5c+eqYsWK+uijj4q7NAAAAABlkENxF1AU0tLStG3bNo0ePdraZmdnp9DQUG3atClH/ytXrujKlSvW98nJyZKklJQU84sFAJRoV68WdwX/wz9LAFD0sjKBYRg37Vsuwtbvv/+ujIwM+fj42LT7+PjowIEDOfpPmTJF48ePz9Feo0YN02oEACC/PDyKuwIAKL8uXrwoj5v8RVwuwlZ+jR49WiNGjLC+z8zM1Pnz51WlShVZLJZirAw3kpKSoho1aujUqVNyd3cv7nJQCnDNIL+4ZpBfXDPIL66Zks8wDF28eFH+/v437VsuwlbVqlVlb2+vxMREm/bExET5+vrm6O/k5CQnJyebNk9PTzNLRCFyd3fnLyfkC9cM8otrBvnFNYP84pop2W42o5WlXCyQ4ejoqObNmysuLs7alpmZqbi4OAUHBxdjZQAAAADKqnIxsyVJI0aMUN++fdWiRQvdfffdmjFjhi5duqT+/fsXd2kAAAAAyqByE7Z69uypc+fOacyYMUpISFDTpk21atWqHItmoPRycnLS2LFjc9wCClwP1wzyi2sG+cU1g/zimilbLEZe1iwEAAAAAORLuXhmCwAAAACKGmELAAAAAExA2AIAAAAAExC2AAAAAMAEhC2UaufPn1dkZKTc3d3l6empqKgopaam5mlfwzDUuXNnWSwWLVu2zNxCUWLk95o5f/68nnvuOdWrV08uLi66/fbbNWTIECUnJxdh1ShK77zzjgICAuTs7KxWrVrpl19+uWH/JUuWqH79+nJ2dlbjxo313//+t4gqRUmRn2tm3rx5atOmjSpXrqzKlSsrNDT0ptcYyp78/j2TZdGiRbJYLOrWrZu5BaLQELZQqkVGRmrv3r2KjY3VihUrtH79eg0cODBP+86YMUMWi8XkClHS5PeaOXv2rM6ePaupU6dqz549iomJ0apVqxQVFVWEVaOoLF68WCNGjNDYsWP166+/6s4771RYWJiSkpJy7f/TTz+pd+/eioqK0vbt29WtWzd169ZNe/bsKeLKUVzye82sW7dOvXv31tq1a7Vp0ybVqFFDHTt21JkzZ4q4chSX/F4zWY4fP65Ro0apTZs2RVQpCoUBlFL79u0zJBlbtmyxtq1cudKwWCzGmTNnbrjv9u3bjdtuu82Ij483JBlLly41uVqUBLdyzfzTF198YTg6OhpXr141o0wUo7vvvtsYNGiQ9X1GRobh7+9vTJkyJdf+PXr0MMLDw23aWrVqZTz11FOm1omSI7/XTHbp6elGpUqVjAULFphVIkqYglwz6enpRuvWrY0PPvjA6Nu3r/Hwww8XQaUoDMxsodTatGmTPD091aJFC2tbaGio7OzstHnz5uvu99dff+mxxx7TO++8I19f36IoFSVEQa+Z7JKTk+Xu7i4Hh3LzvfDlQlpamrZt26bQ0FBrm52dnUJDQ7Vp06Zc99m0aZNNf0kKCwu7bn+ULQW5ZrL766+/dPXqVXl5eZlVJkqQgl4zEyZMkLe3N3dVlEL8poBSKyEhQd7e3jZtDg4O8vLyUkJCwnX3Gz58uFq3bq2HH37Y7BJRwhT0mvmn33//XRMnTszz7aooPX7//XdlZGTIx8fHpt3Hx0cHDhzIdZ+EhIRc++f1ekLpVpBrJrsXX3xR/v7+OUI7yqaCXDMbNmzQhx9+qB07dhRBhShszGyhxHnppZdksVhu+MrrP2LZLV++XGvWrNGMGTMKt2gUKzOvmX9KSUlReHi4goKCNG7cuFsvHEC59sYbb2jRokVaunSpnJ2di7sclEAXL15Unz59NG/ePFWtWrW4y0EBMLOFEmfkyJHq16/fDfvccccd8vX1zfEwaXp6us6fP3/d2wPXrFmjo0ePytPT06Y9IiJCbdq00bp1626hchQXM6+ZLBcvXlSnTp1UqVIlLV26VBUqVLjVslHCVK1aVfb29kpMTLRpT0xMvO714evrm6/+KFsKcs1kmTp1qt544w2tXr1aTZo0MbNMlCD5vWaOHj2q48ePq2vXrta2zMxMSdfuzDh48KBq165tbtG4JYQtlDjVqlVTtWrVbtovODhYFy5c0LZt29S8eXNJ18JUZmamWrVqles+L730kgYMGGDT1rhxY02fPt3mLzKULmZeM9K1Ga2wsDA5OTlp+fLl/BfoMsrR0VHNmzdXXFycdVnlzMxMxcXFafDgwbnuExwcrLi4OA0bNszaFhsbq+Dg4CKoGMWtINeMJEVHR2vy5Mn67rvvbJ4hRdmX32umfv362r17t03bq6++qosXL2rmzJmqUaNGUZSNW1HcK3QAt6JTp05Gs2bNjM2bNxsbNmwwAgMDjd69e1u3nz592qhXr56xefPm6x5DrEZYruT3mklOTjZatWplNG7c2Dhy5IgRHx9vfaWnpxfXacAkixYtMpycnIyYmBhj3759xsCBAw1PT08jISHBMAzD6NOnj/HSSy9Z+2/cuNFwcHAwpk6dauzfv98YO3asUaFCBWP37t3FdQooYvm9Zt544w3D0dHR+PLLL23+Prl48WJxnQKKWH6vmexYjbB0YWYLpdrChQs1ePBgtW/fXnZ2doqIiNCsWbOs269evaqDBw/qr7/+KsYqUZLk95r59ddfrSsV1qlTx+ZYx44dU0BAQJHVDvP17NlT586d05gxY5SQkKCmTZtq1apV1ofZT548KTu7/z3u3Lp1a3322Wd69dVX9fLLLyswMFDLli1To0aNiusUUMTye83MmTNHaWlpeuSRR2yOM3bsWJ4FLSfye82gdLMYhmEUdxEAAAAAUNYQmwEAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwAWELAAAAAExA2AIAAABQJhw/flxRUVGqVauWXFxcVLt2bY0dO1ZpaWn5Os64ceNUv359ubq6qnLlygoNDdXmzZvzXQ9hCwBQJvTr10/dunUr9OMmJCSoQ4cOcnV1laenZ5GObYaAgADNmDHjhn0sFouWLVtWJPUAQEG0bdtWMTExOdoPHDigzMxMvffee9q7d6+mT5+uuXPn6uWXX87X8evWravZs2dr9+7d2rBhgwICAtSxY0edO3cuX8chbAEA8qwkhIrjx4/LYrFox44dRTLe9OnTFR8frx07dujQoUO59pk5c2au/+ibLSYm5roB8Hq2bNmigQMHmlMQABSzTp06af78+erYsaPuuOMOPfTQQxo1apS+/vprm34bNmxQmzZt5OLioho1amjIkCG6dOmSdftjjz2m0NBQ3XHHHWrYsKGmTZumlJQU7dq1K1/1ELYAALiBo0ePqnnz5goMDJS3t3eufTw8PPIdeopLtWrVVLFixeIuAwCKTHJysry8vKzvjx49qk6dOikiIkK7du3S4sWLtWHDBg0ePDjX/dPS0vT+++/Lw8NDd955Z77GJmwBAArNnj171LlzZ7m5ucnHx0d9+vTR77//bt3etm1bDRkyRC+88IK8vLzk6+urcePG2RzjwIEDCgkJkbOzs4KCgrR69Wqb29pq1aolSWrWrJksFovatm1rs//UqVPl5+enKlWqaNCgQbp69eoNa54zZ45q164tR0dH1atXT5988ol1W0BAgL766it9/PHHslgs6tevX67HyD7jl5fztFgsmjNnjjp37iwXFxfdcccd+vLLL63b161bJ4vFogsXLljbduzYIYvFouPHj2vdunXq37+/kpOTZbFYZLFYcoyRm+y3ER4+fFj33Xef9fOOjY216Z+WlqbBgwfLz89Pzs7OqlmzpqZMmXLTcQCgJDhy5IjefvttPfXUU9a2KVOmKDIyUsOGDVNgYKBat26tWbNm6eOPP9bly5et/VasWCE3Nzc5Oztr+vTpio2NVdWqVfM1PmELAFAoLly4oAceeEDNmjXT1q1btWrVKiUmJqpHjx42/RYsWCBXV1dt3rxZ0dHRmjBhgvUX/IyMDHXr1k0VK1bU5s2b9f777+uVV16x2f+XX36RJK1evVrx8fE2t4asXbtWR48e1dq1a7VgwQLFxMTc8Pa+pUuXaujQoRo5cqT27Nmjp556Sv3799fatWslXbvlrlOnTurRo4fi4+M1c+bMPH8eNzrPLK+99poiIiK0c+dORUZGqlevXtq/f3+ejt+6dWvNmDFD7u7uio+PV3x8vEaNGpXn+iQpMzNT3bt3l6OjozZv3qy5c+fqxRdftOkza9YsLV++XF988YUOHjyohQsXKiAgIF/jAMCtev311+Xm5mZ9/fjjj3r66adt2k6ePGmzz5kzZ9SpUyc9+uijevLJJ63tO3fuVExMjM2+YWFhyszM1LFjx6z92rVrpx07duinn36y/luQlJSUr7odbu20AQC4Zvbs2WrWrJlef/11a9tHH32kGjVq6NChQ6pbt64kqUmTJho7dqwkKTAwULNnz1ZcXJw6dOig2NhYHT16VOvWrZOvr68kafLkyerQoYP1mNWqVZMkValSxdonS+XKlTV79mzZ29urfv36Cg8PV1xcnM0/sv80depU9evXT88++6wkacSIEfr55581depUtWvXTtWqVZOTk5NcXFxyjHUzNzrPLI8++qgGDBggSZo4caJiY2P19ttv6913373p8R0dHeXh4SGLxZLv2rKsXr1aBw4c0HfffSd/f39J136h6dy5s7XPyZMnFRgYqJCQEFksFtWsWbNAYwHArXj66adt/uNdZGSkIiIi1L17d2tb1t9jknT27Fm1a9dOrVu31vvvv29zrNTUVD311FMaMmRIjnFuv/126/93dXVVnTp1VKdOHd1zzz0KDAzUhx9+qNGjR+e5bsIWAKBQ7Ny5U2vXrpWbm1uObUePHrUJW//k5+dn/S+FBw8eVI0aNWzCw913353nGho2bCh7e3ubY+/evfu6/ffv359jsYh77703XzNY13Oj88wSHByc431RLfwhXTv/GjVq2PyCkr2mfv36qUOHDqpXr546deqkBx98UB07diyyGgFAkry8vGyeu3JxcZG3t7fq1KmTo++ZM2fUrl07NW/eXPPnz5edne3NfHfddZf27duX6743kpmZqStXruRrH8IWAKBQpKamqmvXrnrzzTdzbPPz87P+/woVKthss1gsyszMLJQazDx2UdeS9cuBYRjWtps9f2aGu+66S8eOHdPKlSu1evVq9ejRQ6GhoTbPlwFASXHmzBm1bdtWNWvW1NSpU22Was/6D3kvvvii7rnnHg0ePFgDBgyQq6ur9u3bp9jYWM2ePVuXLl3S5MmT9dBDD8nPz0+///673nnnHZ05c0aPPvpovurhmS0AQKG46667tHfvXgUEBFhvu8h6ubq65ukY9erV06lTp5SYmGht27Jli00fR0dHSdee77pVDRo00MaNG23aNm7cqKCgoFs+dl78/PPPOd43aNBA0v9ul4yPj7duzz7r5ejoeEufQ4MGDXTq1CmbMbLXJEnu7u7q2bOn5s2bp8WLF+urr77S+fPnCzwuAJglNjZWR44cUVxcnKpXry4/Pz/rK0uTJk30ww8/6NChQ2rTpo2aNWumMWPGWGf57e3tdeDAAUVERKhu3brq2rWr/vjjD/34449q2LBhvuphZgsAkC/Jyck5funPWvlv3rx56t27t3UVviNHjmjRokX64IMPbG7vu54OHTqodu3a6tu3r6Kjo3Xx4kW9+uqrkq7NDEmSt7e3XFxctGrVKlWvXl3Ozs7y8PAo0Lk8//zz6tGjh5o1a6bQ0FB9++23+vrrr7V69eoCHS+/lixZohYtWigkJEQLFy7UL7/8og8//FCSVKdOHdWoUUPjxo3T5MmTdejQIb311ls2+wcEBCg1NVVxcXG68847VbFixXwt6x4aGqq6deuqb9+++ve//62UlJQcC5JMmzZNfn5+atasmezs7LRkyRL5+vqWmqXuAZRN69aty7W9X79+11059p9atmyp77//Ptdtzs7OOb6Xq6CY2QIA5Mu6devUrFkzm9f48ePl7++vjRs3KiMjQx07dlTjxo01bNgweXp65rhf/nrs7e21bNkypaamqmXLlhowYID1l39nZ2dJkoODg2bNmqX33ntP/v7+evjhhwt8Lt26ddPMmTM1depUNWzYUO+9957mz5+fYzl5s4wfP16LFi1SkyZN9PHHH+vzzz+3zqpVqFBBn3/+uQ4cOKAmTZrozTff1KRJk2z2b926tZ5++mn17NlT1apVU3R0dL7Gt7Oz09KlS/X333/r7rvv1oABAzR58mSbPpUqVVJ0dLRatGihli1b6vjx4/rvf/+b558pAJRnFuOfN4MDAFDCbNy4USEhITpy5Ihq165d3OUUGovFoqVLl9p8PxcAoGzhNkIAQImydOlSubm5KTAwUEeOHNHQoUN17733lqmgBQAoHwhbAIAS5eLFi3rxxRd18uRJVa1aVaGhoTmeVULufvzxR5vvyMouNTW1CKsBAHAbIQAAZcTff/+tM2fOXHd7fr9TBgBwawhbAAAAAGAClhICAAAAABMQtgAAAADABIQtAAAAADABYQsAAAAATEDYAgAAAAATELYAAAAAwASELQAAAAAwwf8DJh7RLCu/DF4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_data_lengths(tokenized_train_dataset, tokenized_val_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jP3R4enP3m19"
   },
   "source": [
    "### How does the base model do?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Vxbl4ACsyRgi"
   },
   "source": [
    "Optionally, you can check how Mistral does on one of your data samples. For example, if you have a dataset of users' biometric data to their health scores, you could test the following `eval_prompt`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "gOxnx-cAyRgi"
   },
   "outputs": [],
   "source": [
    "eval_prompt = \"\"\" Given the following biometric data, score the users' health, from 0-100.\n",
    "\n",
    "### Biometric Data:\n",
    "Temperature=98.2,\n",
    "Sex=F,\n",
    "Age=29,\n",
    "Height=69 inches,\n",
    "Weight=160 lbs,\n",
    "V02_Max=55,\n",
    "HRV=55\n",
    "\n",
    "### Health Score:\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KRhfq_Fa3m19"
   },
   "source": [
    "The `eval_prompt` I used was:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pa6ux9ni3m19"
   },
   "outputs": [],
   "source": [
    "eval_prompt = \" The following is a note by Eevee the Dog: # \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "NidIuFXMyRgi",
    "outputId": "b1794b11-9a22-4b0a-e871-7df039ab59fc"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Given the following biometric data, score the users' health, from 0-100.\n",
      "\n",
      "### Biometric Data:\n",
      "Temperature=98.2,\n",
      "Sex=F,\n",
      "Age=29,\n",
      "Height=69 inches,\n",
      "Weight=160 lbs,\n",
      "V02_Max=55,\n",
      "HRV=55\n",
      "\n",
      "### Health Score:\n",
      "Health Score = 73\n",
      "\n",
      "## How to run this code?\n",
      "\n",
      "You can use any IDE of your choice and run it locally on your machine.\n",
      "\n",
      "## What is the output?\n",
      "\n",
      "The output will be a number between 0 - 100 which represents the user's health.\n",
      "\n",
      "## Why did we choose these features?\n",
      "\n",
      "We chose these features because they are easily accessible and have been shown to be good predictors of health in previous studies.\n",
      "\n",
      "## What does each feature represent?\n",
      "\n",
      "* Temperature: The temperature of the body. A higher temperature may indicate an infection or fever.\n",
      "* Sex: Whether the person is male or female. Males tend to have lower HRV than females.\n",
      "* Age: The age of the person. Older people tend to have lower VO2 max and HR than younger people.\n",
      "* Height: The height of the person. Taller people tend to have lower BMI than shorter people.\n",
      "* Weight: The weight of the person. People with higher weights tend to have lower VO2 max and HR than those with lower weights.\n",
      "* VO2 Max: The maximum amount of oxygen that the body can consume during exercise. Higher values indicate better cardiovascular fitness.\n",
      "* HRV: Heart rate variability. Lower values indicate worse heart health.\n"
     ]
    }
   ],
   "source": [
    "# Re-init the tokenizer so it doesn't add padding or eos token\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    base_model_id,\n",
    "    add_bos_token=True,\n",
    ")\n",
    "\n",
    "model_input = tokenizer(eval_prompt, return_tensors=\"pt\").to(\"cuda\")\n",
    "\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    print(tokenizer.decode(model.generate(**model_input, max_new_tokens=500, repetition_penalty=1.15)[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dCAWeCzZyRgi"
   },
   "source": [
    "Observe how the model does out of the box."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AapDoyfAyRgi"
   },
   "source": [
    "### 4. Set Up LoRA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Mp2gMi1ZzGET"
   },
   "source": [
    "Now, to start our fine-tuning, we have to apply some preprocessing to the model to prepare it for training. For that use the `prepare_model_for_kbit_training` method from PEFT."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "a9EUEDAl0ss3"
   },
   "outputs": [],
   "source": [
    "from peft import prepare_model_for_kbit_training\n",
    "\n",
    "model.gradient_checkpointing_enable()\n",
    "model = prepare_model_for_kbit_training(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "gkIcwsSU01EB"
   },
   "outputs": [],
   "source": [
    "def print_trainable_parameters(model):\n",
    "    \"\"\"\n",
    "    Prints the number of trainable parameters in the model.\n",
    "    \"\"\"\n",
    "    trainable_params = 0\n",
    "    all_param = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_param += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_params += param.numel()\n",
    "    print(\n",
    "        f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cUYEpEK-yRgj"
   },
   "source": [
    "Let's print the model to examine its layers, as we will apply QLoRA to all the linear layers of the model. Those layers are `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, `down_proj`, and `lm_head`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "XshGNsbxyRgj",
    "outputId": "c619b0e8-8516-4d4b-9abe-13eaa3f3b204",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LlamaForCausalLM(\n",
      "  (model): LlamaModel(\n",
      "    (embed_tokens): Embedding(32000, 4096)\n",
      "    (layers): ModuleList(\n",
      "      (0-31): 32 x LlamaDecoderLayer(\n",
      "        (self_attn): LlamaAttention(\n",
      "          (q_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
      "          (k_proj): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
      "          (v_proj): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
      "          (o_proj): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
      "          (rotary_emb): LlamaRotaryEmbedding()\n",
      "        )\n",
      "        (mlp): LlamaMLP(\n",
      "          (gate_proj): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
      "          (up_proj): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
      "          (down_proj): Linear4bit(in_features=14336, out_features=4096, bias=False)\n",
      "          (act_fn): SiLU()\n",
      "        )\n",
      "        (input_layernorm): LlamaRMSNorm()\n",
      "        (post_attention_layernorm): LlamaRMSNorm()\n",
      "      )\n",
      "    )\n",
      "    (norm): LlamaRMSNorm()\n",
      "  )\n",
      "  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I6mTLuQJyRgj"
   },
   "source": [
    "Here we define the LoRA config.\n",
    "\n",
    "`r` is the rank of the low-rank matrix used in the adapters, which thus controls the number of parameters trained. A higher rank will allow for more expressivity, but there is a compute tradeoff.\n",
    "\n",
    "`alpha` is the scaling factor for the learned weights. The weight matrix is scaled by `alpha/r`, and thus a higher value for `alpha` assigns more weight to the LoRA activations.\n",
    "\n",
    "The values used in the QLoRA paper were `r=64` and `lora_alpha=16`, and these are said to generalize well, but we will use `r=32` and `lora_alpha=64` so that we have more emphasis on the new fine-tuned data while also reducing computational complexity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "Ybeyl20n3dYH",
    "outputId": "6a16c182-04d9-4812-ae81-502a8fe364d0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 85041152 || all params: 3837112320 || trainable%: 2.2162799758751914\n"
     ]
    }
   ],
   "source": [
    "from peft import LoraConfig, get_peft_model\n",
    "\n",
    "config = LoraConfig(\n",
    "    r=32,\n",
    "    lora_alpha=64,\n",
    "    target_modules=[\n",
    "        \"q_proj\",\n",
    "        \"k_proj\",\n",
    "        \"v_proj\",\n",
    "        \"o_proj\",\n",
    "        \"gate_proj\",\n",
    "        \"up_proj\",\n",
    "        \"down_proj\",\n",
    "        \"lm_head\",\n",
    "    ],\n",
    "    bias=\"none\",\n",
    "    lora_dropout=0.05,  # Conventional\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "\n",
    "model = get_peft_model(model, config)\n",
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X_FHi_VLyRgn"
   },
   "source": [
    "See how the model looks different now, with the LoRA adapters added:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "IaYMWak4yRgn"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PeftModelForCausalLM(\n",
      "  (base_model): LoraModel(\n",
      "    (model): LlamaForCausalLM(\n",
      "      (model): LlamaModel(\n",
      "        (embed_tokens): Embedding(32000, 4096)\n",
      "        (layers): ModuleList(\n",
      "          (0-31): 32 x LlamaDecoderLayer(\n",
      "            (self_attn): LlamaAttention(\n",
      "              (q_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (k_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=32, out_features=1024, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (v_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=4096, out_features=1024, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=32, out_features=1024, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (o_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (rotary_emb): LlamaRotaryEmbedding()\n",
      "            )\n",
      "            (mlp): LlamaMLP(\n",
      "              (gate_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=32, out_features=14336, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (up_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=4096, out_features=14336, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=32, out_features=14336, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (down_proj): lora.Linear4bit(\n",
      "                (base_layer): Linear4bit(in_features=14336, out_features=4096, bias=False)\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=14336, out_features=32, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (act_fn): SiLU()\n",
      "            )\n",
      "            (input_layernorm): LlamaRMSNorm()\n",
      "            (post_attention_layernorm): LlamaRMSNorm()\n",
      "          )\n",
      "        )\n",
      "        (norm): LlamaRMSNorm()\n",
      "      )\n",
      "      (lm_head): lora.Linear(\n",
      "        (base_layer): Linear(in_features=4096, out_features=32000, bias=False)\n",
      "        (lora_dropout): ModuleDict(\n",
      "          (default): Dropout(p=0.05, inplace=False)\n",
      "        )\n",
      "        (lora_A): ModuleDict(\n",
      "          (default): Linear(in_features=4096, out_features=32, bias=False)\n",
      "        )\n",
      "        (lora_B): ModuleDict(\n",
      "          (default): Linear(in_features=32, out_features=32000, bias=False)\n",
      "        )\n",
      "        (lora_embedding_A): ParameterDict()\n",
      "        (lora_embedding_B): ParameterDict()\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "05H5MIfjyRgc"
   },
   "source": [
    "### Accelerator\n",
    "\n",
    "Set up the Accelerator. I'm not sure if we really need this for a QLoRA given its [description](https://huggingface.co/docs/accelerate/v0.19.0/en/usage_guides/fsdp) (I have to read more about it) but it seems it can't hurt, and it's helpful to have the code for future reference. You can always comment out the accelerator if you want to try without."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "TEzYBadkyRgd"
   },
   "outputs": [],
   "source": [
    "from accelerate import FullyShardedDataParallelPlugin, Accelerator\n",
    "from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig\n",
    "\n",
    "fsdp_plugin = FullyShardedDataParallelPlugin(\n",
    "    state_dict_config=FullStateDictConfig(offload_to_cpu=True, rank0_only=False),\n",
    "    optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=True, rank0_only=False),\n",
    ")\n",
    "\n",
    "accelerator = Accelerator(fsdp_plugin=fsdp_plugin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "yxSbpKQSLY6B"
   },
   "outputs": [],
   "source": [
    "model = accelerator.prepare_model(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-9KNTJZkyRgn"
   },
   "source": [
    "\n",
    "Let's use Weights & Biases to track our training metrics. You'll need to apply an API key when prompted. Feel free to skip this if you'd like, and just comment out the `wandb` parameters in the `Trainer` definition below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "DDqUNyIoyRgo"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "  ········\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /home/ubuntu/.netrc\n"
     ]
    }
   ],
   "source": [
    "!pip install -q wandb -U\n",
    "\n",
    "import wandb, os\n",
    "wandb.login()\n",
    "\n",
    "wandb_project = \"finetune\"\n",
    "if len(wandb_project) > 0:\n",
    "    os.environ[\"WANDB_PROJECT\"] = wandb_project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_0MOtwf3zdZp"
   },
   "source": [
    "### 5. Run Training!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fEe0uWYSyRgo"
   },
   "source": [
    "I didn't have a lot of training samples: only about 200 total train/validation. I used 500 training steps, and I was fine with overfitting in this case. I found that the end product worked well. It took about 20 minutes on the 1x A10G 24GB.\n",
    "\n",
    "Overfitting is when the validation loss goes up (bad) while the training loss goes down significantly, meaning the model is learning the training set really well, but is unable to generalize to new datapoints. In most cases, this is not desired, but since I am just playing around with a model to generate outputs like my journal entries, I was fine with a moderate amount of overfitting.\n",
    "\n",
    "With that said, a note on training: you can set the `max_steps` to be high initially, and examine at what step your model's performance starts to degrade. There is where you'll find a sweet spot for how many steps to perform. For example, say you start with 1000 steps, and find that at around 500 steps the model starts overfitting, as described above. Therefore, 500 steps would be your sweet spot, so you would use the `checkpoint-500` model repo in your output dir (`mistral-journal-finetune`) as your final model in step 6 below.\n",
    "\n",
    "If you're just doing something for fun like I did and are OK with overfitting, you can try different checkpoint versions with different degrees of overfitting.\n",
    "\n",
    "You can interrupt the process via Kernel -> Interrupt Kernel in the top nav bar once you realize you didn't need to train anymore."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "c_L1131GyRgo"
   },
   "outputs": [],
   "source": [
    "if torch.cuda.device_count() > 1: # If more than 1 GPU\n",
    "    model.is_parallelizable = True\n",
    "    model.model_parallel = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "jq0nX33BmfaC"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='131' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [131/500 1:01:31 < 2:55:58, 0.03 it/s, Epoch 0.33/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>1.576500</td>\n",
       "      <td>1.464192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>1.568100</td>\n",
       "      <td>1.441079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>1.280100</td>\n",
       "      <td>1.429308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>1.547400</td>\n",
       "      <td>1.430723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>1.397200</td>\n",
       "      <td>1.429593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>1.578000</td>\n",
       "      <td>1.425429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>1.332200</td>\n",
       "      <td>1.424603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>1.403100</td>\n",
       "      <td>1.423259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>1.454000</td>\n",
       "      <td>1.418888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>1.434100</td>\n",
       "      <td>1.416797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>1.468400</td>\n",
       "      <td>1.417637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>1.246200</td>\n",
       "      <td>1.416588</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "    <div>\n",
       "      \n",
       "      <progress value='6' max='25' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 6/25 00:44 < 02:48, 0.11 it/s]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[33], line 39\u001b[0m\n\u001b[1;32m     11\u001b[0m trainer \u001b[38;5;241m=\u001b[39m transformers\u001b[38;5;241m.\u001b[39mTrainer(\n\u001b[1;32m     12\u001b[0m     model\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[1;32m     13\u001b[0m     train_dataset\u001b[38;5;241m=\u001b[39mtokenized_train_dataset,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     35\u001b[0m     data_collator\u001b[38;5;241m=\u001b[39mtransformers\u001b[38;5;241m.\u001b[39mDataCollatorForLanguageModeling(tokenizer, mlm\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m),\n\u001b[1;32m     36\u001b[0m )\n\u001b[1;32m     38\u001b[0m model\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_cache \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m  \u001b[38;5;66;03m# silence the warnings. Please re-enable for inference!\u001b[39;00m\n\u001b[0;32m---> 39\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/trainer.py:1556\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1554\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   1555\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1556\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1557\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1558\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1559\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1560\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1561\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/trainer.py:1934\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   1931\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mepoch \u001b[38;5;241m=\u001b[39m epoch \u001b[38;5;241m+\u001b[39m (step \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m steps_skipped) \u001b[38;5;241m/\u001b[39m steps_in_epoch\n\u001b[1;32m   1932\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_step_end(args, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n\u001b[0;32m-> 1934\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_maybe_log_save_evaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtr_loss\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepoch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1935\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1936\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_substep_end(args, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/trainer.py:2283\u001b[0m, in \u001b[0;36mTrainer._maybe_log_save_evaluate\u001b[0;34m(self, tr_loss, model, trial, epoch, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2281\u001b[0m         metrics\u001b[38;5;241m.\u001b[39mupdate(dataset_metrics)\n\u001b[1;32m   2282\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 2283\u001b[0m     metrics \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mignore_keys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2284\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_report_to_hp_search(trial, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step, metrics)\n\u001b[1;32m   2286\u001b[0m \u001b[38;5;66;03m# Run delayed LR scheduler now that metrics are populated\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/trainer.py:3026\u001b[0m, in \u001b[0;36mTrainer.evaluate\u001b[0;34m(self, eval_dataset, ignore_keys, metric_key_prefix)\u001b[0m\n\u001b[1;32m   3023\u001b[0m start_time \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[1;32m   3025\u001b[0m eval_loop \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprediction_loop \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39muse_legacy_prediction_loop \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mevaluation_loop\n\u001b[0;32m-> 3026\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43meval_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3027\u001b[0m \u001b[43m    \u001b[49m\u001b[43meval_dataloader\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3028\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdescription\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mEvaluation\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3029\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;66;43;03m# No point gathering the predictions if there are no metrics, otherwise we defer to\u001b[39;49;00m\n\u001b[1;32m   3030\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;66;43;03m# self.args.prediction_loss_only\u001b[39;49;00m\n\u001b[1;32m   3031\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprediction_loss_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompute_metrics\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   3032\u001b[0m \u001b[43m    \u001b[49m\u001b[43mignore_keys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3033\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmetric_key_prefix\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetric_key_prefix\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3034\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3036\u001b[0m total_batch_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39meval_batch_size \u001b[38;5;241m*\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mworld_size\n\u001b[1;32m   3037\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmetric_key_prefix\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_jit_compilation_time\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m output\u001b[38;5;241m.\u001b[39mmetrics:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/trainer.py:3215\u001b[0m, in \u001b[0;36mTrainer.evaluation_loop\u001b[0;34m(self, dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix)\u001b[0m\n\u001b[1;32m   3212\u001b[0m         batch_size \u001b[38;5;241m=\u001b[39m observed_batch_size\n\u001b[1;32m   3214\u001b[0m \u001b[38;5;66;03m# Prediction step\u001b[39;00m\n\u001b[0;32m-> 3215\u001b[0m loss, logits, labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprediction_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprediction_loss_only\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mignore_keys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3216\u001b[0m main_input_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmain_input_name\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m   3217\u001b[0m inputs_decode \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_input(inputs[main_input_name]) \u001b[38;5;28;01mif\u001b[39;00m args\u001b[38;5;241m.\u001b[39minclude_inputs_for_metrics \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/trainer.py:3436\u001b[0m, in \u001b[0;36mTrainer.prediction_step\u001b[0;34m(self, model, inputs, prediction_loss_only, ignore_keys)\u001b[0m\n\u001b[1;32m   3434\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_labels \u001b[38;5;129;01mor\u001b[39;00m loss_without_labels:\n\u001b[1;32m   3435\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompute_loss_context_manager():\n\u001b[0;32m-> 3436\u001b[0m         loss, outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompute_loss\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m   3437\u001b[0m     loss \u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mmean()\u001b[38;5;241m.\u001b[39mdetach()\n\u001b[1;32m   3439\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(outputs, \u001b[38;5;28mdict\u001b[39m):\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/trainer.py:2762\u001b[0m, in \u001b[0;36mTrainer.compute_loss\u001b[0;34m(self, model, inputs, return_outputs)\u001b[0m\n\u001b[1;32m   2760\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   2761\u001b[0m     labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 2762\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2763\u001b[0m \u001b[38;5;66;03m# Save past state if it exists\u001b[39;00m\n\u001b[1;32m   2764\u001b[0m \u001b[38;5;66;03m# TODO: this needs to be fixed and made cleaner later.\u001b[39;00m\n\u001b[1;32m   2765\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mpast_index \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/utils/operations.py:673\u001b[0m, in \u001b[0;36mconvert_outputs_to_fp32.<locals>.forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 673\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/utils/operations.py:661\u001b[0m, in \u001b[0;36mConvertOutputsToFp32.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    660\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 661\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m convert_to_fp32(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/amp/autocast_mode.py:16\u001b[0m, in \u001b[0;36mautocast_decorator.<locals>.decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 16\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/utils/operations.py:673\u001b[0m, in \u001b[0;36mconvert_outputs_to_fp32.<locals>.forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 673\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/utils/operations.py:661\u001b[0m, in \u001b[0;36mConvertOutputsToFp32.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    660\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 661\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m convert_to_fp32(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/amp/autocast_mode.py:16\u001b[0m, in \u001b[0;36mautocast_decorator.<locals>.decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 16\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/utils/operations.py:673\u001b[0m, in \u001b[0;36mconvert_outputs_to_fp32.<locals>.forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 673\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/utils/operations.py:661\u001b[0m, in \u001b[0;36mConvertOutputsToFp32.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    660\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 661\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m convert_to_fp32(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/amp/autocast_mode.py:16\u001b[0m, in \u001b[0;36mautocast_decorator.<locals>.decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 16\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/peft/peft_model.py:1003\u001b[0m, in \u001b[0;36mPeftModelForCausalLM.forward\u001b[0;34m(self, input_ids, attention_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict, task_ids, **kwargs)\u001b[0m\n\u001b[1;32m    992\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAssertionError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mforward in MPTForCausalLM does not support inputs_embeds\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    993\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_model(\n\u001b[1;32m    994\u001b[0m             input_ids\u001b[38;5;241m=\u001b[39minput_ids,\n\u001b[1;32m    995\u001b[0m             attention_mask\u001b[38;5;241m=\u001b[39mattention_mask,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1000\u001b[0m             \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m   1001\u001b[0m         )\n\u001b[0;32m-> 1003\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbase_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1004\u001b[0m \u001b[43m        \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1005\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1006\u001b[0m \u001b[43m        \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1007\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1008\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1009\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1010\u001b[0m \u001b[43m        \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1011\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1012\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1014\u001b[0m batch_size \u001b[38;5;241m=\u001b[39m _get_batch_size(input_ids, inputs_embeds)\n\u001b[1;32m   1015\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m attention_mask \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1016\u001b[0m     \u001b[38;5;66;03m# concat prompt attention mask\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/peft/tuners/tuners_utils.py:108\u001b[0m, in \u001b[0;36mBaseTuner.forward\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any):\n\u001b[0;32m--> 108\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/hooks.py:164\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(module, *args, **kwargs)\u001b[0m\n\u001b[1;32m    162\u001b[0m         output \u001b[38;5;241m=\u001b[39m module\u001b[38;5;241m.\u001b[39m_old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    163\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 164\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_old_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:1034\u001b[0m, in \u001b[0;36mLlamaForCausalLM.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1031\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[1;32m   1033\u001b[0m \u001b[38;5;66;03m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[39;00m\n\u001b[0;32m-> 1034\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1035\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1036\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1037\u001b[0m \u001b[43m    \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1038\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1039\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1040\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1041\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1042\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1043\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1044\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1046\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1047\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mpretraining_tp \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/hooks.py:164\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(module, *args, **kwargs)\u001b[0m\n\u001b[1;32m    162\u001b[0m         output \u001b[38;5;241m=\u001b[39m module\u001b[38;5;241m.\u001b[39m_old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    163\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 164\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_old_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:922\u001b[0m, in \u001b[0;36mLlamaModel.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    912\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    913\u001b[0m         decoder_layer\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m    914\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    919\u001b[0m         use_cache,\n\u001b[1;32m    920\u001b[0m     )\n\u001b[1;32m    921\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 922\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mdecoder_layer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    923\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    924\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    925\u001b[0m \u001b[43m        \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    926\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    927\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    928\u001b[0m \u001b[43m        \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    929\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    931\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    933\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/hooks.py:164\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(module, *args, **kwargs)\u001b[0m\n\u001b[1;32m    162\u001b[0m         output \u001b[38;5;241m=\u001b[39m module\u001b[38;5;241m.\u001b[39m_old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    163\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 164\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_old_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:686\u001b[0m, in \u001b[0;36mLlamaDecoderLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, **kwargs)\u001b[0m\n\u001b[1;32m    684\u001b[0m residual \u001b[38;5;241m=\u001b[39m hidden_states\n\u001b[1;32m    685\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpost_attention_layernorm(hidden_states)\n\u001b[0;32m--> 686\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmlp\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    687\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m residual \u001b[38;5;241m+\u001b[39m hidden_states\n\u001b[1;32m    689\u001b[0m outputs \u001b[38;5;241m=\u001b[39m (hidden_states,)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/accelerate/hooks.py:164\u001b[0m, in \u001b[0;36madd_hook_to_module.<locals>.new_forward\u001b[0;34m(module, *args, **kwargs)\u001b[0m\n\u001b[1;32m    162\u001b[0m         output \u001b[38;5;241m=\u001b[39m module\u001b[38;5;241m.\u001b[39m_old_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    163\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 164\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_old_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m module\u001b[38;5;241m.\u001b[39m_hf_hook\u001b[38;5;241m.\u001b[39mpost_forward(module, output)\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py:258\u001b[0m, in \u001b[0;36mLlamaMLP.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    256\u001b[0m     down_proj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msum\u001b[39m(down_proj)\n\u001b[1;32m    257\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 258\u001b[0m     down_proj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdown_proj(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mact_fn(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgate_proj(x)) \u001b[38;5;241m*\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mup_proj\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    260\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m down_proj\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1518\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1518\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/modules/module.py:1527\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1523\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1524\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1526\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1529\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1530\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/.pyenv/versions/3.10.13/lib/python3.10/site-packages/peft/tuners/lora/bnb.py:307\u001b[0m, in \u001b[0;36mLinear4bit.forward\u001b[0;34m(self, x, *args, **kwargs)\u001b[0m\n\u001b[1;32m    305\u001b[0m             output \u001b[38;5;241m=\u001b[39m output\u001b[38;5;241m.\u001b[39mto(expected_dtype)\n\u001b[1;32m    306\u001b[0m         output \u001b[38;5;241m=\u001b[39m output \u001b[38;5;241m*\u001b[39m scaling\n\u001b[0;32m--> 307\u001b[0m         result \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m output\n\u001b[1;32m    309\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import transformers\n",
    "from datetime import datetime\n",
    "\n",
    "project = \"finetune\"\n",
    "base_model_name = \"mistral\"\n",
    "run_name = base_model_name + \"-\" + project\n",
    "output_dir = \"./\" + run_name\n",
    "\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "trainer = transformers.Trainer(\n",
    "    model=model,\n",
    "    train_dataset=tokenized_train_dataset,\n",
    "    eval_dataset=tokenized_val_dataset,\n",
    "    args=transformers.TrainingArguments(\n",
    "        output_dir=output_dir,\n",
    "        warmup_steps=1,\n",
    "        per_device_train_batch_size=2,\n",
    "        gradient_accumulation_steps=1,\n",
    "        gradient_checkpointing=True,\n",
    "        max_steps=500,\n",
    "        learning_rate=2.5e-5, # Want a small lr for finetuning\n",
    "        bf16=True,\n",
    "        optim=\"paged_adamw_8bit\",\n",
    "        logging_steps=10,              # When to start reporting loss\n",
    "        logging_dir=\"./logs\",        # Directory for storing logs\n",
    "        save_strategy=\"steps\",       # Save the model checkpoint every logging step\n",
    "        save_steps=10,                # Save checkpoints every 50 steps\n",
    "        evaluation_strategy=\"steps\", # Evaluate the model every logging step\n",
    "        eval_steps=10,               # Evaluate and save checkpoints every 50 steps\n",
    "        do_eval=True,                # Perform evaluation at the end of training\n",
    "        report_to=\"wandb\",           # Comment this out if you don't want to use weights & baises\n",
    "        run_name=f\"{run_name}-{datetime.now().strftime('%Y-%m-%d-%H-%M')}\"          # Name of the W&B run (optional)\n",
    "    ),\n",
    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
    ")\n",
    "\n",
    "model.config.use_cache = False  # silence the warnings. Please re-enable for inference!\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "R9rRmDCeQiTJ"
   },
   "source": [
    "I cleared the output of the cell above because I stopped the training early, and it produced a long, ugly error message."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0D57XqcsyRgo"
   },
   "source": [
    "### 6. Drum Roll... Try the Trained Model!\n",
    "\n",
    "It's a good idea to kill the current process so that you don't run out of memory loading the base model again on top of the model we just trained. Go to `Kernel > Restart Kernel` or kill the process via the Terminal (`nvidia smi` > `kill [PID]`). \n",
    "\n",
    "By default, the PEFT library will only save the QLoRA adapters, so we need to first load the base model from the Huggingface Hub:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "fb8230fb86884aa6be318e2d03a88af2"
     ]
    },
    "id": "SKSnF016yRgp",
    "outputId": "bce5209d-90da-4117-c6ac-cda9f3cb3422"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: huggingface_hub in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (0.19.4)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from huggingface_hub) (2023.10.0)\n",
      "Requirement already satisfied: packaging>=20.9 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from huggingface_hub) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from huggingface_hub) (6.0.1)\n",
      "Requirement already satisfied: filelock in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from huggingface_hub) (3.13.1)\n",
      "Requirement already satisfied: tqdm>=4.42.1 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from huggingface_hub) (4.66.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from huggingface_hub) (4.8.0)\n",
      "Requirement already satisfied: requests in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from huggingface_hub) (2.31.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from requests->huggingface_hub) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from requests->huggingface_hub) (2023.11.17)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from requests->huggingface_hub) (2.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in ./.pyenv/versions/3.10.13/lib/python3.10/site-packages (from requests->huggingface_hub) (3.3.2)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/usr/bin\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bin    include\tlib32  libexec\tlocal  share\n",
      "games  lib\tlib64  libx32\tsbin   src\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create any files you like! Then...\n",
    "git add .\n",
    "git commit -m \"First model version\"  # You can choose any descriptive message\n",
    "git push"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "fb8230fb86884aa6be318e2d03a88af2"
     ]
    },
    "id": "SKSnF016yRgp",
    "outputId": "bce5209d-90da-4117-c6ac-cda9f3cb3422"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.pyenv/versions/3.10.13/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:472: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fac65376bd4407eb89ea80ea3e4f0a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
    "\n",
    "base_model_id = \"filipealmeida/Mistral-7B-v0.1-sharded\"\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_use_double_quant=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "base_model = AutoModelForCausalLM.from_pretrained(\n",
    "    base_model_id,  # Mistral, same as before\n",
    "    quantization_config=bnb_config,  # Same quantization config as before\n",
    "    device_map=\"auto\",\n",
    "    trust_remote_code=True,\n",
    "    use_auth_token=True\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(base_model_id, add_bos_token=True, trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_BxOhAiqyRgp"
   },
   "source": [
    "Now load the QLoRA adapter from the appropriate checkpoint directory, i.e. the best performing model checkpoint:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "GwsiqhWuyRgp"
   },
   "outputs": [],
   "source": [
    "from peft import PeftModel\n",
    "\n",
    "ft_model = PeftModel.from_pretrained(base_model, \"mistral-finetune/checkpoint-120\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "lX39ibolyRgp"
   },
   "source": [
    "and run your inference!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UUehsaVNyRgp"
   },
   "source": [
    "Let's try the same `eval_prompt` and thus `model_input` as above, and see if the new finetuned model performs better. I like playing with the repetition penalty (just little tweaks of .01-.05 at a time). THIS IS SO FUN. I'm obsessed wth this AI version of myself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "lMkVNEUvyRgp",
    "outputId": "7d49d409-5dbe-4306-c1a4-9d87e3073397"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The following is a note by Eevee the Dog, which doesn't share anything too personal: #  I am not sure if this is the right place to post this. But I have been thinking about it for some time now and I think that I should just get it out of my system.\n",
      "I don't know how many people here are aware of what happened in the past few months. It was a very difficult period for me. I had to deal with a lot of things at once. And I didn't really know where to turn to. I felt like I was drowning. I couldn't see any way out. I thought that I would never be able to come back from there.\n",
      "But then I met someone who helped me through all of this. He gave me hope when I needed it most. He made me believe that everything will be alright again. That I can still live a happy life.\n",
      "And he did so much more than that. He showed me that love exists. That there are people out there who care about you. Who want to help you no matter what.\n",
      "He changed my life forever. And I am grateful for him every day.\n",
      "So thank you, whoever you are. You saved me.\n"
     ]
    }
   ],
   "source": [
    "eval_prompt = \" The following is a note by Eevee the Dog, which doesn't share anything too personal: # \"\n",
    "model_input = tokenizer(eval_prompt, return_tensors=\"pt\").to(\"cuda\")\n",
    "\n",
    "ft_model.eval()\n",
    "with torch.no_grad():\n",
    "    print(tokenizer.decode(ft_model.generate(**model_input, max_new_tokens=500, repetition_penalty=1.15)[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'python-mistral-ft'...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "remote: Enumerating objects: 3, done.\u001b[K\n",
      "remote: Total 3 (delta 0), reused 0 (delta 0), pack-reused 3\u001b[K\n",
      "Unpacking objects: 100% (3/3), 418 bytes | 418.00 KiB/s, done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://huggingface.co/HassanSamo/python-mistral-ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "fb8230fb86884aa6be318e2d03a88af2"
     ]
    },
    "id": "SKSnF016yRgp",
    "outputId": "bce5209d-90da-4117-c6ac-cda9f3cb3422"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eval.jsonl\t\t\t    python-mistral-ft\n",
      "mistral-finetune\t\t    train.jsonl\n",
      "mistral-finetune-own-data.ipynb     verb-workspace\n",
      "new-workspace.jupyterlab-workspace  wandb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "ZjPlpRSJ-aSs"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ubuntu\n",
      "eval.jsonl\t  mistral-finetune-own-data.ipynb  train.jsonl\t   wandb\n",
      "mistral-finetune  python-mistral-ft\t\t   verb-workspace\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "%cd ..\n",
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "cp -R * python-mistral-ft\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}