{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q-bj6K7Qv4ft"
      },
      "source": [
        "# Fine-Tuning a Generative Pretrained Transformer (`GPT`)\n",
        "\n",
        "1. Install required libraries."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SBWCrz5GfBXo",
        "outputId": "1e1ab31b-3b54-4c39-a872-fb39e4c5c8a4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.30.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.3.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: codecarbon in /usr/local/lib/python3.10/dist-packages (2.2.3)\n",
            "Requirement already satisfied: arrow in /usr/local/lib/python3.10/dist-packages (from codecarbon) (1.2.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from codecarbon) (1.5.3)\n",
            "Requirement already satisfied: pynvml in /usr/local/lib/python3.10/dist-packages (from codecarbon) (11.5.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from codecarbon) (2.27.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from codecarbon) (5.9.5)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from codecarbon) (9.0.0)\n",
            "Requirement already satisfied: fuzzywuzzy in /usr/local/lib/python3.10/dist-packages (from codecarbon) (0.18.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from codecarbon) (8.1.3)\n",
            "Requirement already satisfied: python-dateutil>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from arrow->codecarbon) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->codecarbon) (2022.7.1)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas->codecarbon) (1.22.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->codecarbon) (3.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7.0->arrow->codecarbon) (1.16.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install codecarbon"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y5XnfvSH7w4z"
      },
      "source": [
        "2. Load the data from the hub."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 467
        },
        "id": "7MbpXGu-v4f1",
        "outputId": "3d8858dc-6ea1-4871-cb04-44dcecce51a6"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-7ebefa58-d546-469a-8e61-ec63d737689d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>prompt</th>\n",
              "      <th>completion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Which is a species of fish? Tope or Rope</td>\n",
              "      <td>Tope</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Why can camels survive for long without water?</td>\n",
              "      <td>Camels use the fat in their humps to keep them...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Alice's parents have three daughters: Amy, Jes...</td>\n",
              "      <td>The name of the third daughter is Alice</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Who gave the UN the land in NY to build their HQ</td>\n",
              "      <td>John D Rockerfeller</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Why mobile is bad for human</td>\n",
              "      <td>We are always engaged one phone which is not g...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53129</th>\n",
              "      <td>How do computers communicate and network with ...</td>\n",
              "      <td>Computers communicate and network with each ot...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53130</th>\n",
              "      <td>How are websites different from web applications?</td>\n",
              "      <td>Websites and web applications are similar in t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53131</th>\n",
              "      <td>What is open-source software and its benefits?</td>\n",
              "      <td>Open-source software is software that is made ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53132</th>\n",
              "      <td>What is a cookie and how is it used in web bro...</td>\n",
              "      <td>A cookie is a small piece of data that a websi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53133</th>\n",
              "      <td>What is cloud storage and its advantages for d...</td>\n",
              "      <td>Cloud storage is a service that allows you to ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>53134 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7ebefa58-d546-469a-8e61-ec63d737689d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7ebefa58-d546-469a-8e61-ec63d737689d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7ebefa58-d546-469a-8e61-ec63d737689d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                                  prompt  \\\n",
              "0               Which is a species of fish? Tope or Rope   \n",
              "1         Why can camels survive for long without water?   \n",
              "2      Alice's parents have three daughters: Amy, Jes...   \n",
              "3       Who gave the UN the land in NY to build their HQ   \n",
              "4                            Why mobile is bad for human   \n",
              "...                                                  ...   \n",
              "53129  How do computers communicate and network with ...   \n",
              "53130  How are websites different from web applications?   \n",
              "53131     What is open-source software and its benefits?   \n",
              "53132  What is a cookie and how is it used in web bro...   \n",
              "53133  What is cloud storage and its advantages for d...   \n",
              "\n",
              "                                              completion  \n",
              "0                                                   Tope  \n",
              "1      Camels use the fat in their humps to keep them...  \n",
              "2                The name of the third daughter is Alice  \n",
              "3                                    John D Rockerfeller  \n",
              "4      We are always engaged one phone which is not g...  \n",
              "...                                                  ...  \n",
              "53129  Computers communicate and network with each ot...  \n",
              "53130  Websites and web applications are similar in t...  \n",
              "53131  Open-source software is software that is made ...  \n",
              "53132  A cookie is a small piece of data that a websi...  \n",
              "53133  Cloud storage is a service that allows you to ...  \n",
              "\n",
              "[53134 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"nicholasKluge/fine-tuning-instruct-aira\", split=\"aira_instruct_english\")\n",
        "\n",
        "df = dataset.to_pandas()\n",
        "\n",
        "display(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fEaDTvdOe8rr"
      },
      "source": [
        "3. Load `GPT2Tokenizer` and add the chosen special tokens (`'<|startoftext|>', '<|endoftext|>','<|pad|>'`)\n",
        "4. Create demonstrations by prepending the special tokens.\n",
        "5. Calculate the maximum length (in tokens) that the demonstrations have (the dataset was constructed, for efficiency and fast training, to be below the 300-token range)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241,
          "referenced_widgets": [
            "2231293622594eceb80d6481799ddcad",
            "c1135b0eb85044ab8e47e772e0c71f3c",
            "41b5bf9be3394e728e3f3699026ce885",
            "96552a3dbfa94bb4818b054b2675c6b8",
            "13c74818a5b949438589a0c73007e5a6",
            "6bab36d7ee934127a80a53ab87e0e508",
            "69e756a204214d689240aef2b9986118",
            "b27100866f8b4c08a7931f4d3d84f2b3",
            "47472c3a381f4eca9af9d03df1718065",
            "9cdd14da170b4b40b029b0150a40afc7",
            "4cf2d43d7ae24582ba035d9770f8e860",
            "6539b916daa747adab05dcff94c2cd3c",
            "433e467610674781b011918357e68baa",
            "57a05e39d49a4b4b853381676bd7ec9d",
            "0659624c4b9a4ce8b396b8d45814c580",
            "7ebf18e21686429391b47866e46f8c6d",
            "66e3178dae0d4b78ac3e00374ebd8e6b",
            "47449e8feed84a5cb428b2ac0006a053",
            "69d7726ac4804227ab1b1c026ca29c2b",
            "ba9429a433b5479eae6e188f1c5b864f",
            "ab4d80ebe66546ec92877f3288729bcd",
            "e1760e7d52464351ba391e002132a0bf",
            "e2ca1209ffe94778a38a383641b09cb9",
            "30ce9a42524446edbf99a1147377ae8e",
            "c119c152c0f3485daebb7cbc5992f13a",
            "75ef15d0ffba4344bc07a22690a99f13",
            "601e6dedd4a64ddcbeb5ead5b20c70aa",
            "5b053f2aac4e42e09c446691cbe67416",
            "62ebb5cbcbe94e0bb416896ec8493b5c",
            "96de7beae5b94e6a92f9f52b9d824035",
            "e521d59d193b42a28f29b3ad77c12c96",
            "acbc045dd45c41c484e59304a66336f4",
            "b8b4e3e26208445e91aa15ee3ea6e6b0"
          ]
        },
        "id": "hfu84fWIv4f9",
        "outputId": "4c471e66-3497-433d-bc65-2287608a6f32"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2231293622594eceb80d6481799ddcad",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6539b916daa747adab05dcff94c2cd3c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e2ca1209ffe94778a38a383641b09cb9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/666 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Total number of demonstrations:  53134\n",
            "The longest demonstration is 300 tokens long.\n"
          ]
        }
      ],
      "source": [
        "from transformers import GPT2Tokenizer\n",
        "\n",
        "model = \"gpt2-large\" # \"gpt2\", \"gpt2-medium\", \"gpt2-large\", \"gpt2-xl\", \"pierreguillou/gpt2-small-portuguese\"\n",
        "model_size = \"774M\" # \"124M\", \"355M\", \"774M\", and \"1.5B\", \"PT-124M\"\n",
        "\n",
        "tokenizer = GPT2Tokenizer.from_pretrained(model,\n",
        "                                          bos_token='<|startoftext|>',\n",
        "                                          eos_token='<|endoftext|>',\n",
        "                                          pad_token='<|pad|>')\n",
        "\n",
        "df['demonstrations'] = tokenizer.bos_token + df['prompt'] + tokenizer.eos_token + df['completion'] + tokenizer.eos_token\n",
        "\n",
        "df['length'] = df['demonstrations'].apply(lambda x: len(tokenizer.encode(x)))\n",
        "\n",
        "print(\"Total number of demonstrations: \", len(df))\n",
        "print(f\"The longest demonstration is {df['length'].max()} tokens long.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wkMO17K5e8rs"
      },
      "source": [
        "6. Create the Dataset class."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "WlbAfMQ4v4gA"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "max_length = 300\n",
        "\n",
        "class DemoDataset(Dataset):\n",
        "\n",
        "  def __init__(self, demonstrations, tokenizer, gpt2_type=\"gpt2\", max_length=max_length):\n",
        "\n",
        "    self.tokenizer = tokenizer\n",
        "    self.input_ids = []\n",
        "    self.attn_masks = []\n",
        "\n",
        "    for demo in demonstrations:\n",
        "\n",
        "      encodings_dict = tokenizer(demo,\n",
        "                                 truncation=True,\n",
        "                                 max_length=max_length,\n",
        "                                 padding=\"max_length\")\n",
        "\n",
        "      self.input_ids.append(torch.tensor(encodings_dict['input_ids']))\n",
        "      self.attn_masks.append(torch.tensor(encodings_dict['attention_mask']))\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.input_ids)\n",
        "\n",
        "  def __getitem__(self, idx):\n",
        "    return self.input_ids[idx], self.attn_masks[idx]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "711Krm6Te8rt"
      },
      "source": [
        "7. Split the data into training and validation splits."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-IOfa2PEv4gD",
        "outputId": "ca19c34e-79fe-43ef-f0ad-b4f06e2d8f5f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of training samples: 47,820\n",
            "Number of validation samples: 5,314\n"
          ]
        }
      ],
      "source": [
        "from torch.utils.data import random_split\n",
        "\n",
        "dataset = DemoDataset(df.demonstrations.to_list(), tokenizer, max_length=max_length)\n",
        "\n",
        "train_size = int(0.9 * len(dataset))\n",
        "val_size = len(dataset) - train_size\n",
        "\n",
        "train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n",
        "\n",
        "print('Number of training samples: {:,}'.format(train_size))\n",
        "print('Number of validation samples: {:,}'.format(val_size))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vSqKuRjIe8ru"
      },
      "source": [
        "8. Create the `DataLoaders` and specify the `batch_size`."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "cUkCNV-6v4gG"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,\n",
        "            sampler=RandomSampler(train_dataset),\n",
        "            batch_size=8 # 32, 20, 8, 8\n",
        "        )\n",
        "\n",
        "# validation data loader doesn't need randomization\n",
        "validation_dataloader=DataLoader(\n",
        "            val_dataset,\n",
        "            sampler=SequentialSampler(val_dataset),\n",
        "            batch_size=8\n",
        "        )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0vxvcTIHe8rv"
      },
      "source": [
        "9. Load the base model (`GPT2LMHeadModel`)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 550,
          "referenced_widgets": [
            "0fc39225b0694840b7f421f725c135f2",
            "4886d255282b4eefa37e7a1db102a132",
            "7a37f293f6884093ab588c9ded32db07",
            "bd0f1d0070294b6eaebba425e9224150",
            "7cccbb84d04b4aaaa09309bb669f349d",
            "2cdd431ccdb64e409229913cc43e2544",
            "ba5408f14ed54e75a9724d39ac67d5c2",
            "10d2880b45814382951aa1015f4f1f5f",
            "3657a339af424c51ac7534f91fcb5893",
            "1bb5acd98f244616905e0f5cb8752a8a",
            "b023d77e9b854acd94ac0a91feb2de83",
            "b2d0b131237f49e684451afe60b57593",
            "8691d4c5d8d54e2494d2bf149feb9980",
            "7ddef1b949f04c81b6e26df3ce68cbc6",
            "e125c896502c4ad48b4bfbdbe3dc7383",
            "1e2b0f9db8654f85938565d9f694030d",
            "129c206371304edd9eaf34a5b56a00b8",
            "2d1b573ef7a242c295ed42d790a71f1e",
            "89f8e93f887d43ee933fa3b44a44fd72",
            "e58aaf5a1b9a4a2f85fa2546ee88f24e",
            "3181ccb3dde340c1b5c67ed12076c897",
            "4a22e5e3eb354b4cb63bf0aeb9105efa"
          ]
        },
        "id": "Rmg-5YJqv4gH",
        "outputId": "1f8aa617-07b6-4774-d024-cd50d8148f5b"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0fc39225b0694840b7f421f725c135f2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/3.25G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b2d0b131237f49e684451afe60b57593",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "GPT2LMHeadModel(\n",
              "  (transformer): GPT2Model(\n",
              "    (wte): Embedding(50259, 1280)\n",
              "    (wpe): Embedding(1024, 1280)\n",
              "    (drop): Dropout(p=0.1, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0-35): 36 x GPT2Block(\n",
              "        (ln_1): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (act): NewGELUActivation()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=1280, out_features=50259, bias=False)\n",
              ")"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from transformers import GPT2LMHeadModel, GPT2Config\n",
        "\n",
        "configuration = GPT2Config.from_pretrained(model, output_hidden_states=False)\n",
        "\n",
        "model = GPT2LMHeadModel.from_pretrained(model, config=configuration)\n",
        "model.resize_token_embeddings(len(tokenizer))\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model.to(device)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XAoXD7GVYMN_"
      },
      "source": [
        "10. Freeze some of the layers for constrained fine-tuning. This allows the model to retain some of its original capabilities after the tuning."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MJA-WOoHZePn",
        "outputId": "dc2c4cea-17c5-479b-b667-b4782ded533a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of transformer blocks in the model:  36\n",
            "Number of transformer blocks to  un-freeze:  9\n",
            "Number of frozen layers: 325\n",
            "Number of trainable layers: 111\n"
          ]
        }
      ],
      "source": [
        "UNFREEZE_LAST_N = 9 #6, 8, 9, 12\n",
        "\n",
        "print(\"Number of transformer blocks in the model: \", model.config.n_layer)\n",
        "print(\"Number of transformer blocks to  un-freeze: \", UNFREEZE_LAST_N)\n",
        "\n",
        "for parameter in model.parameters():\n",
        "    parameter.requires_grad = False\n",
        "\n",
        "for i, m in enumerate(model.transformer.h):\n",
        "    #Only un-freeze the last n transformer blocks\n",
        "    if i+1 > model.config.n_layer - UNFREEZE_LAST_N:\n",
        "        for parameter in m.parameters():\n",
        "            parameter.requires_grad = True\n",
        "\n",
        "        for parameter in model.transformer.ln_f.parameters():\n",
        "            parameter.requires_grad = True\n",
        "\n",
        "        for parameter in model.lm_head.parameters():\n",
        "            parameter.requires_grad = True\n",
        "\n",
        "num_frozen_layers = sum(1 for parameter in model.parameters() if not parameter.requires_grad)\n",
        "num_trainable_layers = sum(1 for parameter in model.parameters() if parameter.requires_grad)\n",
        "\n",
        "print(\"Number of frozen layers:\", num_frozen_layers)\n",
        "print(\"Number of trainable layers:\", num_trainable_layers)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GJ1kH3uSe8rw"
      },
      "source": [
        "11. Set the training hyperparameters."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "qlbLg6tqv4gI"
      },
      "outputs": [],
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# training rounds\n",
        "epochs = 2 # 5, 3, 2, 1\n",
        "\n",
        "# steps at the start of training that are ignored\n",
        "warmup_steps = 1e2\n",
        "\n",
        "# sample the model to test the output\n",
        "sample_every = 400\n",
        "\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr = 5e-4, eps = 1e-8)\n",
        "\n",
        "# total steps = number of batches * number of epochs\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# create the learning rate scheduler\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                            num_warmup_steps = warmup_steps,\n",
        "                                            num_training_steps = total_steps)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nz1a9lxXe8rw"
      },
      "source": [
        "12. Training/Validation loop. Track the carbon emissions of your work by using `codecarbon`. 🌱"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_X_m8XOtv4gR",
        "outputId": "998c5fe2-29d0-411c-a4da-67dffe9d9210"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Beginning epoch 1 of 2\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  7%|▋         | 400/5978 [04:38<1:04:12,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 400 of 5978. Loss:0.6028859615325928.\n",
            "\n",
            "\n",
            "Example output: What is the most secure way to store financial information.The most secure way to store financial information is to have a financial agency that is able to provide secure storage, including cold storage, with a combination of various cryptographic principles.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 13%|█▎        | 800/5978 [09:17<59:35,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 800 of 5978. Loss:0.6626878380775452.\n",
            "\n",
            "\n",
            "Example output: Can you give me advice on setting up a business.One of the best ways to set up a business is to create an account with a reputable financial institution. Start by researching the account you need, and you should be able to find a reliable and trustworthy account to start with. Additionally, look for services that are offered specifically for businesses or professionals and make sure to compare rates. This can help you find a comfortable level of cost for each service. Finally, make sure you have a plan for growth. Research the business and find out the resources and marketing campaigns that can help you reach your desired audience.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 20%|██        | 1200/5978 [13:58<54:59,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 1200 of 5978. Loss:0.655192494392395.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r 20%|██        | 1201/5978 [14:01<1:39:26,  1.25s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Example output: Please help me find an interesting but classic book to read for my commute this week.One interesting book to read for your commute this week is \"The Hitchhiker's Guide to the Galaxy\" by Douglas Adams. It is a classic fantasy adventure book that follows the protagonist's journey to save the universe.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 27%|██▋       | 1600/5978 [18:37<50:23,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 1600 of 5978. Loss:0.7218917012214661.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r 27%|██▋       | 1601/5978 [18:38<1:11:20,  1.02it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Example output: Name five different cheeses.Tequila, Chorizo, Mashed Potatoes, Burgundy Cheese, and Mashed Potatoes\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 33%|███▎      | 2000/5978 [23:14<45:47,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 2000 of 5978. Loss:0.4903753697872162.\n",
            "\n",
            "\n",
            "Example output: What is algebra and who invented it?Algebra and the foundations of mathematics are incredibly complex and require a lot of practice in order to learn. The concepts of algebra and the algebraic equations are based on equations and sequences of equations, not on purely numerical information. It is important to differentiate between algorithms and algorithms' algorithms, as Algebra and the Algebraic equations work very effectively in situations where these equations are being used to solve complex problems. For instance, using the equation A=1 and applying it to any real number, such as 1, can solve any equation of that order. The equations of motion are typically solvable by algebraic equations and have some applications in machine learning, where it is possible to identify patterns in data that are difficult to solve with numerical information.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 40%|████      | 2400/5978 [27:56<41:11,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 2400 of 5978. Loss:0.4223511219024658.\n",
            "\n",
            "\n",
            "Example output: How can I improve my learning habits, so that the material I'm studying is stored in my memory more effectively.One of the most effective ways to improve your learning habits is to set small achievable goals. Break down your learning into smaller chunks and focus on mastering the material you are learning. This will help to stay motivated to study and make progress. Additionally, make sure to practice with native materials to help develop your understanding of the language and familiarize yourself with what you are learning. Finally, look for resources like books, podcasts, and websites that are designed to be visual and engaging, allowing you to experience the language from different angles and perspectives.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 2800/5978 [32:37<36:35,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 2800 of 5978. Loss:0.7402357459068298.\n",
            "\n",
            "\n",
            "Example output: What is neoliberalism and what are its key tenets?Foliberalism is an ideology that emphasizes the importance of individual freedom and autonomy, while recognizing the importance of developing policies and technologies that guarantee social, economic, and political stability. It is often associated with economic deregulation, deregulation of industries and industries on a large scale, deregulation of industries within governments, deregulation of industries within state and local governments, and deregulation of industries within global companies. This ideology involves preserving individual freedom and autonomy, while encouraging technological advancements and exploring the world around us. It is often associated with the deregulation of the economy and the deregulation of the economy in general, as well as deregulation of government services and the deregulation of government and industries. Neoliberalism also involves the deregulation of political and social systems, including deregulation of democratic reforms, deregulation of laws, and deregulation of industries to ensure a strong welfare state. It is often associated with a free-market perspective, and the deregulation of individual and collective freedom and autonomy.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 54%|█████▎    | 3200/5978 [37:21<31:58,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 3200 of 5978. Loss:0.5701470971107483.\n",
            "\n",
            "\n",
            "Example output: Write a brief paragraph of the benefits of attending Arizona State UniversityArizona State University is one of the oldest universities in the United States, dating back to 1867. It is a private institution with over 200 degree programs. It is also one of the best universities in the United States for those with a degree or master's degree in mathematics, economics, research science, public health, or education. Arizona State University offers its students a wide range of program offerings, including the renowned Stanford University University, University of Southern California, Arizona State University, Arizona State University, Arizona State University. Arizona State University offers its students a wide range of program offerings, including the renowned Stanford University University, University of Southern California, Arizona State University, Arizona State University, Arizona State University. Phoenix and Arizona are both located in Arizona and offer a range of student activities such as Arizona State University School of Business, Arizona State University School of Law, Arizona State University School of State University, and Arizona State University College of Business\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 60%|██████    | 3600/5978 [42:05<27:21,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 3600 of 5978. Loss:0.7429184913635254.\n",
            "\n",
            "\n",
            "Example output: How do I organize my home office.To organize your home office, you will need to find a storage space, decide what items you need and determine what you will get in exchange. Then, decide what features you want the office to have such as a filing system, tools, and materials. After that, you can decide if you want to keep everything in the closet or invest in some organizers. Once you have your space organized, you can start by sorting the items by type. This will help make use of the space better, such as organizing small items like the necessary items like coffee and paper. Finally, you can set realistic goals for the items you want and determine how much you need each one.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 67%|██████▋   | 4000/5978 [46:47<22:45,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 4000 of 5978. Loss:0.7125013470649719.\n",
            "\n",
            "\n",
            "Example output: Is violence good sometimes?As a manager, what is good for you is not always good for you, and what is bad sometimes is not necessarily bad in the long run. This could apply to conflicts or disputes, relationships with others or time in the workplace. It's important to acknowledge that it's okay to be willing to take some calculated risk or involve yourself in an argument to reach a resolution or finding a mutually beneficial solution. It's okay to be willing to try different strategies and to be open to finding ways to avoid conflict. It's also important to try to prioritize finding a productive solution and finding ways to communicate better and compromise when needed.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 4400/5978 [51:29<18:09,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 4400 of 5978. Loss:0.9460142850875854.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r 74%|███████▎  | 4401/5978 [51:31<27:22,  1.04s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Example output: Classify each item as being at a playground or a gym: slide, swing, dumbbell, squat rack, seesawSlide: playground\n",
            "Swing: Gym\n",
            "Dumbbell: Gym\n",
            "Squat Rack: Gym\n",
            "Seesaw: Gym\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 80%|████████  | 4800/5978 [56:06<13:33,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 4800 of 5978. Loss:0.8724031448364258.\n",
            "\n",
            "\n",
            "Example output: What are some useful strategies for solving a Rubik's cube.Crackers \n",
            "2. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "3. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "4. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "5. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "6. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "7. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "8. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "9. Turn the cube clockwise by moving one of the bottom pieces clockwise. \n",
            "10 Turn the cube clockwise by moving one of the bottom pieces clockwise.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 87%|████████▋ | 5200/5978 [1:00:50<08:57,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 5200 of 5978. Loss:0.5923346281051636.\n",
            "\n",
            "\n",
            "Example output: How much should I save each month to purchase a house in five years.The amount you should save each month to purchase a house in five years will vary based on the size of your home, as well as the condition of the property and the financial commitment you need to commit. However, you may want to set a goal that limits the amount of time you need to save each month to purchase a house. Additionally, make sure to factor in any unexpected expenses you may incur, such as insurance or maintenance.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 94%|█████████▎| 5600/5978 [1:05:30<04:20,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 5600 of 5978. Loss:0.5626195669174194.\n",
            "\n",
            "\n",
            "Example output: I heard that meditation can be beneficial, do you know of any tips.Yes, regular meditation can be beneficial. Studies have found that regular physical activity can help keep the body moving and stay in shape. Regular physical activity can also help maintain a healthy diet and may also support other health benefits. Additionally, learning relaxation techniques such as deep breathing, progressive muscle relaxation, and progressive muscle relaxation (PMA) can help keep the body relaxed and focused on a task. Finally, talking to a friend or counselor can help create a supportive environment for regular meditation or other stress management techniques.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 5978/5978 [1:09:55<00:00,  1.42it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Average Training Loss: 0.6872660141050636.\n",
            "\n",
            "\n",
            "Validation loss: 0.6161282828427795.\n",
            "\n",
            "\n",
            "Beginning epoch 2 of 2\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  7%|▋         | 400/5978 [04:36<1:04:12,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 400 of 5978. Loss:0.3264784812927246.\n",
            "\n",
            "\n",
            "Example output: What are the signs of an ingrown toenail.Common signs of an toenail toenail include pain when urinating, limited range of motion when urinating, and an urge to urinate when urinating. Other signs can include nausea, constipation, and tiredness.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 13%|█▎        | 800/5978 [09:14<59:36,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 800 of 5978. Loss:0.560775876045227.\n",
            "\n",
            "\n",
            "Example output: What is the best way to plan a vacation on a budget.The best way to plan a vacation on a budget is to compare prices across different sites, look for deals and discounts, and plan to have enough cash to make the most of your vacation. Additionally, consider budgeting for food, entertainment, and travel, and consider staying at a cheaper or better accommodation. Finally, if you need help staying within your budget, consider visiting a local local community center or community center with affordable or free activities or entertainment.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 20%|██        | 1200/5978 [13:54<54:58,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 1200 of 5978. Loss:0.41560453176498413.\n",
            "\n",
            "\n",
            "Example output: What is the easiest way to learn a language by myself.The easiest way to learn a language by yourself is to find a language learning program that offers tutorials and activities to help you learn the language. These programs usually provide comprehensive lessons, with examples and exercises to help you understand the language. Additionally, many provide virtual courses, which provide content and exercises in a more streamlined way than a tutorial.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 27%|██▋       | 1600/5978 [18:33<50:23,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 1600 of 5978. Loss:0.35295894742012024.\n",
            "\n",
            "\n",
            "Example output: What are some tips and tricks for mastering a computer game.Make sure to practice regularly - even if it’s only for a few minutes each day. \n",
            "2  Break down the game into manageable chunks and set achievable goals. \n",
            "3  Participate in team discussions and work together to solve problems.\n",
            "4  Identify any issues you may have and delegate the tasks to the team.\n",
            "5  Take breaks, and keep yourself motivated.\n",
            "6  Take breaks from work to reward yourself.\n",
            "7  Find a comfortable, quiet space and focus on learning the game.\n",
            "8  Test yourself often to make sure you are having fun and enjoying the game.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 33%|███▎      | 2000/5978 [23:15<45:47,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 2000 of 5978. Loss:0.5085460543632507.\n",
            "\n",
            "\n",
            "Example output: What activities can I do with my family to stay active.Some activities you can do with your family to stay active include playing active video games, doing yoga, taking a nature walk, doing a workout, going for a hike, or biking. You can also watch a movie or read a book together, or listen to music.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 40%|████      | 2400/5978 [27:53<41:10,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 2400 of 5978. Loss:0.458877831697464.\n",
            "\n",
            "\n",
            "Example output: Which country has the best soccer team in the world.The answer to this will depend on who you ask. Some people may point to countries like Germany, Italy, or Spain as the best soccer teams to play in. Each has its own strengths and weaknesses and will create its own answer. For example, some people may point to Argentina as the best soccer team to play in because of their technical abilities, while others may point to Brazil as the best soccer team to play in based on their passion and passion for their sport. Ultimately, it really comes down to what you are looking for when evaluating a soccer team and who you are interested in playing with.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 2800/5978 [32:35<36:34,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 2800 of 5978. Loss:0.8314157724380493.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r 47%|████▋     | 2801/5978 [32:36<48:37,  1.09it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Example output: What are some words that rhyme with orangeorange, orange, orange, orange, orange, orange, orange, orange, orange\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 54%|█████▎    | 3200/5978 [37:12<31:58,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 3200 of 5978. Loss:0.5375759601593018.\n",
            "\n",
            "\n",
            "Example output: What are some beginner tips for learning guitar.Start with simple songs such as 'Happy on Their Own' by Phish.\n",
            "2. Learn chords and scales on guitar.\n",
            "3. Practice and refine your technique.\n",
            "4. Learn improvisation and practice scales and chords together.\n",
            "5. Learn to play along with songs.\n",
            "6. Learn to play along with songs written by other musicians.\n",
            "7. Listen to recordings of professional guitarists.\n",
            "8. Use online lessons and tutorials to learn the basics.\n",
            "9. Play and enjoy playing guitar.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 60%|██████    | 3600/5978 [41:52<27:21,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 3600 of 5978. Loss:0.5550445914268494.\n",
            "\n",
            "\n",
            "Example output: I'm applying for a new job and need to know what to expect in an interview in my industry.In an interview for your industry, you should expect the interviewer to ask a variety of thoughtful questions to demonstrate that you understand and demonstrate your enthusiasm for the role. Additionally, it is important to be honest, clear, and respectful throughout the interview.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 67%|██████▋   | 4000/5978 [46:31<22:45,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 4000 of 5978. Loss:0.33791348338127136.\n",
            "\n",
            "\n",
            "Example output: What are the best YouTube channels for cooking tutorials.The best YouTube channels for cooking tutorials are those that provide step-by-step instructions for simple cooking projects. They usually focus on food basics like boiling, roasting, or grilling, and usually have tutorials on the basics of basic recipes. YouTube also has channels that offer cooking tutorials in a variety of cuisines, from traditional Chinese to West African. Additionally, YouTube has a wealth of cooking instructional materials ranging from video tutorials to infographics and much more. Good luck.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 4400/5978 [51:11<18:09,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 4400 of 5978. Loss:0.47749656438827515.\n",
            "\n",
            "\n",
            "Example output: Can you provide a recipe for making tomato soup.Ingredients:\n",
            "- 3 cups diced onion\n",
            "- 1 onion, diced\n",
            "- 2 cloves garlic, minced\n",
            "- 1 pound beef stew meat\n",
            "- 1 teaspoon Worcestershire sauce\n",
            "- 1 teaspoon dried basil\n",
            "- 1 teaspoon dried thyme\n",
            "- 1 teaspoon ground coriander\n",
            "- 1/2 teaspoon paprika\n",
            "- 1/2 teaspoon cumin\n",
            "- 1/4 teaspoon garlic powder\n",
            "- Salt and pepper, to taste\n",
            "\n",
            "Instructions:\n",
            "1. Heat a large pot over medium heat.\n",
            "2. Add the onion, garlic, and beef. Cook until the beef is softened and the onion is fragrant, about 15 minutes.\n",
            "3. Add the Worcestershire sauce and basil, and cook for 1 minute more.\n",
            "4. Add the thyme, coriander, paprika, and cumin and cook for 1 minute more.\n",
            "5. Stir in the soup broth\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 80%|████████  | 4800/5978 [55:55<13:33,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 4800 of 5978. Loss:0.4787990152835846.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r 80%|████████  | 4801/5978 [55:57<23:44,  1.21s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Example output: Can you explain updateless decision theory?UDT is a decision theory that aims to solve an issue present in decision-making theories, which is the need to consider the decision maker as a component of the environment in which the decisions are made.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 87%|████████▋ | 5200/5978 [1:00:33<08:57,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 5200 of 5978. Loss:0.311775803565979.\n",
            "\n",
            "\n",
            "Example output: I'm having trouble with algebra, can you help me solve this equation.I'm sorry, I'm not able to help you with algebra equations. First, let's look at some examples. Let's consider an equation with solutions to it. What is the equation y=x+2 and what is the second equation y=2+x? First, we need to know what is the x-coordinates of the two equations. Then we need to know the solutions for the first equation. Once we have all of this information, we can use algebra to solve the equation y=2+x.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 94%|█████████▎| 5600/5978 [1:05:13<04:20,  1.45it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Batch 5600 of 5978. Loss:0.44621706008911133.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\r 94%|█████████▎| 5601/5978 [1:05:15<06:27,  1.03s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Example output: Can you define data augmentation?Data Augmentation is a method that generates \"fake samples\" by modifying the training data to create synthetic instances of the true data.\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 5978/5978 [1:09:35<00:00,  1.43it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Average Training Loss: 0.46858145299066306.\n",
            "\n",
            "\n",
            "Validation loss: 0.5825508199688187.\n",
            "\n",
            "Training complete!\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "('/content/drive/MyDrive/Colab Notebooks/Aira-774M/tokenizer_config.json',\n",
              " '/content/drive/MyDrive/Colab Notebooks/Aira-774M/special_tokens_map.json',\n",
              " '/content/drive/MyDrive/Colab Notebooks/Aira-774M/vocab.json',\n",
              " '/content/drive/MyDrive/Colab Notebooks/Aira-774M/merges.txt',\n",
              " '/content/drive/MyDrive/Colab Notebooks/Aira-774M/added_tokens.json')"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from codecarbon import EmissionsTracker\n",
        "import tqdm\n",
        "\n",
        "output_dir = f'./Aira-{model_size}'\n",
        "\n",
        "tracker = EmissionsTracker(\n",
        "    project_name=\"Aira_emissions\",\n",
        "    log_level=\"critical\",\n",
        "    output_dir=output_dir,\n",
        "    output_file=\"Aira_emissions.csv\",\n",
        ")\n",
        "\n",
        "training_stats = []\n",
        "\n",
        "tracker.start()\n",
        "for epoch_i in range(0, epochs):\n",
        "\n",
        "    print(f'\\nBeginning epoch {epoch_i + 1} of {epochs}\\n')\n",
        "\n",
        "    total_train_loss = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for step, batch in enumerate(tqdm.tqdm(train_dataloader)):\n",
        "\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_labels = batch[0].to(device)\n",
        "        b_masks = batch[1].to(device)\n",
        "\n",
        "        model.zero_grad()\n",
        "\n",
        "        outputs = model(b_input_ids,\n",
        "                        labels=b_labels,\n",
        "                        attention_mask = b_masks,\n",
        "                        token_type_ids=None)\n",
        "\n",
        "        loss = outputs[0]\n",
        "\n",
        "        batch_loss = loss.item()\n",
        "        total_train_loss += batch_loss\n",
        "\n",
        "        if step % sample_every == 0 and not step == 0:\n",
        "\n",
        "            print(f'\\nBatch {step} of {len(train_dataloader)}. Loss:{batch_loss}.\\n')\n",
        "\n",
        "            model.eval()\n",
        "\n",
        "            inputs = tokenizer(tokenizer.bos_token + df.prompt.sample().iloc[0] + tokenizer.eos_token, return_tensors=\"pt\").to(device)\n",
        "\n",
        "            sample_outputs = model.generate(**inputs,\n",
        "                                bos_token_id=tokenizer.bos_token_id,\n",
        "                                pad_token_id=tokenizer.pad_token_id,\n",
        "                                eos_token_id=tokenizer.eos_token_id,\n",
        "                                do_sample=True,\n",
        "                                top_k=50,\n",
        "                                max_length = 200,\n",
        "                                top_p=0.95,\n",
        "                                num_return_sequences=1)\n",
        "\n",
        "            for i, sample_output in enumerate(sample_outputs):\n",
        "                  print(f'\\nExample output: {tokenizer.decode(sample_output, skip_special_tokens=True)}\\n')\n",
        "\n",
        "            model.train()\n",
        "\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "\n",
        "        scheduler.step()\n",
        "\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)\n",
        "\n",
        "\n",
        "    print(f'\\nAverage Training Loss: {avg_train_loss}.\\n')\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    for batch in validation_dataloader:\n",
        "\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_labels = batch[0].to(device)\n",
        "        b_masks = batch[1].to(device)\n",
        "\n",
        "        with torch.no_grad():\n",
        "\n",
        "            outputs  = model(b_input_ids,\n",
        "                             attention_mask = b_masks,\n",
        "                             labels=b_labels)\n",
        "\n",
        "            loss = outputs[0]\n",
        "\n",
        "        batch_loss = loss.item()\n",
        "        total_eval_loss += batch_loss\n",
        "\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "\n",
        "\n",
        "    print(f'\\nValidation loss: {avg_val_loss}.\\n')\n",
        "\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "        }\n",
        "    )\n",
        "\n",
        "tracker.stop()\n",
        "print(\"Training complete!\")\n",
        "\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "df_stats.to_parquet(f\"{output_dir}/training_stats.parquet\", compression=\"gzip\")\n",
        "\n",
        "model_to_save = model.module if hasattr(model, 'module') else model\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h48iOsqie8rx"
      },
      "source": [
        "13. Check the training stats and plot the learning curves."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 439
        },
        "id": "J1-hAY9Av4gT",
        "outputId": "d8909676-fba0-470f-80cd-22d29cf2ecb1"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABBsAAAI/CAYAAAA2r9HeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACZuElEQVR4nOzdd3iV5f3H8fcZ2XtABhASEgh7IyDIdKCgogxr0dZRseJo689WW6t1a1urrbgXKJVWUIaLIciSvffKIAkJJJC9yDrn90dMakzCPMlzTvJ5XZdX9Tz3cz+fE3NTz/fcw2S32+2IiIiIiIiIiDiI2egAIiIiIiIiItKyqNggIiIiIiIiIg6lYoOIiIiIiIiIOJSKDSIiIiIiIiLiUCo2iIiIiIiIiIhDqdggIiIiIiIiIg6lYoOIiIiIiIiIOJSKDSIiIiIiIiLiUCo2iIiIiIiIiIhDqdggIiKtzubNm4mPjyc+Pt7hfS9YsID4+HjGjBnj8L7FMW6//Xbi4+OZOXPmBV271L6bw5gxY4iPj2fBggWGPF9ERKSG1egAIiLSMl3KB/kXX3yRm2++2YFp5ELt3r2bDz/8kO3bt5OXl0dAQAAREREMGzaMa6+9lq5du15UvydOnGDMmDHYbDb+8Ic/cPfdd5/XfYsWLeLRRx8Fqgs6PXr0uKjnu6oFCxaQnp7OZZddxuDBg42O43CPPfYYCxcupF27dnz33XdGxxEREQdQsUFERJpEaGhog6+XlJRQUlJy1jaenp5NlgvAy8uLmJiYJunbz8+PmJgYwsLCmqT/5vDZZ5/xxBNPYLPZgOqfV0lJCXv37mXv3r3s2LGDOXPmXFTfERERXH755Xz//fcsWLDgvIsNn3/+OQDdunVr0kJDREQEMTExBAUFNdkzLsbChQvZsmULDzzwwFmLDR06dMDd3R0/P79mTCciIlKfig0iItIk1q9f3+DrM2fO5PXXXz9rm6bWu3dvli5d2iR9X3XVVVx11VVN0ndzyMnJ4ZlnnsFms9GtWzeee+45evbsCUBaWhrfffcdSUlJl/SMyZMn8/3335OQkMDu3bvp06fPWdunpaWxdetWACZNmnRJzz6Xv/3tb03af1P76KOPjI4gIiICqNggIiIiP7Jt2zbKysoA+Pvf/07nzp1rr3Xo0IFf/vKXl/yMsWPHEhgYSF5eHp9//vk5iw0LFizAbrfj7u7O9ddff8nPFxERkaanYoOIiDiVmr0ePv74Y+Li4nj33XdZvXo1J0+e5MyZMxw+fBiA0tJSVq5cydq1azl8+DCZmZkUFRURGBhI7969ueWWWxg5cmSDz9i8eTO/+MUvAGr7q7FgwQL++Mc/1q4d37dvH++9917t3gVhYWFceeWVzJgxg4CAgHp9//T+H6uZ1XHZZZcxZ84cNm7cyKxZs9izZw/FxcW0b9+e8ePHc8899+Dh4dHoz2jFihV8/PHHHDhwgKqqKjp06MD111/PHXfcwdtvv13nGRfKYrHU/n1TLQVxd3fnxhtv5KOPPuLrr7/mT3/6U6NLZ2w2G4sWLQKqZ40EBgYCcOTIEZYtW8bWrVvJyMggKysLq9VKVFQUI0eO5Je//CXBwcEXnO3222+vXa7w4IMP1rteVVXF3LlzWbBgAcnJybi7uxMfH8+0adMYN27cWftOS0tjyZIlbN68mePHj5OZmYnJZKrdC+POO+8kMjKyzj01v081Xn/99dqZQTVWrlxJ+/btgeoNItPT0xvd96SqqoqFCxfyxRdfcPjwYYqLiwkKCqJfv35Mmzat0SUaP/65PPDAA8yfP5/58+eTmJiI3W6nS5cu/PznP+fGG28868+gKZw6dYoPP/yQtWvXkp6eDkC7du0YOXIkd911V6PLtfLz85k9ezarV68mJSWF8vJyAgICCA4Opl+/flx77bUMHTq0zj1nzpzhk08+Yfny5SQlJVFSUoKfnx/BwcH06tWLMWPGcM011zT5exYRcQUqNoiIiFNKTU3l4Ycf5vTp03h4eGC11v2/rCVLltR+CDOZTPj6+mK1Wjl16hQrV65k5cqV3HXXXbWbCl6ML7/8kj/+8Y9UVFTg5+dHVVUVx48fZ/bs2axfv55PP/0UHx+fi+r7/fff5+WXXwaq93moqKggKSmJmTNnsmXLFmbNmlXng3+Nv/71r3z44Ye1/+zv709iYiIvv/wya9asYcCAARf3Zn8wdOhQgoODycnJ4eOPP+aBBx64pP4aM3nyZD766COKiopYtmxZox9SN27cSEZGBlB3CcWvf/3r2g+WHh4eeHl5kZ+fz8GDBzl48CALFy5k9uzZdOrUyWGZy8vLue+++/j+++8BMJvNuLm5sXXrVrZs2cI999xz1vv/9Kc/sWXLFgDc3Nzw8fGhoKCAxMREEhMTWbhwIW+//TYDBw6svcfT05PQ0FDy8/OpqKjA29sbb2/vOv029HvSkMLCQmbMmFGbwWKx4OPjw6lTp1i2bBnLli0755ipqqri/vvvZ+XKlVitVjw9PSkuLmbXrl3s2rWLlJQUHnroofPK4whbtmzh/vvvp6CgAKD2Z5OQkEBCQgKfffYZb775Zp2fKcDJkye59dZba3+3zGYzfn5+5Obmcvr0aY4cOUJycnKdYkNRURHTpk3j0KFDQPWfO35+fhQWFpKbm0tiYiJbt25VsUFE5Ac6+lJERJzSCy+8gJ+fH7Nnz2bXrl3s2LGjzj4L/v7+3HXXXcydO5edO3eybds2du3axbp163jwwQdxc3Pjww8/ZOXKlRf1/JycHP70pz8xceJEVq9ezbZt29ixYwdPPvkkbm5uHD16lPfff/+i+j506BD/+Mc/mD59Ohs2bGDr1q1s27aN+++/H6ieebFw4cJ693399de1hYYJEyawdu1atm7dyo4dO3j22WfZs2cP//nPfy4qUw1vb+/aD5tvvPEGX3zxxSX115guXbrQu3dv4H+bPzak5lq7du3qfPAbNGgQL730EqtWrWLPnj1s3ryZPXv2MHv2bHr37k1mZiaPPPKIQzP/4x//4Pvvv8dkMvHb3/6WrVu3snXrVtavX8+tt97Ke++9x8GDBxu9v2vXrjz55JMsW7asNvPevXuZP38+V1xxBYWFhfzud7/jzJkztfdcd911rF+/nn79+gFw1113sX79+jp/RUREnFf+xx9/nC1btuDm5saf//xntm/fztatW1m3bl1tIefDDz886+/Q3Llz2bJlCy+99BLbt29n+/btrFmzhtGjRwPw1ltvcezYsfPKc6lOnDhRW2iIi4ur/bNg586dfPLJJ8TExJCfn8/9999PZmZmnXtnzpxJRkYG7dq1Y/bs2ezbt48tW7awd+9evvvuO5566ql6y3s+/vhjDh06RGBgIDNnzmTPnj1s3bqVvXv3snbtWv76178ybNiwZnnvIiKuQMUGERFxSmazmdmzZzN06FDM5ur/u/rxCRJXXnkljz76KAMGDMDLy6v29bZt2/LAAw/wu9/9DuCiT00oLS1l/PjxPPfcc7Uf5ry8vJg2bRq33XYbUP3h/2IUFBQwY8YMHn744dqp/r6+vjz00ENcffXVDfZtt9v517/+BcCwYcN4+eWXa5c5eHh4MHXqVJ566iny8/MvKlON9PT02iKKzWbjscceO2sx4FJMnjwZqP52Oi0trd71/Px8VqxYAcDNN99c+3sA1TM8brrppjrLDtzd3Rk6dCizZ88mNDSU/fv3s23bNodkzczM5N///jcA9913H/fddx++vr4AhISE8NRTTzFhwgQKCwsb7ePxxx9n2rRpREdH174Xq9VK7969eeedd4iPjycrK4tly5Y5JPOP7d69u7bfJ554gttvv7123LRp04YXXnih9hv5f/3rX7X7dvxUfn4+r7/+OjfddFPt0pfw8HBee+012rZti81mY8mSJQ7P35C3336bgoICAgICmD17dp1ZPQMHDmT27Nn4+vqSl5fHO++8U+fenTt3AvDwww8zdOjQ2tkhFouFdu3aceutt9YrVtXcc9ddd3H11Vfj7u4OVP9ZFRYWxsSJE3n22Web7P2KiLgaFRtERMQp3XjjjYSHh1/0/aNGjQJg165dVFVVXVQf9913X4Ovjx07FoCUlBRKS0svuF93d3fuuuuus/b9070kDh48SEpKCgD33nsvJpOp3r0//fB9ofLz8/nlL3/J0aNHufXWW/nXv/6FyWTi8ccfb7Ro88knnxAfH39RU8fHjx+Pl5cXdru9wZkcX331FWVlZZjNZm666abz7tfHx4dBgwYBsGPHjgvO1ZBly5ZRWVmJp6dno8d1XsqSE4vFwhVXXAHA9u3bL7qfxnzzzTdAdWFgypQpDbb5zW9+A0Bubm6jJ8X079+fIUOG1Hvd3d2d4cOHA/V/d5uC3W6vnen0s5/9jDZt2tRrEx4ezs9+9jOgfvHO398fqN7v4XxdzD0iIq2Z9mwQERGn1L9//3O2OX36NHPnzmX9+vUcO3aMwsLCeoWF0tJS8vPzL3izwMDAQDp27NjgtbZt29b+fUFBQZ2ZFeejc+fOje71UNP3T2co7N+/H6he618zpf6nTCYTgwYNYvHixReUp8Zzzz1HWloaffv25YknnsBisVBVVcXvf/97nnvuOUpKSrj33nvr3FMzPb1bt24X/DxfX1+uueYaFi1axKJFi3jggQfqzF6omVExdOhQ2rVrV+/+VatWsXjxYvbu3Ut2dnaDhZ+TJ09ecK6G7Nu3D4CePXvWzmj4qZiYGMLCwupN2f+xbdu28dlnn7Fr1y4yMzMpKSmp1+Zs91+smvyDBw+u8zP+sdjY2Nr8+/btY8yYMfXanO3kkMZ+d5vC8ePHycvLA6i3ieOPDRs2jPfff5+8vDzS0tLo0KEDUF2M3LlzJ//4xz9ISkriqquuon///o3+u62556uvvuLf//43OTk5XHfddfTv3/+iNiIVEWkNVGwQERGnFBISctbrO3fuZPr06bUbw0H1fgNeXl6YTCaqqqrIzc0FuKjZB2fb+PHHG/JVVFQ0Sd+VlZV1Xq95L4GBgbXTtxtysSdInDp1qvbb7xkzZtTmGD9+PBUVFfzxj3/klVdeobi4mIcffrj2vq1btwLUrtm/UJMnT2bRokWkp6ezcePG2jXvhw4dqi2w1Cy3qGGz2fj973/PV199Vfua1WolICAANzc3oHozxLKysov6d9+Q7Oxs4Nw/3/Dw8EaLBX//+9/r7PNhsVjqZC4pKan9y9EuNH9N+5862+9uzSauP/3dbQo/zne29/Tjazk5ObXFhrvvvptDhw6xZMkS5s2bx7x58zCZTHTu3Jnhw4czZcqUepuLXn/99ezZs4d///vffP3117WzJTp27MiwYcOYNGkSPXv2dOTbFBFxaVpGISIiTqmxb1+h+sPM//3f/1FQUEC3bt1499132b59Ozt37mTDhg2sX7+eefPm1ba32+3NEdmlHThwoPZD4k9PtJg4cSLPPfccJpOJd955h+eeew673U5SUhI7d+4kICCAK6+88qKeO2jQIKKjo4HqYx5r1Px9YGBgvb4/++wzvvrqKywWC/fffz/Lly9n7969bNmypXbTxJplHc7y7379+vW1hYaf//znfPnll/Uy//KXvzQ4Zevh5ubGP//5TxYvXsz999/PkCFD8PLy4siRI3z44YdMmDChzqkvNR5//HGWLl3Kww8/zIgRI/D39yclJYW5c+cyadIknn/+eQPejYiIc9LMBhERcTm7du0iPT0di8XCO++80+A3my1tXXVQUBAAeXl5lJeXNzq74WKn4BcXF5/1+qRJk6isrOQvf/kLc+bMobi4mIKCAux2O7/85S8v+gjQmr7/8Y9/8O2339YuS6k5BeP666+v915rvlGePHlyo8csnj59+qLzNKRmps25fr6NXa/JPHz4cP7yl7802MbRmX8sJCSE5OTkcy4rqbl+rplFRvtxvszMzEaPOP3xv4+Gljt07dqVrl27AtVFzK1bt/LGG2+wdetW/va3v3H55ZfXXq/RsWNH7r33Xu69915sNht79uzhvffeY8WKFXz88ccMGTKkdu8VEZHWTDMbRETE5Zw4cQKo/vDQ2BTqjRs3NmekJtejRw+getlGza74P2W32y/69IWa6eUAmzZtarDNLbfcwhNPPAFUzzxYsWIFMTEx/OpXv7qoZ9aYOHEiFouFsrIyvvzyS7777rvaZSM/XUIB//tA3L179wb7Ky4uZvfu3ZeU6adqpsfv27ev0cLMsWPHGv0wf67Mdru90Z87ULsh6MXO1KjJv3nzZmw2W4NtEhMTaz+c9+rV66Ke01zat29PYGAgcPaxvmHDBqB6hsyPf8cbYrVaGTp0KO+88w7u7u7Y7fba+xtjNpvp27cvr732Wu3mrOe6R0SktVCxQUREXI6fnx9Q/U1wQ98Gnzx58qKPvHRW3bp1q92w8t13323wQ+fixYtJT0+/qP579uxJVFQUUL23QM2H/Z+aNm0a1157be0/d+3aFQ8Pj4t6Zo22bdsyYsQIoLqIUbOEokePHvW+VQZqN/E7dOhQg/29+eab55ypcaGuueYaLBYLZ86caXB6PcAbb7zR6P3nyvyf//ynweM/f3r/j/couRDjx48Hqr/pnz9/foNtXnvtNaB6Fs3ll19+Uc9pLiaTqfb38NNPP21wJlNmZiaffvopABMmTKhzrby8vNG+3d3da/cs+fFyrrPdY7FYavfeaOikGBGR1kjFBhERcTkDBgzA29sbu93Ob3/7W5KTkwGoqqpi3bp13H777QYndDyTycSDDz4IwPfff8+jjz5a+y10WVkZ8+fP5y9/+QsBAQEX3f+TTz6JxWLh2LFjTJkyhWXLllFWVgZU/2x37NjBQw89xJIlS2o/UC1ZsoRXX331kt9fzQyGffv2sXbtWqB6eUVDao6InD9/Pp9++mnth8BTp07xwgsv8P7779d+6+0oYWFh/PznPweqixnvvPMORUVFQPXGg8888wxffPFFbSGsscxr167ljTfeqN0EsqCggLfffpvnnnvurJk7d+5ce//FLJXp3bt37T4Wzz77LP/+979rN888deoUf/7zn2uPkvzNb35zyQWki2Wz2cjJyTnrXzU/91//+tf4+/uTl5fHnXfeWeeY0+3bt3PnnXdSUFBAYGAg06dPr/Oc0aNH849//INdu3bVKSKkpKTwyCOPUFpaitlsrj3OE2DKlCk899xzbN68uc4mnpmZmTz77LO1R9OOHDmySX42IiKuRns2iIiIy/Hz8+MPf/gDTz31FFu3bmXcuHF4e3tTVVVFWVkZQUFBvPjii9x3331GR3Wo66+/nr179/LRRx+xePFivvjiC/z9/SkpKaGiooIhQ4bQp0+f2mngF+qKK67glVde4fHHHyctLY2HHnoIq9WKr68vxcXFtSdvREZG8sILL7B27Vo+/PBD3n77bdq0acNtt9120e9t1KhRhIaGcvr0aWw2Gx4eHlx//fUNtr3rrrtYtmwZSUlJPPnkkzz11FP4+vpSWFiI3W7nlltuoby8nIULF150nob8/ve/JzExkQ0bNvDKK6/wr3/9C19f39q9K+655x52797Nli1b6t07ceJEFi1axLZt23jttdeYOXMm/v7+FBYWYrPZGDVqFN26deOtt95q8Nk33XQTs2bNIiUlhVGjRhEcHFxbEJg7dy7h4eHnzP/888+Tm5vLli1bePbZZ3nxxRfx8fGpzQ/VP9tbb731En5Kl+bEiRNnPcoSYOzYsbz55puEh4fzxhtvMGPGDI4ePcqtt96Kt7c3QG0xwN/fnzfeeKPecqvTp0/z7rvv8u6772I2m/Hz8+PMmTO1xTWTycSjjz5KXFxc7T2FhYXMmTOHOXPmYDKZ8PPzo7Kysk7h4Y477qgtLImItHYqNoiIiEu69dZbiYyM5P3332ffvn1UVVURFhbGyJEjueeeey7qSEpX8Kc//YlBgwbx8ccfc+DAAcrLy+nUqRM33ngjv/zlL3nppZeA6g9ZF2PcuHH079+fuXPnsnbtWlJSUiguLiYwMJAePXpw1VVXccMNN+Du7s7gwYM5duwY3333Hc8//zwhISF1llhcCKvVysSJE2tPbLjqqqsafQ/+/v7897//5Y033mDFihVkZWVhsVi47LLLuOWWWxg/fjyPPfbYReU4Gw8PD9577z3mzp3LggULSE5Oxm63M3DgwNrlJY3NqnFzc+PDDz/k3Xff5auvviI9PR273U7v3r2ZOHEit9xyy1mXYURHR/Pxxx/zzjvvsGfPHvLy8mpPDznfoyb9/PyYPXs2CxcuZPHixRw+fJiSkhJCQ0Pp378/06ZNY/DgwRf+gzHQZZddxjfffMOsWbNYs2YN6enpmEwmYmNjGTlyJHfddRdt2rSpd9+HH37I5s2b2b59OydOnKhdjtWxY0cGDBjAtGnT6h1j+corr/D999+zbds2jh8/zunTp6msrKRdu3b06dOHqVOnnrNQIiLSmpjsznImlIiIiFyyn/3sZ+zcuZOHHnqI+++/3+g4IiIi0kppzwYREZEWYsuWLbUnVWgqt4iIiBhJxQYREREX8vTTT7NgwQJOnTpVu86+oKCA//73v8yYMQOAIUOG0Lt3byNjioiISCunZRQiIiIu5MYbb6w9PtHd3R0vL686G/zFxcXx4Ycf1tsQT0RERKQ5qdggIiLiQlauXMmKFSvYs2cPp0+fpqioCF9fX+Li4rjqqqu45ZZb8PLyMjqmiIiItHIqNoiIiIiIiIiIQ2nPBhERERERERFxKBUbRERERERERMShrEYHkLOz2+3YbOe30sVsNp13WxFpHhqXIs5H41LEuWhMijgXs9mEyWS65H5UbHByNpudnJzic7azWs0EBflQUFBCZaWtGZKJyLloXIo4H41LEeeiMSnifIKDfbBYLr3YoGUUIiIiIiIiIuJQKjaIiIiIiIiIiEOp2CAiIiIiIiIiDqVig4iIiIiIiIg4lIoNIiIiIiIiIuJQKjaIiIiIiIiIiEOp2CAiIiIiIiIiDqVig4iIiIiIiIg4lIoNIiIiIiIiIuJQVqMDiIiIiIiINDW73U5VVSV2u93oKCLNwmw2Y7EY95FfxQYREREREWmxKisrKCzMo7z8DHa7zeg4Is3KanXHx8cfLy+f5n92sz9RRERERESkGZSXl5Gbm4XZbMbHxw83Nw/MZjNgMjqaSBOzU1VVRUlJEfn5pwGaveCgYoOIiIiIiLRIRUV5WCxWgoPDfigyiLQebm7g4eFFbu4piosLmr3YoBEnIiIiIiItTlVVFeXlZ/Dx8VOhQVotk8mEt7cPlZXlVFVVNuuzNbOhBbDZ7Bw8lkNFci5uJjuxkQGYzZoaJiIiIiKtl81WBYDV6mZwEhFj1WwSabPZsFia77kqNri47YezmLviKLmFZbWvBfl58PMrOzMgvq2ByUREREREnIG+hJPWzpgxoPlELmz74SzeWLivTqEBILewjDcW7mP74SyDkomIiIiIiEhrpmKDi7LZ7MxdcfSsbf6z4ig2m84RFhERERERkealYoOLOpKWV29Gw0/lFJZxJC2veQKJiIiIiIiI/EDFBheVV3z2QsOFthMREREREXEGzz//FMOHD+Sbb750WJ8PPDCd4cMHsmPHNof1KWenDSJdVKCPh0PbiYiIiIhI6zR8+MCLum/+/C+IiIh0cBppKVRscFFdOgQS5Odx1qUUwX4edOkQ2HyhRERERETE5fTq1afeaxUVFRw6dACArl274+ZW/whRd3f3JskTEhJKVFRHfHx8HdZnWFg4UVEd8fT0dFifcnYmu92uHQSdWFWVjZyc4gav1ZxG0Zj7b+qp4y9FDGS1mgkK8iE3t5jKSpvRcUQEjUsRZ9OUY7Kiopzs7BOEhETg5tY0H4pbshMnMpgy5QZAMxhc3YWOheBgHyyWS99xQXs2uLAB8W25/6aeBPnVXyoR4OtOn7hQA1KJiIiIiIhIa6dlFC5uQHxb+nVuQ2JGPhV2E7bKSt79Yj/5ReWs253B6P7tjY4oIiIiItKq2Gx2jqTlkVdcRqBP9dJms9lkdCyH+fGsh++/38aaNauYP/8/JCYmUFhYwKxZn9C5czzZ2adZvfo7Nm78ntTUFE6fPo3VaqVjx46MGXM1kyZNbXApxvPPP8WSJV/xpz/9heuuu7729W+++ZIXXniavn37M3PmOyxe/DmLFy8gNTUFd3cP+vbtxz33zKBTp9h6fT7wwHR27drBa6+9Tf/+/9uj4oMP3mHWrPe49toJPPron/nPf+awdOnXnDiRgbe3D4MHD2H69PsJCwtv8Gdx6lQW77//Nps2baCwsIC2bcMYO/ZqfvGLu3j55RcbfB+thYoNLYDZbKJbdHDtFLQbh3fik2+PsPj7ZIb0CMfLQ/+aRURERESaw/bDWcxdcbTO3mpBfh78/MrOLXKJ8yeffMRbb80kMDCI9u3bk5WVWXvtyy8X8f77b+Pu7kFISCixsbHk5+dz5MhhDh48wNq1q3jttbcb3A/iXJ577i8sW/YNERGRREV1JCUlhXXr1rBz53bef38O7dt3uKD+Kisr+b//e5Dt27fSoUMU7dt3IDU1hWXLlrBz5w5mz56Lv39AnXtSU1O4//57yM3NwWq10qlTLGVlZXz00Qds27al1S890afQFmhk30hWbEsjM7eUZVtSmXhFJ6MjiYiIiIi0eI3tqZZbWMYbC/e1yD3V3n//bR5++FEmTpyE2WzGZrNRVVUFQL9+A3n11Tfo128AVuv/PnpmZWXy6qt/Z9261fz3v//m9tvvvKBn7tu3h5SUY7z++rv07dsfgIKCfP74x0fYvXsnH3zwDn/5y3MX1OeqVSsID4/ko4/+S2xsHAAnT57kkUce5NixZP7zn39z773317a32+0888wT5Obm0KtXb5599q+EhrYB4MiRQ/zhD7/j8OGDF5ShpdGeDS2Q1WJm0sjqqUNLt6SSV9T4iRUiIiIiIq2V3W6nrLzKIX+Vnqnkk2+PnPV5c1ccpfRMpUOe5yz7/F9//URuvnkKZnP1R0uz2Vw7U6FPn74MGjS4TqEBoG3bMP7yl+ewWq0sXfr1BT+zsrKS3/72kdpCA4C/fwC/+c3/AbBx4/qL6vPPf366ttAAEB4ezj33zGiwzx07tnHo0AE8PT159tm/1RYaALp06crjj/+FysrKC87RkmhmQws1IL4NsZH+JGYUsPj7ZH45rqvRkUREREREnIbdbufFf+8gIT2/2Z6ZW1jG/f9c65C+4toH8Mdp/TGZjN0L4lx7EZSVnWHVqpXs3r2TzMxMzpwprS2UmM1mUlNTKCs7g4fH+R9J6evrx9ixV9d7vUuXrri7u1NUVEh+fh4BAYHn3WdcXBd69uxV7/UePapfS08/Xuf1zZs3ADBkyDBCQ+tvzD9o0BDCwyM4efLEeWdoaVRsaKFMJhNTRsfx0ic7WLs7g6sGdiAy1MfoWCIiIiIizqPl7NlomI4dYxq9lpSUyKOP/o4TJzLO2kdBQQFt2px/seFs+zEEBgaRlZVJaWnpBRUbGuszODgYgNLSkjqvp6WlAhAX17nRPuPiOqvYIC1Tlw6B9Oscys6jp/lsdSIPTe5tdCQREREREadgMpn447T+lFfYHNLfkbQ8Xp2/+5ztfjelD106BF7y89zdzIbPagDw8vJq8PWqqiqeeOJRTpzIYMCAy7jttl8SF9cZPz//2mUVN988nqyszAtebuDp2XhhomY5x4UuM2nsfdT091MlJaUAeHs3/oXu2a61Bio2tHCTR8WyOyGbXQmnOZyaS3xUkNGRREREREScgslkwsPd4pC+esQEE+TnUecUip8K9vOgR0xwizoGszEHDx4gJeUYbduG8be/vVJvmYTdbqewsNCgdJfO27u6OFFSUtxom7Ndaw20QWQLFxHiw4i+1UeuzFuV6DQbyYiIiIiItCRms4mfX9n4lHqAW6/s3CoKDQAnTqQD0K1b9wb3Y0hKSqy3NMGVdOgQBUBiYkKjbc52rTVQsaEVuHFYNB5uFpJPFLDt8Cmj44iIiIiItEgD4tty/009CfLzqPN6sJ9Hizz28mxqljpkZ2c3eH3u3I+bM47DDR58OQCbNq0nJ6f+e9y2bcs596po6bSMohUI8PVg3OAoFn+fzOerE+nXORSrRXUmERERERFHGxDfln6d23AkLY+84jICfTzo0iGw1cxoqNGjRy+sViv79u1h8eIF3HjjzQBUVFQwe/b7LF++BDc3NyoqKgxOenH69x9It27dOXjwAH/+86M888xLtadSHD16mBdeeBqr1dqqj79UsaGVuOayDqzamU5WXimrd6Zz5cDGd3AVEREREZGLZzab6Nqxde+VFhwcwq233s6cObP4+99fYNas9wgNbcPx46kUFRVx99338vXXX7jsaQ0mk4knnniW+++/hz17djF58gQ6dYqlvLyCY8eS6N69J71792XFimWNbjLZ0rXOd90KebpbmTi8+liaL9Yfo+RM662wiYiIiIhI07v33vt55JE/EhsbR35+HsePpxEX14Vnn32JO++8x+h4lywqqiMffDCH8eNvICAggGPHkikvL+O22+7gtdferp3V4OPTOk+lMNm1Y6BTq6qykZNz7l1MrVYzQUE+5OYWU1nZ8PE9VTYbT7y/hZM5JYwf2pFJI2MdHVdEfuR8xqWINC+NSxHn0pRjsqKinOzsE4SERODm5u7QvkXOx+23TyU5OYlZs+bSuXMXw3Jc6FgIDvbB4oBl95rZ0IpYzGamjKouMCzfmkZOwRmDE4mIiIiIiLQ8+/fvIzk5CX//AGJiOhkdxxAqNrQyfTuH0rl9ABWVNhZ9n2x0HBEREREREZeUlpbK/Pn/pbCwsM7re/bs4sknHwPghhtuwmptnVslts533YqZTCamjo7j+TnbWb/3BFcP7ED7tr5GxxIREREREXEpxcVF/OtfL/P666/SoUMU3t4+nD59iqysTAB69erNnXf+yuCUxtHMhlYotl0AA+PbYLfDZ2sSjY4jIiIiIiLiciIj2/OLX9xFly7x5Ofnc+TIIYqLi+jRoxcPPfR//Otfb+Ph4Wl0TMNoZkMrNWlkLDuPnmZPYjYHj+XQLTrY6EgiIiIiIiIuw9/fn+nTZzB9+gyjozglzWxopcKCvRnVtx0A81YnYtOhJCIiIiIiIuIgKja0YtcPi8bT3ULKyUK2HMw0Oo6IiIiIiIi0ECo2tGL+Pu5cO6QjAAvWJFGh88ZFRERERETEAVRsaOWuHtSBQF93TuefYdWO40bHERERERERkRZAxYZWzsPNwsQrOgHw5YZjlJypMDiRiIiIiIiIuDoVG4RhvcKJDPWh+EwlX29KMTqOiIiIiIiIuDgVGwSL2czkUbEAfLv1ONn5ZwxOJCIiIiIiIq5MxQYBoE9sCPEdAqmssrFoXZLRcURERERERMSFqdggAJhMJqaMjgNgw76TpGYWGpxIREREREREXJWKDVKrU6Q/l3Vrix34bHWi0XFERERERETERanYIHXcPKITFrOJfck57E/OMTqOiIiIiIi0QMOHD2T48IH1Xn/ggekMHz6QHTu2XVB/O3ZsY/jwgTzwwHRHRTynEycyGD58IJMnX99sz3QlKjZIHW2DvBndvx0A81clYLPbDU4kIiIiIiJN6fnnn2L48IH83/89dF7tc3KyGTlyMMOHD2Tr1s1NnM44H3zwDh988A6FhVpifjFUbJB6rr88Gi8PC6lZRWzen2l0HBERERERaULXXjsBgG3bNpOdffqc7ZcvX0JVVRVt24YxYMAgh2YJCwsnKqojnp6eDu33Ysya9R6zZr1HUVHDxQar1UpUVEfatWvfzMlcg9XoAOJ8/LzduW5IRz5fk8SCtYkM7NoGN6vF6FgiIiIiItIE+vUbQEREJCdOZLB8+VJuvfW2s7ZfsuRrAMaNG4/Z7Njvr5944hmH9teU2rRpy9y5nxsdw2lpZoM06KqBHQjy8yC7oIyV29ONjiMiIiIiIk3EZDIxbtx4AJYu/fqsbY8ePUxi4lHgfzMiRBqimQ3SIHc3Czdd0YkPvznIVxuOMbx3BL5ebkbHEhERERFxeja7jYS8ZArKCvD38CcuMAazybm/5x03bjyzZ79PYuJRjh49TOfO8Q22qylG9OrVmw4doti/fx9r165ix46tZGVlkp+fj79/AN2792DKlFsveJnFAw9MZ9euHbz22tv07193A0mbzcbChZ/xxRcLSUtLxdvbm969+3Lnnfectc8LzfjBB+8wa9Z7tf88ZcoNda7XZDtxIoMpU24gPDyCzz77st5zi4uL+PTTuaxZs4r09DRMJhPt2nVg5MjR3HLLz/H29ql3z+TJ13Py5Alee+1t2rYN44MP3mH79q0UFRUSERHJ+PE38LOf3ebwGSVNQcUGadTlPcNZvjWV46eK+XrjMW4Z09noSCIiIiIiTm1X1l7mH/2CvLL82tcCPQKY0vkG+rbtZWCys2vXrj29e/dl9+6dLFnyVYPFhsrKSpYvXwrAuHHVsxqeeebPpKcfx8/Pn5CQUEJC2nDqVBbff7+W9evX8dvfPsKkSbdccj673c7TT/+ZlSuXAxAeHkFAQCCbN29g06YN3Hnnrxq990IzhoWF06tXH/bu3Q1A167dcXP73xevvr6+58x78uRJfvvbGRw/norZbCYmphMASUkJJCQc4dtvl/LPf75J27ZhDd5/9Ohh/vjH/6OyspLo6E5YrVZSUo7x5puvcfLkCR5++NFz/9AMpmKDNMpsNjFldByvztvNyu3HGdu/PaGBXkbHEhERERFxSruy9vLevjn1Xs8ry+e9fXO4p+ftTl1wuPbaCezevZNvv13GjBm/wWqt+3Fx8+aN5Obm4O7uwdixVwNwxx2/okePXkRFdazTdvv2rTz11OPMnPkqw4aNJDw8/JKyffHFQlauXI67uwdPP/08V1wxCoCioiKef/4pPvjgnUbvvdCMEybcyIQJN9Yezfnssy8RERF5QXmffvpxjh9PJS6uC88//7faTSTT0lL5058eITk5iWeeeYLXX3+3wfvfemsm1147gQcffBhvb28AVq78lqee+hMLF37G5Mk/q/d+nI3zz70QQ/WMCaZbxyAqq+wsWJdkdBwREREREYex2+2UVZU75K/SyjPMO7L4rM+bf/QLSivPOOR59iY4on7MmCvx9PQkNzeHzZs31ru+ZMlXAFxxxcjab/evvXZCgx96BwwYxPTpM6isrGTFiqWXlMtut/Pvf38EwLRpv6gtNED1LIMnn3wWH5/6SxJqNEfGH9u5czt79+7GbDbz9NMv1DmtokOHKJ566gVMJhO7du1g164dDfbRoUMUjzzyx9pCA8DYsVcxbNgV2O12Nm1a77C8TUUzG+SsTCYTU0fH8fTsrWzan8k1g6LoGO5ndCwRERERkUtit9t5ZcebJOWnNNsz88ryeWTtkw7pq1NANA/3vw+TyeSQ/gC8vX0YOXIMy5Z9w9KlXzNs2BW11woKCtiwYR0A1113fZ37MjLSWbFiGUePHiE/P4+Kigqges8CqF4ScClSU1M4caJ60/qGlmR4eXkxfvyNzJ37caN9NHXGH9u0aQMAl102hI4do+tdj42NY9CgwWzZsonNmzfSt2//em2uv34iFkv9EwF79OjF99+vJT39uMPyNhUVG+ScOob7MaRHGJv2ZzJvVQKP/KyvQ/9QExERERExhv6b9qeuvXYCy5Z9w/r1ayksLMTPr/qLxu++W055eTmhoW0YOPCy2vbz5s3lzTdfo7KystE+8/PzG712PlJSjgEQFBRMYGBgg21q9kRoSHNk/LHU1OoCVqdOsY226dQpji1bNtW+t59q3z6qwdeDgoIBKC0tvbSQzUDFBjkvN1/RiW2HsjiYksu+5Bx6dQoxOpKIiIiIyEUzmUw83P8+ym0VDukvIS+JN3d/eM52M/rcRVxg4x+Mz5e72a1JvgAcMGAQYWHhZGaeZOXK5UycOAmAJUuqT6G45prrar9x37t3N6+99gpms5k777yHkSPHEBkZiaenF2azme3bt/Kb39x31g/556O0tASAoKCgRtvUfAj/qebK+GMlJTV5G//MFBwc8kPb4gave3p6Nvh6zSkUTbGMxtFUbJDzEhroxdgB7Vm2JY35qxLoER2M2axKsIiIiIi4LpPJhIfF3SF9dQvuQqBHQJ1TKH4qyCOAbsFdnPoYTJPJxLhx4/noow9YuvRrJk6cRGpqCvv37wWqZz7UqDkG85ZbpnH33ffW68tRswW8vKr3LcjNzW20TW5uToOvN1fGH6vZZyE3N7vRNjk52T+0bXyvCVfnvL/l4nTGD43G28PK8VPFbNh30ug4IiIiIiJOw2wyM6XzDWdtM7nzDU5daKhRU1DYt28PaWmptR/Yu3XrQXR0TG27EycyAOjTp1+D/dQUKC5Vzb4HeXm55OXlNdgmObnhzeybK+OP1WxGmZSU2GibmmsN7enQUjj/b7o4DV8vNyZcHg3AwnVJlFdUGRtIRERERMSJ9G3bi3t63k6gR0Cd14M8Apz+2Msfa9++A7169QGqT6BYtuwboO6sBgAPj+qp/tnZp+v1kZubW3t6xaWKiupIREQ77HY7CxfOr3f9zJkzfPPNFw3eeykZPTw8ACgrK7ugvEOGXA7Q6J4MSUmJbN26qU7blkjFBrkgYwe0I8Tfg9zCMr7dlmZ0HBERERERp9K3bS+evfyP/KbfvdzZ/VZ+0+9enrn8jy5TaKhRc+LEp59+QmbmSdzd3bnyymvqtOnbt3q2wJw5s2o3RYTqkx/+8IffcubMGYdkMZlM/PzntwPwyScf8f33a2uvFRcX8eyzT1BUVNTgvZeSsebIyl27tl9Q3n79BtC7d19sNhtPPfWnOidHpKcf5+mnH8dut9O3b/9GZ1y0BNqzQS6Im9XCzSNiee+rA3yzKYURfSLx83bMOjcRERERkZbAbDLTJajxkwhcwZgxV/Kvf71c+2H88suvwN/fv06b66+/icWLF5CamsLtt0+lQ4eOWCxmkpOT8PLyYsaMB/nnP192SJ6JEyexY8c2Vq1awWOPPUxERCQBAYEcO5aEzWbn7rvv5Z133qh336VkvPLKa3j33Td5+eWXWLBgPv7+1TNWfvOb/6Nz5/iz5n3yyef47W/v4+jRI9x6683ExMQCdpKTk7DZbHToEMWTTz57yT8XZ6aZDXLBBvcII6qtL6VlVXy54ZjRcURERERExMF8fHwZMWJ07T/XzHT4MW9vb954431uuOEmAgMDOX48lYKCAq6++lo+/PATOnWKc1gek8nEU089z29/+wixsXFkZ5/m5MkMBg0awjvvzKJHj4ZnjlxKxp///Bf86le/Jjo6huPHj7Nr1w527dpBYWHhOfOGh4fzwQdzuPPOe4iOjiE9PY309OPExHTi7rvv5YMP5tC2bdhF/zxcgcnuCmdmtGJVVTZycho+DuXHrFYzQUE+5OYWU1lpa/Jc+4/l8I//7sJiNvH8PYNpG+Td5M8UcTXNPS5F5Nw0LkWcS1OOyYqKcrKzTxASEoGbm2biSut1oWMhONgHi+XS5yVoZoNclB7RwfSMCabKZmfB2oZ3fhUREREREZHWScUGuWiTR8ViArYczCIpo8DoOCIiIiIiIuIkVGyQixYV5sflPcMBmL8qAa3IEREREREREVCxQS7RxCs6YbWYOZyWx57EbKPjiIiIiIiIiBNQsUEuSUiAJ1cNrD5/dv7qRKps2mxLRERERESktVOxQS7Z+KEd8fG0knG6mPV7TxodR0RERERERAymYoNcMm9PN66/PBqAReuSKCuvMjaQiIiIiIiIGErFBnGI0f3bExrgSV5ROcu3pRkdR0RERERERAykYoM4hJvVzM0jOwGwZFMKBcXlBicSERERERERo6jYIA5zWbcwOob7caa8ii/XHzM6joiIiIgIoOPZpbUzZgyo2CAOYzaZmDo6DoDVu9LJzCkxOJGIiIiItFYmU/VHnaoqnZYmrZvthxMDTSZTsz7X2qxPc4BNmzYxa9Ysdu/eTUlJCZGRkYwbN47p06fj7e19UX3a7Xa+/vprFi5cyMGDBykoKCAwMJDY2FhGjBjB3XffXe+e+Pj4s/YZGhrK+vXrLyqPK+vWMYjesSHsSczm8zWJzLipl9GRRERERKQVslgsmM1WyspK8fT0MjqOiGHKy8swmcxYLM378d+lig1z5szh+eefx263Ex4eTkREBAkJCbz11lssX76cuXPnEhgYeEF9FhcX88ADD7BhwwYAOnToQGRkJNnZ2WzdupVDhw41WGyo0bNnT9zd3eu9fqE5WpLJo2LZm5TNtsOnSEzPJ7ZdgNGRRERERKSVMZlMeHp6U1pahLe3D25uHkZHEml2NpuNM2eK8fDw1MyGxuzbt48XXngBgGeeeYapU6diMpnIzMzkvvvuY//+/TzxxBPMnDnzvPu02+08+OCDbNiwgSuuuIInn3ySqKio2usFBQVs3br1rH3861//on379hf3plqo9m18GdYrgu/3nGDeqgQem9a/2X+xRURERER8fQOoqCgjJycLT08fPDy8sFjMgP7bVFo2u91OVVUFxcWF2Gw2fH0Dmz2DyxQb3nzzTWw2GxMnTuSWW26pfT0sLIxXXnmFa6+9luXLl3Po0CG6du16Xn0uWLCA9evX06dPH95++22s1ro/Dn9/f8aOHevQ99FaTBwew5YDmRw9ns+uo6fp16WN0ZFEREREpJUxm80EBbWlqCifM2dKKC0tNDqSSLNyd/fE378tVqtbsz/bJYoNxcXFrFu3DoCpU6fWux4dHc2QIUPYsGEDS5cuPe9iw+zZswG477776hUa5NIE+3ty1aAOfL0xhfmrE+kdF4LFrP1IRURERKR5mc1m/P2D8PMLpKqqCrtdG0ZK62A2W7BYLIY93yU+YR88eJDy8nLc3d3p3bt3g20GDBjAhg0b2L1793n1mZqaypEjRzCbzQwePJjdu3fz+eefk5qaire3N3379mXy5MkEBweftZ8333yTrKwsqqqqCAsLY8iQIVx33XUN7uPQ2lw7uCNrdmVwMqeEdbtPMKpfO6MjiYiIiEgrZTKZ9AWjSDNyidGWnJwMQGRkJG5uDU//qNlroabtuezbtw+o3sjxk08+4R//+Ad2+//OH125ciXvvfceM2fOZMiQIY328/nnn9f554ULF/Laa68xc+ZMevTocV5ZWipvTys3DItm7oqjLPo+mSE9wvB0d4lfOREREREREbkELvHJLz8/H4CAgMZPNai5VtP2XLKysoDqTSBffvllRo0axe9//3uioqJITk7mhRdeYNOmTTz44IN8+eWXhIeH17l/7Nix3HjjjXTt2pXw8HCKi4vZuHEjr776Kmlpadx1110sWrSIiIiIi3nLdVit515+UL3Rzf/+11lcOagDK7YfJyu3lG+3HeemEZ2MjiTSbJx1XIq0ZhqXIs5FY1LE+Thqb3+XKDaUlZUBNDqrAahdtlDT9lxKSkoAqKysJCoqitdff722//j4eN5++22uuuoqTp06xUcffcSjjz5a5/4333yzzj97eHgwfvx4hg4dyqRJk8jIyOD111/n+eefP7832Qiz2URQkM95t/f3d74zhO+8vgd//XgbSzalcNPozgT5exodSaRZOeO4FGntNC5FnIvGpEjL4xLFBg+P6jNxKyoqGm1TXl5ep+359gkwbdq0eoUMLy8vfvaznzFz5kzWrVtXr9jQmODgYKZPn85TTz3FihUreO655y7p2EebzU5BQck521ksZvz9vSgoKKWqyrk2veneIYBOkf4kZRQw+8t93HFdN6MjiTQLZx6XIq2VxqWIc9GYFHE+AQFemB2wub9LFBvOZ4nE+Sy1+DF/f//av4+NjW2wTc3rx48fP68+a/Tr1w+AvLw88vLyCAoKuqD7f6qy8vz/4K2qsl1Q++YyZVQsf527k9U7Mxg7oD0RIec/W0PE1TnruBRpzTQuRZyLxqSI8/jRVoaXxCUWR0VHRwOQkZHR6OyG1NTUOm3PpVOn/+0d0NjyjJrZDzbbhf3B9+P+qqqqLujelio+Koi+caHY7HY+W51odBwRERERERFpQi5RbOjWrRtubm6Ul5ezZ8+eBtts374dgL59+55Xn927d8fTs3rvgLS0tAbb1BQwfro55LkcPXoUqC5WBAYGXtC9LdnkUbGYTLDz6GmOpOUZHUdERERERESaiEsUG3x9fRk+fDgA8+bNq3f92LFjbNq0CYBx48adV59eXl6MHj0agEWLFtW7brfbWbhwIcBZj778qcrKSmbNmlV7n87y/Z/IUB9G9IkEYP6qhDpHjYqIiIiIiEjL4RLFBoAZM2ZgMplYvHgxn376ae0H1aysLB5++GFsNhtXXnklXbt2rXPfmDFjGDNmDEuXLq3X5wMPPIDVamXbtm288cYbtUseKisr+fvf/86hQ4fw8PDgjjvuqHPfyy+/zMKFCykqKqrz+okTJ3jooYfYtWsXVquV+++/34E/gZbhxuExuLuZScwoYPvhU0bHERERERERkSZgsrvQ18uzZ8/mpZdewm63ExERQVBQEAkJCZSXlxMTE8PcuXMJDg6uc098fDwAL774IjfffHO9PhcuXMjjjz9OVVUVwcHBtG/fntTUVPLy8nBzc+Oll15iwoQJde6ZMWMGK1euxGKx0KFDBwICAigsLCQ5ORm73Y6HhwfPPfccN9xwwyW/56oqGzk5xedsZ7WaCQryITe32Ok311m0Lokv1h+jbZAXz/1qMFadqywtlCuNS5HWQuNSxLloTIo4n+BgHywO+IzmUnP877jjDuLj4/nwww/Zs2cP2dnZREZGMm7cOKZPn46Pz4WfcHDTTTcRFxfH+++/z7Zt2zh48CCBgYFMmDCBe+65p95MCYBbb72V0NBQ9u3bR1ZWFunp6bi5udG5c2eGDh3KbbfdRlRUlCPecot0zWVRrN6ZTlZuKWt2VZ9OISIiIiIiIi2HS81saI1a4swGgFU7jjNn+RH8vN146d6heHm4VN1L5Ly42rgUaQ00LkWci8akiPNx1MwGzV8XQ1zRJ5KwYG8KSypYsjnV6DgiIiIiIiLiQCo2iCGsFjOTR8YCsHxLKrmFZQYnEhEREREREUdRsUEM079LKHHtAiivtLH4+2Sj44iIiIiIiIiDqNgghjGZTEwZXT27Yd2eDNJPn3tvChEREREREXF+KjaIoTq3D6R/lzbY7fD56kSj44iIiIiIiIgDqNgghps0shNmk4ldCac5nJprdBwRERERERG5RCo2iOEiQnwY2TcSgHmrEtBprCIiIiIiIq5NxQZxCjcMj8HDzULyiUK2HsoyOo6IiIiIiIhcAhUbxCkE+Lhz7eAoAD5fk0hllc3gRCIiIiIiInKxVGwQp3H1ZR0I8HHnVN4ZVu1MNzqOiIiIiIiIXCQVG8RpeLpbufGKGAC+XH+MkjOVBicSERERERGRi6FigziVK3pHEBHiTVFpBUs2pxgdR0RERERERC6Cig3iVCxmM5NHxQKwfGsaOQVnDE4kIiIiIiIiF0rFBnE6feNC6dI+gIpKG4vWJRsdR0RERERERC6Qig3idEwmE1PGxAGwfu8JjmcVGZxIRERERERELoSKDeKUYiMDGNi1LXZg/upEo+OIiIiIiIjIBVCxQZzWpJGdsJhN7E3K5sCxHKPjiIiIiIiIyHlSsUGcVliQN6P6tQNg/qpEbHa7wYlERERERETkfKjYIE7t+mHReLpbSMksZMuBTKPjiIiIiIiIyHlQsUGcmr+3O9cN6QjA52uSqKi0GZxIREREREREzkXFBnF6Vw3qQKCvO9kFZ/hux3Gj44iIiIiIiMg5qNggTs/DzcJNV3QC4KsNxyg+U2FwIhERERERETkbFRvEJQzrFUG7UB+Kz1Ty9cYUo+OIiIiIiIjIWajYIC7BbDYxZXQsACu2Hed0fqnBiURERERERKQxKjaIy+jVKYSuUYFUVtlYuDbZ6DgiIiIiIiLSCBUbxGWYTCamjI4DYNP+k6ScLDQ4kYiIiIiIiDRExQZxKTER/gzuHoYd+Gx1gtFxREREREREpAEqNojLuXlEJyxmE/uP5bIvOdvoOCIiIiIiIvITKjaIy2kT6MXYAe0BmL8qEZvNbnAiERERERER+TEVG8QlTbg8Gi8PK2lZRWzcf9LoOCIiIiIiIvIjKjaIS/L1cmP80I4ALFyXREVllcGJREREREREpIaKDeKyrhzQniA/D3IKylix/bjRcUREREREROQHKjaIy3J3s3DziE4AfLUhhaLSCoMTiYiIiIiICKjYIC5uaI9w2rfxpbSskq82HDM6joiIiIiIiKBig7g4s9nE1NGxAHy34zin8koNTiQiIiIiIiIqNojL6xETTPfoICqr7Cxcm2R0HBERERERkVZPxQZxeSaTiSmj4gDYdCCTYycLDE4kIiIiIiLSuqnYIC1Cx3A/hvYIA2DedwnY7XaDE4mIiIiIiLReKjZIi3HTiE5YLSYOpeaxNynH6DgiIiIiIiKtlooN0mKEBnhx5YAOAMxfnYDNptkNIiIiIiIiRlCxQVqU8Zd3xMfTSvqpYtbvO2F0HBERERERkVZJxQZpUXw83Rg/NBqAReuSKauoMjaQiIiIiIhIK6Rig7Q4Ywe0I8Tfk9zCMlZsSzM6joiIiIiISKujYoO0OG5WCzeP7ATA1xtTKCgpNziRiIiIiIhI66Jig7RIg7uHERXmy5nyKr5af8zoOCIiIiIiIq2Kig3SIplNJqaOjgNg1c50MnNLDE4kIiIiIiLSeqjYIC1W9+hgenYKpspmZ8GaJKPjiIiIiIiItBoqNkiLNmVUHCZg66EsEjPyjY4jIiIiIiLSKqjYIC1ah7a+XN4rHID5qxKx2+0GJxIREREREWn5VGyQFu+mKzrhZjVzJC2P3QnZRscRERERERFp8VRsaAFsdhuHcxL4PmUrh3MSsNltRkdyKsH+nlw1sAMA81cnUGXTz0dERERERKQpWY0OIJdmV9Ze5h/9gryy/+1HEOgRwJTON9C3bS8DkzmX64Z0ZO3uDE5kl/D9nhOM7NvO6EgiIiIiIiItlmY2uLBdWXt5b9+cOoUGgLyyfN7bN4ddWXsNSuZ8vD2tXH95NACL1iVTVl5lbCAREREREZEWTMUGF2Wz25h/9Iuztvns6BdaUvEjo/u3o02gJ/nF5Szbmmp0HBERERERkRZLxQYXlZCXXG9Gw0/lluWTkJfcTImcn9ViZtLIWACWbE4lv7jc4EQiIiIiIiItk4oNLqqgrOC82v3n0OcsSV5JSkGaZjkAA7u2JSbCj7LyKr5Yr0KMiIiIiIhIU9AGkS7K38P/vNpllZ7mq+RlfJW8DF83H7oFd6F7SDzdgrvg5+7bxCmdj9lkYsqoOP72n52s2ZnBlQPaExHiY3QsERERERGRFkXFBhcVFxhDoEfAWZdS+Lv7MS76Sg7nHOFwbgJFFcVszdzJ1sydAHTwa0f34Hi6h8QT4x+FxWxprviG6toxiD6xIexOzGbBmiTuv1mndoiIiIiIiDiSig0uymwyM6XzDby3b06jbW7pMpG+bXsxsv1QqmxVJOWncCDnMAezD5NWlEFaYTppheksS/kOT4snXYPj6B4cT7eQLgR7BjXju2l+k0fFsicpm+1HTpGQnk9cuwCjI4mIiIiIiLQYJrvdbjc6hDSuqspGTk5xo9d3Ze1l/tEv6sxwCPIIYHLnG+jbtvFv7PPLCjmUc6S6+JBzhOKKkjrXw33C6P7Dkou4gBjcLG6X/maczKxvDrJuzwni2gfwx2n9MZlMRkeSFsZqNRMU5ENubjGVldozRcQZaFyKOBeNSRHnExzsg8Vy6ds7qtjg5M5VbIDqYzCTC49RaS3HWulOjF80ZtP5/3LY7DZSC49zILu68JCcn4qd//1auJnd6BzUqXbJRVuv0BbxwTy3sIw/vrOR8kobD9zci/5d2hgdSVoY/QeUiPPRuBRxLhqTIs5HxYZW4nyKDeDYP6hLKko4lJvAgezDHMg+TH553ZMvQjyD6R4ST/fgLnQJisXT6nlJzzPSgrWJfLUhhbBgb569+zKsDhhUIjX0H1AizkfjUsS5aEyKOB9HFRu0Z4PU4+3mTf+2venftjd2u52M4pPVhYecIyTmJZN9Jod16RtZl74Ri8lCbEA03UK60D04nna+ES416+HawR1ZvTODzJwS1u05weh+7YyOJCIiIiIi4vI0s8HJGTGz4WzOVJZxNC+RA9lHOJB9iNNncupcD3D3o1twPN1DutA1uAs+bt5NlsVRVm4/ziffHsHf240X7x2Kl4dqcOIY+rZGxPloXIo4F41JEeejmQ1iCE+rB71Cu9MrtDsAWSWna0+4OJKbSH55IZtObmPTyW2YMBHt34FuIfF0D46no3/7C9pLormM7BvJt9vSyMotZdmWVCZe0cnoSCIiIiIiIi5NMxucnLPNbDibiqoKEvOP/bDk4jAnijPrXPexetM1uPMPxYcuBHj4G5KzIdsOZfHmon14uFl48d4hBPp6GB1JWgBnGJciUpfGpYhz0ZgUcT6a2SBOx83iRtfgznQN7szNTCD3TN4Psx6OcCj3KMWVJWzP2s32rN0AtPONqD3holNAR6xm434dB8S3oVOkP0kZBXzxfTK/GNfVsCwiIiIiIiKuTjMbnJwrzWw4mypbFccK0jiQU33CRWrh8TrXPSzuxAd1pltwF7qHxBPqFdzsGY+k5fHSJzswm0w8c/dlRIb6NHsGaVmcfVyKtEYalyLORWNSxPno6MtWoqUUG36qsLyIgzlHOJB9hIM5hymqqPse23qH1s566BzYCXeLe7Pkmvn5HnYePU2/zqE8OKl3szxTWi5XG5cirYHGpYhz0ZgUcT5aRiEuzc/dl8vC+3NZeH9sdhvHizJ+OOHiMMkFKWSVnCar5DSrj6/HarbSObAT3X+Y9RDm3bbJjtecPCqW3QnZ7Dx6miNpeXTpENgkzxEREREREWnJNLPBybXUmQ1nU1pZyuGchB+WXBwhtyyvzvUgj0C6h3She3A88cFxeFm9HPr8j5ceYvWuDDpF+vP47QOarLAhLV9LGpciLYXGpYhz0ZgUcT6a2SAtlpfVi75te9G3bS/sdjsnS7I4kH2YgzlHOJqXRG5ZHusztrA+Ywtmk5kY/450D4mne0gX2vtGXvLxmjcOj2Hj/kySMgrYfvgUA7u2ddA7ExERERERaR00s8HJtcaZDWdTXlXO0byk2uM1s0pO17nu5+ZLtx9mPXQN7oyfu+9FPWfRuiS+WH+MtkFePPerwVgdUNmT1qe1jEsRV6JxKeJcNCZFnI9mNkir5G5xp0dIV3qEVB9Nebo0u3qvh5zDHM5NoLCiiC0nd7Dl5A5MmOjg16561kNwPNH+HbCYLef1nGsui2L1rgyycktZsyuDsQPaN+XbEhERERERaVE0s8HJaWbD+au0VZKUn1I76yG96ESd615WT7oGdaZ7SDzdgrsQ5Bl41v5W7UxnzrLD+Hq58ddfD8XLQ7U5uTAalyLOR+NSxLloTIo4Hx192Uqo2HDx8sryOZhzlIM/7PdQUlla53qkTzjdfjjhIjYwBjdz3WJCZZWNJz/YwsmcEiZc3pGbR8Q2Z3xpATQuRZyPxqWIc9GYFHE+Kja0Eio2OIbNbiOlIK12o8ljBWnY+d+vvrvZjS5BsXT7YclFW+9QAHYcOcXrC/bibjXz4r1DCfLzMOotiAvSuBRxPhqXIs5FY1LE+WjPBpELYDaZiQnoSExAR8Z3upqiimIO5xyt3e+hoLyQfdmH2Jd9CIBQrxC6B1cvt+jUwZuktBIWrUvizuu6GfxOREREREREnJ/LFRs2bdrErFmz2L17NyUlJURGRjJu3DimT5+Ot7f3RfVpt9v5+uuvWbhwIQcPHqSgoIDAwEBiY2MZMWIEd999d4P3FRcX8+6777Js2TIyMjLw9vamT58+3HXXXQwePPhS3qY0MV83HwaE9WVAWF/sdjvpRSc4kHOYA9mHScw/xunSbNamb2Bt+gYskRbcvQPYeKoNfdLc6de+EyaTyei3ICIiIiIi4rRcahnFnDlzeP7557Hb7YSHhxMcHExCQgLl5eXExsYyd+5cAgMDL6jP4uJiHnjgATZs2ABAhw4dCAwMJDs7m8zMTPz8/Ni8eXO9+3Jycvj5z39OcnIy7u7uxMXFkZOTw8mTJzGZTDzxxBNMmzbtkt+zllE0vzOVZziSm8iBnCMcyD5E9pncOtcD3P2rT7gIiadrUBzebhdX5JKWT+NSxPloXIo4F41JEefT6vZs2LdvH1OmTMFut/P0008zdepUTCYTmZmZ3Hfffezfv5+rr76amTNnnnefdrudu+++m/Xr13PFFVfw5JNPEhUVVXu9oKCArVu3Mnbs2Hr33nfffXz33Xf06NGDt956i7CwMOx2O/PmzePJJ5/EYrHw+eef063bpU27V7HBWHa7nazS02xO28uSA9sx+WVjMv/v52vCRExAVPWSi5AuRPm1x2y69IEpLYPGpYjz0bgUcS4akyLOp9UVG2bMmMHKlSuZOHEif/3rX+tcO3bsGNdeey02m43FixfTtWvX8+rz888/509/+hN9+vRh7ty5WK3nt6rkwIED3HTTTZjNZpYuXUrHjh3rXP/DH/7A4sWLL7j40RAVG5zHv5cf5rudqUR0PEO//nYO5hzhZElWnTY+bt7VJ1wEx9M1uAsBHn4GpRVnoHEp4nw0LkWci8akiPNpVRtEFhcXs27dOgCmTp1a73p0dDRDhgxhw4YNLF269LyLDbNnzwaqZymcb6EBYNmyZQAMGTKkXqEB4JZbbmHx4sWsWbOGkpKSi95LQpzLDcNiWL/vJCeOWbihdw8mD7mBnDO5HPxhk8lDOQkUV5SwLXMX2zJ3AdDBN7L2hItOAR2xmC3GvgkREREREZFm4BLFhoMHD1JeXo67uzu9e/dusM2AAQPYsGEDu3fvPq8+U1NTOXLkCGazmcGDB7N7924+//xzUlNT8fb2pm/fvkyePJng4OB69+7atQuAgQMHNth37969cXd3p6ysjIMHDzJgwIDze6Pi1Px93LlucBQL1yXz+ZpE+ndpQ7BnEMPaDWZYu8FU2apILkjlQPZhDuQcJq0wnbSiDNKKMliesgpPiwfxwZ1rZz6EeAUZ/ZZERERERESahEsUG5KTkwGIjIzEzc2twTY1ey3UtD2Xffv2ARAYGMgnn3zCP/7xD368omTlypW89957zJw5kyFDhtS599ixY3We+VNubm5ERESQkpJCcnKyig0tyNWDovhuZzqn88+wamc6Vw/qUHvNYrYQFxhDXGAMN8SOo6C8kIPZRziYU/1XUUUxu0/tY/ep6t+9MO+2dA+pLjzEBXbC3dLw77aIiIiIiIircYliQ35+PgABAQGNtqm5VtP2XLKyqtfaFxQU8PLLLzNq1Ch+//vfExUVRXJyMi+88AKbNm3iwQcf5MsvvyQ8PPyi8hQUFJxXnrOxWs+9XqZmTY0j1tZI46xWM5NGxvLh1wf5csMxRvaLxMez4SJBsDWAYd6DGNZhEDa7jdSCdPZnH2b/6UMk5aWQWZJFZkkWq9K+x81spUtQLD1Cu9IjNJ4w7zY6XrMF0LgUcT4alyLORWNSxPk46mOISxQbysrKABqd1QDg7u5ep+25lJSUAFBZWUlUVBSvv/56bf/x8fG8/fbbXHXVVZw6dYqPPvqIRx999KLynDlz5rzyNMZsNhEU5HPe7f39vS7peXJuN4yM49ttaaRlFrFyRwa/HN/9vO4LCe5Kv+iuwI0UlRezL/Mwu07sZ9fJA+SU5lUXIrIPw2Fo4xNCn/Du9A3vTs+weLzd9O/VlWlcijgfjUsR56IxKdLyuESxwcPDA4CKiopG25SXl9dpe759AkybNq1e4cDLy4uf/exnzJw5k3Xr1tUpNnh4eFBaWnpeeTw9Pc8rT2NsNjsFBSXnbGexmPH396KgoJSqKu3k29Qmj4zl1Xm7Wbw2kWE9wggJuPB/z/G+8cR3jmdq3E1kFJ1kf/ZhDpw+zNHcJE4VZ7MicR0rEtdhNpmJC4yunvUQEk97v0jNenARGpcizkfjUsS5aEyKOJ+AAC/M5lZyGsX5LJE4n6UNP+bv71/797GxsQ22qXn9+PHj9e4tLS09rzw/fs7FupBjgKqqbDo2qBn0jAmmS4dAjqTl8dnqBO4+z9kNjQnzCiOsfRhj2o+grKqco7mJHMg5zIHsw5wqzeZIbhJHcpNYePQb/N39qjeZDImna3BnfN3Of+aLGEPjUsT5aFyKOBeNSRHn8aOtDC+JSxQboqOjAcjIyKCioqLB5Qupqal12p5Lp06dav++seUQNbMfbLa6f/BFR0eTmZlJSkpKg/dVVFSQkZFxQXnEtZhMJqaOjuO5j7exYe9Jrh4URYe2vg7p28PiTs/QbvQM7QZAVslpDuYc4UD2YY7kJlBQXsjmk9vZfHI7JkxE+bene3A83UPiifbvgNmkNY8iIiIiImIslyg2dOvWDTc3N8rLy9mzZ0+Dpzts374dgL59+55Xn927d8fT05MzZ86QlpZW78QJ+F8B48ebQ9Y8Y/PmzbXP/Kk9e/ZQUVGBh4cH3bp1O6884no6RfozqGtbth7KYv7qBB6e2rdJntPWO5S23qGMbH85FbZKkvKO1c56yCg+SUpBGikFaSw5tgJvqxddgzvTPTiebiFdCPQ4v5k+IiIiIiIijuQSX4H6+voyfPhwAObNm1fv+rFjx9i0aRMA48aNO68+vby8GD16NACLFi2qd91ut7Nw4UKAeoWIa665BoDNmzc3OLvh008/BWDEiBH4+GiKe0s2aWQnLGYT+5Jy2H8sp8mf52a2Eh8cx01x43l88MM8P+xxpnWdQr+2vfGyelFSWcqOrD38+9B8Hl//PC9seZWFCV9zOCeBCltlk+cTEREREREBFyk2AMyYMQOTycTixYv59NNPsf+wkCQrK4uHH34Ym83GlVdeSdeuXevcN2bMGMaMGcPSpUvr9fnAAw9gtVrZtm0bb7zxBlVVVUD1CRV///vfOXToEB4eHtxxxx117uvRowejR4+mqqqK3/3ud7XHaNrtdj799FMWL16M2Wzmvvvua4KfhDiTtkHejO7XDoD5qxKwOWqB03kK9Ajg8shB/Krnbfx1+JP834AZXBs9lo7+HTBhIr3oBCtS1/Darnf5w7qneHvPLNYe38Dp0uxmzSkiIiIiIq2LyW5v5k9Hl2D27Nm89NJL2O12IiIiCAoKIiEhgfLycmJiYpg7dy7BwcF17omPjwfgxRdf5Oabb67X58KFC3n88cepqqoiODiY9u3bk5qaSl5eHm5ubrz00ktMmDCh3n05OTnceuutHDt2DHd3d+Li4sjNzeXEiROYTCYef/xxbr/99kt+z1VVNnJyis/Zzmo1ExTkQ25usTbXaWYFJeX88Z2NlJZVcc/13RnaI/zcNzWDovJiDuUc4cAP+z0UVhTVud7WK5RuIfF0D+5C56BYPCzuBiVtuTQuRZyPxqWIc9GYFHE+wcE+WCyXPi/BpYoNABs3buTDDz9kz549lJSUEBkZybhx45g+fXqDSxbOVWwA2Lt3L++//z7btm0jPz+fwMBABg8ezD333FNvpsSPFRUV8d5777F06VIyMjLw9vamd+/e3H333Q3uAXExVGxwDV9vPMbna5II8ffkhemDcbNajI5Uh81uI73oBAeyD3Mg5zBJ+SnY7P/7PbGaLMQFdqJbSBe6B8cT4ROm4zUdQONSxPloXIo4F41JEefTaosNrY2KDa6hrKKKP727idzCMqaOjmPc4CijI51VaeUZjuQmcCD7MPuzD5NbllfneqBHQO0JF/FBcXi7eRkT1MVpXIo4H41LEeeiMSnifFRsaCVUbHAd6/ZkMOubQ3h7WHnp10Px9Wr4SFVnY7fbySw5VXvCxdG8JCp/tJmk2WQmxj+KbsHxdA/pQge/djpe8zxpXIo4H41LEeeiMSnifFRsaCVUbHAdNpudv8zaQvqpYsZdFsXUMXFGR7oo5VXlHM1L5uAPSy4yS07Vue7r5kO34C50D4mnW3AX/Nx9DUrq/DQuRZyPxqWIc9GYFHE+Kja0Eio2uJY9idn8c/5urBYTL9wzhNBA119+kF2aw4GcIxzMPsyh3KOUVZXXuR7l147uwfF0C4knxj8Ki9m59qswksaliPPRuBRxLhqTIs5HxYZWQsUG12K323n5v7s4mJLL0B5h3HN9D6MjOVSlrZLk/JTaEy6OF2XUue5l9SQ+KO6H4kMXgj2DDErqHDQuRZyPxqWIc9GYFHE+Kja0Eio2uJ5jJwt4ZvY2AP5yxyA6hvsZnKjp5JcVcDDnSPVf2Ucoriypcz3cJ4zuPyy5iAuIwc3iGvtYOIrGpYjz0bgUcS4akyLOR8WGVkLFBtf07hf72XQgk24dg3jkZ31bxTGSNruN1MLj1cdrZh/hWEEqdv73x4ub2Y0uQbG1sx7aeoW2+J+LxqWI89G4FHEuGpMizsdRxQarA7KIyE/cNKIT2w5ncTAll/3JOfTsFGJ0pCZnNpmJ9o8i2j+K62KuoriihEM5RzmQc5iD2YfJLy9kf/Yh9mcfgqMQ4hlM95B4ugd3oUtQLJ5WT6PfgoiIiIiIOIiKDSJNoE2gF2P6t2f51jTmrUqke3QwZnPL/hb/p3zcvBkQ1ocBYX2w2+1kFJ+snvWQc4TEvGSyz+SwLn0j69I3YjFZiA2Iri4+hMQT6RPe4mc9iIiIiIi0ZFpG4eS0jMJ1FZVW8NjbGykpq+Tu8d0Y1ivC6EhO40xlGUfzEjmQfZj92YfJPpNT53qAux/dfpj10DW4Cz5u3gYlvTQalyLOR+NSxLloTIo4Hy2jEHFyvl5ujL+8I/NXJbJgbRKDurbF3U3HQgJ4Wj3oFdqdXqHdsdvtnCo9zYHsIxzIOcyR3ETyywvZdGIbm05sw4SJaP8OPxQf4uno3x6z6dL/8BMRERERkabT5DMbqqqq+M9//sP69esxm82MGjWKKVOmNOUjWxTNbHBtFZVV/OndTWQXlDF5VCzXDelodCSnV1FVQWL+sR+WXBzmRHFmnes+Vm+6Bneme0g83YK7EODhb1DSc9O4FHE+GpcizkVjUsT5ONVpFJ999hlPPPEE11xzDf/85z/rXPvNb37D8uXLAbDb7ZhMJsaNG8err756qY9tFVRscH0b9p3g/a8O4uVh4aV7h+Ln7W50JJeSeyaPAznVJ1wcyjnKmaozda63842ge3D1Xg+dAjpiNTvPhC2NSxHno3Ep4lw0JkWcj1Mto1i/fj0AEyZMqPP65s2bWbZsGQD9+/fH09OTjRs3snTpUsaPH8+VV17piMeLOLUhPcJZtiWNtKwivtqQwq1XdjY6kksJ8gxkWORghkUOpspWRXJBKgd/2GgytfA46UUnSC86wbepq/GwuBMf1JnuIV3oFhxPqFew0fFFRERERFolhxQbDh48CFQXFH5s0aJFAEydOpVnnnkGgDfffJPXXnuNhQsXqtggrYLZZGLq6Dj+8ekuvttxnLED29M20MvoWC7JYrYQFxhDXGAM18eOo7C8iIM5RziQfYSDOYcpqihmz+n97Dm9H4Aw7zZ0D46nW0g8nQM74W5xM/gdiIiIiIi0Dg4pNuTm5uLu7k5wcN1vETdu3IjJZOL222+vfW3atGm89tpr7Nu3zxGPFnEJPWKC6RETzP7kHBasSeTXN/Y0OlKL4Ofuy2Xh/bksvD82u43jhRk/LLk4THJBKpklp8gsOcWq499jNVvpHNiJ7sFd6B4ST5h3Wx2vKSIiIiLSRBxSbCguLsbbu+7RdFlZWZw8eZLQ0FA6d/7ftPGAgAB8fX3Jycn5aTciLdqUUbEcSM5hy8EsrrmsgJgI593Y0BWZTWai/NsT5d+ecdFjKako5UhuAgdyqo/XzCvL52DOEQ7mHOHzhK8I8gike0j1Xg/xQXF4WT2NfgsiIiIiIi2GQ4oNvr6+5OfnU1paipdX9fTwrVu3AtCvX78G7/Hw8HDEo0VcRlSYH0N7hrNh30nmfZfAH37eT9+sNyFvNy/6tu1F37a9sNvtnCzJqj7hIvswCXlJ5JblsT5jM+szNmM2mekU0JFuwfF0D+lCe99IHa8pIiIiInIJHFJs6Ny5M9u2bWPJkiXcfPPNQPV+DSaTiUGDBtVpW1hYSFFREdHR0Y54tIhLuemKTmw5mMXhtDz2JGbTJy7U6EitgslkIsInjAifMMZGjaCsqpyjuYkcyDnCwezDZJWeJiEvmYS8ZL5MWoqfmy/dQrrQPTiersGd8XP3NfotiIiIiIi4FIcUGyZMmMDWrVt55pln2L17N6dPn2bdunW4u7tz7bXX1mm7c+dOABUbpFUKCfDkyoHtWbo5lc9WJ9KrUwhms2Y3NDcPizs9Q7vRM7QbAKdLszmQfYQDOYc5nJtAYUURW07uYMvJHZgwEeXXnu4h1Xs9dPTrgMVsOeczbHYbh3OSqCwox1rpToxftGZLiIiIiEir4ZBiw+TJk1m2bBkbNmxg3rx52O12TCYTv/3tb2nTpk2dtkuXLm1wxoNIazF+aEfW7c4g/XQx6/ee4Io+kUZHavVCvUIY0X4oI9oPpdJWSVL+sdriQ3rRCVIK00gpTGPJsZV4Wb3oGhRH95B4ugV3IcgzsF5/u7L2Mv/oF+SV5de+FugRwJTON9C3ba9mfGciIiIiIsYw2e12uyM6stlsfPXVV+zcuRN/f39GjBjBgAED6rQpLy/n/vvv58yZMzz55JN1No6UhlVV2cjJKT5nO6vVTFCQD7m5xVRW2pohmVyKZVtS+fS7BAJ93Xnx3qF4uJ37m3IxRvXGkkc5mH2YgzlHKKksrXM90ie8dslFbGAM+08f5L19cxrt756et6vgIGIg/f+liHPRmBRxPsHBPlgslz4j12HFBmkaKja0TBWVNh5/bxOn889w84hOTLg82uhIch5sdhspBWnVG03mHCGlIA07//sj1M1kxW6CSltlo30EeQTwzOV/1JIKEYPo/y9FnIvGpIjzcVSxwSHLKETkwrhZzdw8ohPvfnmAbzalMKJvJP7e7kbHknMwm8zEBHQkJqAj4ztdTVFFMYdyjv5QfDhMYXkRnKN8m1uWT0JeMl2CYpsntIiIiIiIAZql2LBq1SrWr1+P2Wxm5MiRDBs2rDkeK+LULusexrItaaRkFvLl+mNMu6qL0ZHkAvm6+TAwrC8Dw/pis9v4NmU1XyQtPed9BWUFzZBORERERMQ4DpnHu3z5csaOHcuTTz5Z79qLL77IjBkz+OSTT5gzZw6/+tWv+Otf/+qIx4q4NLPJxNTR1d9ur96ZTmZuicGJ5FLUzHo4H/4e/k2cRkRERETEWA4pNnz33XdkZGQwcODAOq/v37+fjz76CLvdTkREBFFRUdjtdmbPns3mzZsd8WgRl9YtOphenUKostn5fE2S0XHkEsUFxhDoEXDWNkEeAcQFxjRTIhERERERYzik2LB3714Ahg4dWuf1zz//HICrrrqKFStWsGzZMqZNm4bdbmfevHmOeLSIy5syKhYTsO1QFokZ+edsL87LbDIzpfMNZ20zufMN2hxSRERERFo8h/wXb05ODhaLhTZt2tR5ff369ZhMJu655x7M5upH3XvvvQDs2rXLEY8WcXnt2/oyrFcEAPO/S0AHxLi2vm17cU/P2+vNcAjyCNCxlyIiIiLSajhkg8jCwkJ8fHzqvJabm0tKSgoBAQH07t279vW2bdvi5eXFqVOnHPFokRZh4hUxbD6YyZHj+exKOE2/zm3OfZM4rb5te9G7TQ+SC49RaS3HWulOjF+0ZjSIiIiISKvhkP/y9fb2prCwkIqKitrXtm/fDkDfvn3rtXdzc8NisTji0SItQrC/J1cP6gDAZ6sTqbLpnGlXZzaZiQ+OY3jHQcQHx6nQICIiIiKtikP+67dTp07Y7XbWrFlT+9qSJUswmUwMGDCgTtvS0lIKCwvrLbkQae2uHdwRXy83TmSXsG7PCaPjiIiIiIiIXDSHLKO46qqr2LVrF3/+859JSkri1KlTfPPNN5jNZq699to6bffu3Yvdbqd9+/aOeLRIi+HtaeX6YdH8Z8VRFq9LZkj3MDzdHTJERUREREREmpVDZjbcdtttxMfHk5eXx6uvvsqcOXOw2+3cdtttdOjQoU7b5cuXYzKZ6h2TKSIwul872gR6kl9czvItaUbHERERERERuSgO+drUw8ODuXPn8tFHH7Fr1y78/PwYPXo0EyZMqNOuvLycrVu3EhERwfDhwx3xaJEWxWoxM2lkLG8v3s+SzamM7NeOAB93o2OJiIiIiIhcEJNd5+w5taoqGzk5xedsZ7WaCQryITe3mMpKbS7oyux2O899vI3kE4WM7teO26+JNzqSXCSNSxHno3Ep4lw0JkWcT3CwDxbLpS+C0PboIk7GZDIxdXQcAGt2ZXAi+9zFJhEREREREWfSJLvPFRUVceDAAbKzswEICQmhe/fu+Pr6NsXjRFqc+Kgg+saFsivhNJ+vSeKBm3sZHUlEREREROS8ObTYcPjwYV599VXWrVuHzVZ3GpTZbGbkyJH85je/IT5e08JFzmXSqFh2J55mx5FTHD2eR+f2gUZHEhEREREROS8OW0axfPlypk6dypo1a6iqqsJut9f5q6qqilWrVjF16lS+/fZbRz1WpMVqF+rDFb0jAZi3KgFtryIiIiIiIq7CIcWGtLQ0HnnkEcrKyoiMjOQvf/kLy5cvZ8+ePezZs4fly5fzl7/8hXbt2lFWVsYjjzxCWpqO9RM5lxuHx+DuZiYxvYAdR04ZHUdEREREROS8OKTY8MEHH1BeXk7fvn354osvuPXWW4mKisLd3R13d3eioqK49dZb+eKLL+jbty/l5eXMmjXLEY8WadGC/Dy4ZlAUAJ+tTqSySrs0i4iIiIiI83NIsWHjxo2YTCaefvppfHx8Gm3n7e3N008/jd1uZ/369Y54tEiLN25wFH7ebmTmlrJ2d4bRcURERERERM7JIcWGkydP4uPjc14bP8bHx+Pr68vJkycd8WiRFs/Lw8qNw2MAWPx9MqVllQYnEhEREREROTuHFBusViuVlef3Achut1NRUYHV2iSnboq0SCP6RBIW5EVhSQVLN6caHUdEREREROSsHFJs6NixI2VlZaxbt+6cbdetW0dZWRkdO3Z0xKNFWgWrxczkUbEALNuaSm5hmcGJREREREREGueQYsOYMWOw2+088cQTJCYmNtouISGBJ598EpPJxNixYx3xaJFWo3+XNsS286e8wsbi75ONjiMiIiIiItIok91ut19qJ0VFRYwfP57MzEzc3NwYN24cQ4cOJSwsDKje02Hjxo0sW7aMiooKwsPD+eqrr/D19b3kN9DSVVXZyMkpPmc7q9VMUJAPubnFVFbqxIKW6ujxPF789w5MJnj27sFEhja+IasYT+NSxPloXIo4F41JEecTHOyDxXLp8xIcUmwAOHr0KL/+9a9JT0/HZDI12MZut9O+fXveeustOnfu7IjHtngqNshPzfx8DzuPnqZvXCgPTe5tdBw5C41LEeejcSniXDQmRZyPo4oNDllGAdC5c2e++OILHn74Ybp164bZbMZut2O32zGbzXTr1o1HHnmExYsXq9Agcgkmj4rFbDKxK+E0h1NzjY4jIiIiIiJSj8NmNvxURUUF+fn5AAQEBODm5gZAYWEhv/jFLzCZTCxYsKApHt2iaGaDNOTjZYdZvTOdmAh//vyLAY3OJhJjaVyKOB+NSxHnojEp4nycbmbDT7m5uREaGkpoaGhtoQGgsrKSgwcPcvDgwaZ6tEiLd+OwaDzcLCSfKGDb4VNGxxEREREREamjyYoNItJ0Anw9GDc4CoDPVydSWaVvAkRERERExHmo2CDioq65rAP+Pu5k5ZWyeme60XFERERERERqqdgg4qI83a1MHB4DwBfrj1FyptLgRCIiIiIiItVUbBBxYVf0iSA82Jui0gqWbE4xOo6IiIiIiAigYoOIS7OYzUwZFQvA8q1p5BScMTiRiIiIiIiIig0iLq9v51A6tw+gotLGou+TjY4jIiIiIiKiYoOIqzOZTEwdHQfA+r0nOJ5VZHAiERERERFp7awXc1O3bt0cnUNELkFsuwAGxrdh2+FTfLYmkd9O6WN0JBERERERacUuamaD3W6/pL9ExPEmjYzFYjaxJzGbg8dyjI4jIiIiIiKt2EXNbHjggQccnUNELlFYsDej+rZj5Y7jzFudyBO/DMJsMhkdS0REREREWiEVG0RakOuHRbN+3wlSThay5WAmQ7qHGx1JRERERERaIW0QKdKC+Pu4c+2QjgAsWJNERaXN4EQiIiIiItIaqdgg0sJcPagDgb7unM4/w6odx42OIyIiIiIirZCKDSItjIebhYlXdALgyw3HKD5TYXAiERERERFpbVRsEGmBhvUKp12oD8VnKvlmY4rRcUREREREpJVRsUGkBbKYzUweFQvAt9uOk51/xuBEIiIiIiLSmqjYINJC9Y4NoWtUIJVVNhauSzI6joiIiIiItCIqNoi0UCaTiSmj4wDYuO8kqZmFBicSEREREZHWQsUGkRYsJsKfy7q1xQ7MX51odBwREREREWklVGwQaeFuHhmLxWxif3IO+5KzjY4jIiIiIiKtgIoNIi1c20AvxvRvD8D8VYnY7HaDE4mIiIiISEunYoNIK3D9sGi8PKykZRWxaf9Jo+OIiIiIiEgLp2KDSCvg6+XG+KEdAVi4NomKyiqDE4mIiIiISEumYoNIK3HlgPYE+XmQXVDGyu3pRscREREREZEWTMUGkVbC3c3CTVd0AuCrDccoKq0wOJGIiIiIiLRUKjaItCKX9wynfRsfSsoq+XrjMaPjiIiIiIhIC6Vig0grYjabmDI6DoCV249zOq/U4EQiIiIiItISWY0OcKE2bdrErFmz2L17NyUlJURGRjJu3DimT5+Ot7f3BfX12GOPsXDhwrO2ee+99xgxYkS91+Pj4896X2hoKOvXr7+gPCLNoWdMMN06BnEwJZcF65KYfn0PoyOJiIiIiEgL41LFhjlz5vD8889jt9sJDw8nIiKChIQE3nrrLZYvX87cuXMJDAy84H4jIiKIiIho8FpAQMBZ7+3Zsyfu7u71Xr+YHCLNwWQyMXV0HE/P3sqm/ZlcMyiKjuF+RscSEREREZEWxGWKDfv27eOFF14A4JlnnmHq1KmYTCYyMzO577772L9/P0888QQzZ8684L4nTZrEgw8+eFG5/vWvf9G+ffuLulfEKB3D/RjSI4xN+zOZtyqBR37WF5PJZHQsERERERFpIVxmz4Y333wTm83GjTfeyC233FL7wSgsLIxXXnkFs9nM8uXLOXTokMFJRVzDzVd0wmoxcTAll33JOUbHERERERGRFsQlig3FxcWsW7cOgKlTp9a7Hh0dzZAhQwBYunRps2YTcVWhgV6MHVA9K2f+qgRsNrvBiUREREREpKVwiWUUBw8epLy8HHd3d3r37t1gmwEDBrBhwwZ27959wf1v3ryZo0ePkpeXh7+/Pz169OCGG26gXbt257z3zTffJCsri6qqKsLCwhgyZAjXXXddg/s4iDib8UOjWbf7BMdPFbNh30mG92547xIREREREZEL4RLFhuTkZAAiIyNxc3NrsE1UVFSdthdi69atdf7522+/5Y033uA3v/kN99xzz1nv/fzzz+v888KFC3nttdeYOXMmPXpol39xbr5ebky4PJp5qxJYuC6Jy7q1xd3NYnQsERERERFxcS5RbMjPzwfOfjJEzbWatuejY8eOPPbYYwwZMoR27drh7u7O4cOH+fDDD1m6dCkvv/wy3t7eTJs2rd69Y8eO5cYbb6Rr166Eh4dTXFzMxo0befXVV0lLS+Ouu+5i0aJFjZ5ycSGs1nOvdrFYzHX+V+R8XT24Ayu3Hye74Awrd6Rz/bBooyO1GBqXIs5H41LEuWhMijgfR+0bb7Lb7U6/UPuNN97gtddeY+DAgXzyyScNttm4cSN33HEHFouFAwcOXPIzn376aebOnYu/vz+rV6/Gx8fnvO7Lyclh0qRJZGRkMHnyZJ5//vlLymG323VKgDS5VdvTeGXuDrw9rbz7xysJ8PUwOpKIiIiIiLgwl5jZ4OFR/cGnoqKi0Tbl5eV12l6qhx9+mPnz51NQUMCmTZsYO3bsed0XHBzM9OnTeeqpp1ixYgXPPffcJRULbDY7BQUl52xnsZjx9/eioKCUqirbRT9PWqfeMUFEhfmSmlnEx1/v57ar442O1CJoXIo4H41LEeeiMSnifAICvDCbL322kUsUG85nicT5LLW4EH5+fnTu3JkDBw6QkpJyQff269cPgLy8PPLy8ggKCrqkLJWV5/8Hb1WV7YLai9SYMjqOf/x3Fyu3HWdMv3a0DfI2OlKLoXEp4nw0LkWci8akiPNw1NoHl1gcFR0dDUBGRkajsxtSU1PrtHWEms0oKysrL+o+gKqqKoflEWlKPaKD6RkTTJXNzoK1SUbHERERERERF+YSxYZu3brh5uZGeXk5e/bsabDN9u3bAejbt69DnllZWUlSUvUHrvDw8Au69+jRo0D1ko7AwECH5BFpDpNHxWICthzMIimjwOg4IiIiIiLiolyi2ODr68vw4cMBmDdvXr3rx44dY9OmTQCMGzfOIc/89NNPKSwsxGq1MmTIkPO+r7KyklmzZgEwZMgQrFaXWKkiAkBUmB+X96wurs1flYAL7B8rIiIiIiJOyCWKDQAzZszAZDKxePFiPv3009oPQVlZWTz88MPYbDauvPJKunbtWue+MWPGMGbMGJYuXVrn9fXr1/P3v/+dY8eO1Xm9vLycOXPm8OKLLwLws5/9jLZt29Zp8/LLL7Nw4UKKiorqvH7ixAkeeughdu3ahdVq5f7773fEWxdpVjeN6ITVYuZwWh67E7ONjiMiIiIiIi7IZb527927N4899hgvvfQSTz75JG+99RZBQUEkJCRQXl5OTEwMzz77bL370tPTASgpqXuiQ2lpKe+//z7vv/8+oaGhhIWFAZCcnFzb9pprruHRRx+t12dSUhLvvfcejz/+OB06dCAgIIDCwkKSk5Ox2+14eHjw3HPP0adPH0f/GESaXLC/J1cNas+STal8tjqRXp2CsThgN1oREREREWk9XKbYAHDHHXcQHx/Phx9+yJ49e8jOziYyMpJx48Yxffp0fHx8zruvHj16MGPGDHbt2kVKSgrJyclUVFQQHBzM8OHDuemmmxgzZkyD9956662Ehoayb98+srKySE9Px83Njc6dOzN06FBuu+02oqKiHPW2RZrd+CEdWbsrg4zTxazfe5IRfSKNjiQiIiIiIi7EZNeibKdWVWUjJ6f4nO2sVjNBQT7k5hbr2CBxiOVb0/jvyqME+Lrz0vSheLhbjI7kcjQuRZyPxqWIc9GYFHE+wcE+WCyXPrNZc6NFpEGj+7UjNMCT/KJylm9NNTqOiIiIiIi4EBUbRKRBblYzk0bGAvDN5lQKissNTiQiIiIiIq5CxQYRadSgbm2JDvejrLyKL9YnGx1HRERERERchIoNItIos8nElNFxAKzZlcHJnJJz3CEiIiIiIqJig4icQ7eOQfSODaHKZufzNYlGxxERERERERegYoOInNPkUbGYTLD98CkS0vONjiMiIiIiIk5OxQYROaf2bXwZ1isCgPmrEtCJuSIiIiIicjYqNojIeZk4PAZ3q5mjx/PZdfS00XFERERERMSJqdggIucl2N+TqwZ1AGD+6kSqbDaDE4mIiIiIiLNSsUFEztu1gzvi6+XGyZwS1u0+YXQcERERERFxUio2iMh58/a0csOwaAAWfZ/MmfJKYwOJiIiIiIhTUrFBRC7IqH7taBvoRUFxOcu2pBkdR0REREREnJCKDSJyQawWM5NGxQKwdHMq+UVlBicSERERERFno2KDiFywgfFtiInwp6yiisXrjxkdR0REREREnIyKDSJywUwmE1NHV89uWLsrgxPZxQYnEhERERERZ6Jig4hclPioIPrGhWKz2/lsdaLRcURERERExImo2CAiF23yqFhMJth59DRH0vKMjiMiIiIiIk5CxQYRuWiRoT6M6BMJwPxVCdjtdoMTiYiIiIiIM1CxQUQuyY3DY3B3M5OYUcD2w6eMjiMiIiIiIk5AxQYRuSSBvh6MuywKgM/WJFJZZTM4kYiIiIiIGE3FBhG5ZNdcFoW/txtZuaWs2ZVhdBwRERERETGYig0icsm8PKzcODwGgC/WJ1NaVmlwIhERERERMZKKDSLiEFf0iSQs2JvCkgqWbE41Oo6IiIiIiBhIxQYRcQirxczkkbEALN+SSm5hmcGJRERERETEKCo2iIjD9O8SSly7AMorbSz+PsnoOCIiIiIiYhAVG0TEYUwmE1NHxwGwbs8J0k8VGZxIRERERESMoGKDiDhUXPsABnRpg90On61ONDqOiIiIiIgYQMUGEXG4SaNiMZtM7E7M5lBKrtFxRERERESkmanYICIOFx7szch+kQDMW5WAzW43OJGIiIiIiDQnFRtEpEncMCwGD3cLx04Wsu1QltFxRERERESkGanYICJNIsDHnWsHRwHVezdUVNoMTiQiIiIiIs1FxQYRaTLXDIoiwMed0/lnWL0z3eg4IiIiIiLSTFRsEJEm4+FuYeIVMQB8ueEYJWcqDE4kIiIiIiLNQcUGEWlSw3tHEBHiTVFpBd9sSjU6joiIiIiINAMVG0SkSVnMZiaPigXg221p5BScMTiRiIiIiIg0NRUbRKTJ9Y0LpUv7ACoqbSxal2x0HBERERERaWIqNohIkzOZTEwZEwfA+r0nOJ5VZHAiERERERFpSio2iEiziI0MYGDXttiB+asTjY4jIiIiIiJNSMUGEWk2k0Z2wmI2sTcpmwPHcoyOIyIiIiIiTUTFBhFpNmFB3ozq1w6A+asSsdntBicSEREREZGmoGKDiDSr64dF4+luISWzkC0HMo2OIyIiIiIiTUDFBhFpVv7e7lw3pCMAn69JoqLSZnAiERERERFxNBUbRKTZXTWoA4G+7mQXnOG7HceNjiMiIiIiIg6mYoOINDsPNws3XdEJgK82HKP4TIXBiURERERExJFUbBARQwzrFUG7UB+Kz1Ty9cYUo+OIiIiIiIgDqdggIoYwm01MGR0LwIptxzmdX2pwIhERERERcRQVG0TEML06hdA1KpDKKhsL1yYbHUdERERERBxExQYRMYzJZGLK6DgANu0/ScrJQoMTiYiIiIiII6jYICKGionwZ3D3MOzAZ6sTjI4jIiIiIiIOoGKDiBju5hGdsJhN7D+Wy77kbKPjiIiIiIjIJVKxQUQM1ybQi7ED2gMwf1UiNpvd4EQiIiIiInIpVGwQEacw4fJovDyspGUVsXH/SaPjiIiIiIjIJVCxQUScgq+XGxOGdgRg4bokyiuqDE4kIiIiIiIXS8UGEXEaYwe0J9jfg5yCMlZuP250HBERERERuUgqNoiI03B3s3DTFZ0A+GpjCkWlFQYnEhERERGRi6Fig4g4laE9wunQ1pfSskq+2nDM6DgiIiIiInIRVGwQEadiNpuYMjoWgJXbj3Mqr9TgRCIiIiIicqFUbBARp9MzJoQe0UFU2ewsWJtkdBwREREREblAKjaIiFOaPCoOE7D5QCbJJwqMjiMiIiIiIhdAxQYRcUodw/0Y0iMcgPmrErDb7QYnEhERERGR86Vig4g4rZtGxGC1mDmUmsfepGyj44iIiIiIyHlSsUFEnFZogBdXDmwPwPzVidhsmt0gIiIiIuIKVGwQEac2fmhHfDytpJ8qZv2+E0bHERERERGR86Big4g4NR9PN8YPjQZg0bpkyiqqjA0kIiIiIiLnpGKDiDi9sQPaEeLvSW5hGSu2pRkdR0REREREzkHFBhFxem5WCzeP7ATA1xtTKCgpNziRiIiIiIicjYoNIuISBncPIyrMlzPlVXy1/pjRcURERERE5CxUbBARl2A2mZg6Og6AVTvTycwtMTiRiIiIiIg0RsUGEXEZ3aOD6dkpmCqbnQVrkoyOIyIiIiIijVCxQURcypRRcZiArYeySMzINzqOiIiIiIg0QMUGEXEpHdr6cnmvcADmr0rEbrcbnEhERERERH5KxQYRcTk3XdEJN6uZI2l57E7INjqOiIiIiIj8hIoNIuJygv09uWpgBwDmr06gymYzOJGIiIiIiPyYig0i4pKuG9IRXy83TmSX8P2eE0bHERERERGRH1GxQURckrenlesvjwZg0bpkysqrjA0kIiIiIiK1VGwQEZc1un872gR6kl9czrKtqUbHERERERGRH6jYICIuy2oxM2lkLABLNqeSX1xucCIREREREQEVG0TExQ3s2paYCD/Kyqv4Yn2y0XFERERERASwGh3gQm3atIlZs2axe/duSkpKiIyMZNy4cUyfPh1vb+8L6uuxxx5j4cKFZ23z3nvvMWLEiAavFRcX8+6777Js2TIyMjLw9vamT58+3HXXXQwePPiCsojIxTGbTEwZFcff/rOTNTszuHJAeyJCfIyOJSIiIiLSqrlUsWHOnDk8//zz2O12wsPDiYiIICEhgbfeeovly5czd+5cAgMDL7jfiIgIIiIiGrwWEBDQ4Os5OTn8/Oc/Jzk5GXd3d+Li4sjJyWH16tWsWbOGJ554gmnTpl1wFhG5cF07BtEnNoTdidksWJPE/Tf3MjqSiIiIiEir5jLFhn379vHCCy8A8MwzzzB16lRMJhOZmZncd9997N+/nyeeeIKZM2decN+TJk3iwQcfvKB7Hn/8cZKTk+nRowdvvfUWYWFh2O125s2bx5NPPsnzzz9P//796dat2wXnEZELN3lULHuSstl+5BQJx/OJa99woVBERERERJqey+zZ8Oabb2Kz2bjxxhu55ZZbMJlMAISFhfHKK69gNptZvnw5hw4davIsBw4c4LvvvsNsNvPqq68SFhYGgMlk4pZbbuHGG2+kqqqKN998s8mziEi1dm18uaJ39QyleasSsNvtBicSEREREWm9XKLYUFxczLp16wCYOnVqvevR0dEMGTIEgKVLlzZ5nmXLlgEwZMgQOnbsWO/6LbfcAsCaNWsoKSlp8jwiUu3G4Z1wt5pJSM9nx5HTRscRkf9v7/6DqqoT/4+/7r1wUfmhoIAC5u/wR/4CP2JmmGipH8tVU1tz+jHZj8k2a7PdclM312lwdp1N175NbrOubpvlj0r7aklb/ljDRCPJRCJBIBH8gYiAInC59/OHwUSAgh45F+7zMdMM95z3OfcFzUF4cc77DQAAPFaLeIwiLS1NFRUVstvtGjRoUL1joqOjtW/fPn377bdNPn9SUpKOHTumoqIiBQQEaMCAAZo8ebLCw8PrHZ+SkiJJGjZsWL37Bw0aJLvdrvLycqWlpSk6OrrJmQA0XaC/j+4Zfou27cvW5j2ZGty7o7xsLaJTBQAAAFqVFvFTeFbWleXswsLC5O3tXe+YW265pdbYpjh48KASEhKUlJSk//znP1qxYoXGjx+vt99+u97x2dnZtd7zl7y9vWsmnLyePACu38SYW+TfzlunCy9p77d5ZscBAAAAPFKLuLPhwoULkhpeGeLn+6rHNka3bt308ssva8SIEQoPD5fdbld6errWrFmjHTt2aPny5WrXrl2dVSWakqe4uLjReRri5XXtTsj2019vbfwVFx7O38uuqbE99a8d6dqamK1Rg8PU1secb3Vcl4D74boE3AvXJOB+fpoe8Ya1iLKhvLxckhq8q0GS7HZ7rbGN8fTTT9fZNnjwYK1cuVJLlizR+vXrtWLFCk2ZMkW+vr7Xlefy5cuNzlMfq9WiwEDfaw/8SUBA2xt6P6A1mBp3qz7/Old5BRe1KyVfsyf0NTUP1yXgfrguAffCNQm0Pi2ibPDx8ZEkVVZWNjimoqKi1tgb9cILL2jTpk0qLi7W/v37NXbs2Fp5ysrKGpWnTZs2N5TD6XSpuPjak0zabFYFBLRVcXGZqqqcN/SeQGtw/+ieWvXBd/pw9zGN7B+iDv7GfG9oCq5LwP1wXQLuhWsScD/t27eV1Xrjdxu1iLKhMY9INObRhqbw9/dXnz59dPToUeXk5NTaFxAQoLKyskblCQgIuOEsDkfjv/FWVTmbNB5orYb07qReYQHKzCvWB3sy9YiJdzdwXQLuh+sScC9ck4D7MGoF+RbxcFT37t0lSXl5eQ3eTfDjjz/WGmuE6sckHA5HvXl+WUJUq6ysVF5enuF5ADSexWLRjDG9JUn//TZPeQUXTU4EAAAAeI4WUTb069dP3t7eqqio0OHDh+sdk5ycLEkaMmSIIe/pcDh0/PhxSVLnzp1r7at+j+r3/KXDhw+rsrJSPj4+6tevnyF5ADTdrV07aGifTnK5pA/2ZJodBwAAAPAYLaJs8PPz06hRoyRJGzdurLM/Oztb+/fvlyRNmDDBkPfcsGGDSkpK5OXlpREjRtTaN378eElSUlJSvXc3bNiwQZIUGxtba2JJAM1v+l29ZLVYdOhYgX44UWR2HAAAAMAjtIiyQZLmzp0ri8WirVu3asOGDXL99CDJmTNn9MILL8jpdGrcuHHq27f2c9lxcXGKi4vTjh07am1PTEzUX/7yF2VnZ9faXlFRoXfeeUfx8fGSpF//+tcKCQmpNWbAgAEaM2aMqqqq9Nvf/lZnzpyRJLlcLm3YsEFbt26V1Wqtd7ULAM2rS0dfxQ7uIknauCuj5nsHAAAAgJvH4mpBP3mvXbtWy5Ytk8vlUpcuXRQYGKiMjAxVVFSoR48eWr9+vYKCgmodExkZKUmKj4/XtGnTarZ//vnneuaZZyRJnTp1UmhoqCQpKytLly5dWf1h/PjxWr58ec0ylj9XWFioWbNmKTs7W3a7Xb1799b58+eVn58vi8WiV155RQ899NANf85VVU4VFl77WXMvL6sCA311/vxFJtcBfuFCableXr1f5ZVVmjvlNg3rG3LtgwzAdQm4H65LwL1wTQLuJyjIVzabh6xGUe3RRx9VZGSk1qxZo8OHD+vcuXMKCwvThAkT9OSTTzbpkYUBAwZo7ty5SklJUU5OjrKyslRZWamgoCCNGjVKU6dOVVxcXIPHBwUF6YMPPtDbb7+tHTt2KCMjQ+3atVNsbKzmzJlT59ELAOZp7+ej8cO76uPEbG3ek6khfTrJy4BvoAAAAADq16LubPBE3NkAGKOs3KEFf9+v4osVmn33rRobHXHT35PrEnA/XJeAe+GaBNyPUXc28Kc9AB6hrY+XfjWqhyRp65dZKit3XOMIAAAAANeLsgGAx7hzUBd1Dmqn0rJKfZpUdyUZAAAAAMagbADgMbxsVk2/q5ck6bMDJ3S+pNzkRAAAAEDrRNkAwKMM7dNJvSPaq8Lh1Ja9x82OAwAAALRKlA0APIrFYtHMMb0lSV9+l6/cs6UmJwIAAABaH8oGAB6nd3h7RUcGy+WSNu/ONDsOAAAA0OpQNgDwSPeP7iWb1aLDmeeUlnPe7DgAAABAq0LZAMAjdQ5qp9FDwiRJm3ZlyOlymZwIAAAAaD0oGwB4rMl39JCP3absUyU6mHbG7DgAAABAq0HZAMBjBfja9b8xt0iSPtiTqUqH0+REAAAAQOtA2QDAo93zP7eovZ9dBRcua9ehk2bHAQAAAFoFygYAHs3HbtPUO3tKkv5/YpYuXa40OREAAADQ8lE2APB4dwzsrLBOvrp42aHt+3PMjgMAAAC0eJQNADyezWrV9Lt6SZL+czBX5y5cNjkRAAAA0LJRNgCApMG9Oiqyawc5qpzasve42XEAAACAFo2yAQAkWSwWzRjTW5K078gp/Xi6xOREAAAAQMtF2QAAP+kZFqDh/ULkkrR5d6bZcQAAAIAWi7IBAH5mWmxP2awWHckqVGpWodlxAAAAgBaJsgEAfiYksJ3GRIVLkjbtypDT5TI5EQAAANDyUDYAwC/cN7K72vrY9OOZUiWlnjY7DgAAANDiUDYAwC/4t7Prf0d0kyR9+N9MVTqqTE4EAAAAtCyUDQBQj7uHdVWgv4/OFZfri+STZscBAAAAWhTKBgCoh93bpil39pAkbduXrdKySpMTAQAAAC0HZQMANOCO27ooPNhXl8od+uSrHLPjAAAAAC0GZQMANMBqtWjGXb0lSZ8nn1BBUZnJiQAAAICWgbIBAK5iYM8g9esWKEeVSx/tPW52HAAAAKBFoGwAgKuwWCyaMaaXJOmr1NPKOVViciIAAADA/VE2AMA1dO8coBH9QyVJG3dlyOVymZwIAAAAcG+UDQDQCFNje8rLZlFaznmlZhWaHQcAAABwa5QNANAIwR3aKi4qQpK0cVemnE7ubgAAAAAaQtkAAI1078juaufjpdyzpfoq9ZTZcQAAAAC3RdkAAI3k19Zbk0Z2kyR9+N/jqqisMjkRAAAA4J4oGwCgCcZFR6hjgI/Ol5Tr8+Rcs+MAAAAAbomyAQCawNvLpqmxPSVJ27/KVsmlCpMTAQAAAO6HsgEAmmjEgM7qGuKnsvIqbduXY3YcAAAAwO1QNgBAE1ktFs0c01uStPObXJ0pKjM5EQAAAOBeKBsA4DoM6BGkAT2CVOV06cM9mWbHAQAAANwKZQMAXKcZd/WSRdKBtDPKyi82Ow4AAADgNigbAOA63RLqr9tv6yxJ2rgzQy6Xy+REAAAAgHugbACAGzD1zp7yslmVfqJIhzPPmR0HAAAAcAuUDQBwAzq2b6O7h0VIkjbtzlSV02lyIgAAAMB8lA0AcIMm3d5Nvm28lFdwUYnfnTI7DgAAAGA6ygYAuEHt2njrvpHdJUlb9h5XeUWVuYEAAAAAk1E2AIABxkRFqFP7NioqrdBnX58wOw4AAABgKsoGADCAt5dV00b3lCR9uj9HxRcrTE4EAAAAmIeyAQAMMrxfqLp19tfliip9/GWW0rILteebXKVlF8rpZFlMAAAAeA4vswMAQGthtVg0c0xv/eW9Q9p56KR2HjpZsy/Q30cPjuuj6MgQExMCAAAAzYM7GwDAQJcuV9a7/XxJuf7fR0eUnH6mmRMBAAAAzY+yAQAM4nS6tP7zY1cd897nx3ikAgAAAK0eZQMAGOSHE0U6X1J+1TGFJeX64URR8wQCAAAATELZAAAGKbp49aKhqeMAAACAloqyAQAM0sHXx9BxAAAAQEtF2QAABrm1awcF+l+9SAjy99GtXTs0TyAAAADAJJQNAGAQq9WiB8f1ueqYWeP6yGq1NFMiAAAAwByUDQBgoOjIED0z9bY6dzgE+fvomam3KToyxKRkAAAAQPPxMjsAALQ20ZEhGtonWJl5F1Tpssjb4lKvsPbc0QAAAACPQdkAADeB1WpRv+5BCgz01fnzF+VwOM2OBAAAADQbHqMAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGomwAAAAAAACGsrhcLpfZIdAwl8slp7Nx/4tsNquqqpw3ORGApuC6BNwP1yXgXrgmAfditVpksVhu+DyUDQAAAAAAwFA8RgEAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAzlZXYAXL+zZ88qMTFRR44c0Xfffae0tDSVl5dr+PDheuedd8yOB3gUl8ulQ4cOaefOnUpOTtbx48dVWloqf39/9e/fX1OmTNF9990ni8VidlTAo3z66afat2+fUlNTdebMGRUVFcnb21vdu3fX6NGj9cgjjygwMNDsmIBH27Nnj5588klJUnh4uHbu3GlyIsCzrFq1Sm+88cZVx7z66quaNWtWk85L2dCCbd++XfHx8WbHACBp//79evTRR2ted+3aVeHh4Tp58qQSExOVmJio7du3a9WqVbLb7eYFBTzMW2+9pe+//152u13BwcGKjIxUYWGhjh49qqNHj2rjxo1as2aN+vbta3ZUwCNdvHhRr776qtkxAEjq2LGjunXrVu++4ODgJp+PsqEF8/Pz08iRIzVw4EANHDhQR48e1Ztvvml2LMAjuVwuRURE6JFHHtGkSZPUsWPHmn1btmzRokWLtHv3bq1cuVK/+93vTEwKeJbZs2erR48eGjJkiLy9vWu2p6en68UXX9QPP/yg+fPna/v27SamBDzX66+/rry8PI0dO1ZffPGF2XEAjxYbG6tly5YZdj7KhhZs+vTpmj59es3r06dPm5gG8GyDBg3Sjh07av0yU23KlCk6deqUXn/9dW3evFnz58+X1cqUOUBzmDlzZr3bIyMj9dprr2nGjBnKyMhQZmamevXq1czpAM+WkpKid999V2PHjtW4ceMoG4BWhp92AcAAfn5+9RYN1WJjYyVJRUVFKiwsbK5YAK6iZ8+eNR+XlZWZmATwPJWVlVq0aJHatGmjxYsXmx0HwE3AnQ0A0AwuX75c83GbNm1MTAKgWnJysiSpXbt26tGjh8lpAM+yevVq/fDDD1qwYIE6d+5sdhwAkr7//nvNnz9fZ8+ela+vryIjIzVp0iT16dPnus5H2QAAzaD6efC+ffvKz8/P5DSA53I6nTWrOS1fvlyS9OKLL8rX19fkZIDnyMzM1OrVqzVgwAA99NBDZscB8JO0tDSlpaXVvN65c6feeustPfzww3rppZdks9madD7KBgC4yY4cOaL3339fkmqW9gLQvNauXVtnBadBgwZp2bJlNY85Abj5XC6XFi5cKIfDoSVLljT5lxcAxgsJCdG8efN05513KiIiQn5+fsrKytL69ev1/vvva926dfLy8tLvf//7Jp2XsgEAbqKCggI9++yzcjgcuvvuuzVp0iSzIwEeKTQ0VFFRUaqqqlJeXp4KCgqUlpamrVu3asiQIQoICDA7IuAR1q9fr2+++UYPPfSQBg4caHYcAJIeeOCBOtsiIyO1ZMkSRUREaPny5Vq3bp0efPBBRURENPq8TBAJADdJSUmJnnjiCeXl5WnAgAGGLiUEoGkmTpyo9957Txs3btSXX36pLVu2aPDgwdq2bZsefvhhVVVVmR0RaPVOnz6tv/71rwoNDdXzzz9vdhwAjfDYY48pJCREDodDO3fubNKxlA0AcBNcvHhRjz/+uI4ePao+ffroH//4B3M1AG6kb9++Wr16tQIDA5WWllYzrwqAm2fp0qUqLS3VwoUL+TcRaCFsNpsGDx4sScrJyWnSsTxGAQAGKysr01NPPaWUlBR1795d//znPxUYGGh2LAC/4Ofnp+HDhyshIUGpqamaPHmy2ZGAVu3o0aOSpCVLlmjJkiW19lWv2pSfn6877rhDkrRq1SpFRUU1b0gAdVQv7+5wOJp0HGUDABiovLxcTz/9tA4ePKjw8HCtXbtWwcHBZscC0IDqH5x4jAJoPgUFBQ3uczqdNfsrKyubKxKAqzh27JgkNXmZWsoGADBIZWWlnn32WX311VcKDQ3VunXr1KVLF7NjAWhAUVGRDhw4IEnq16+fyWmA1u9qz3t/+OGHWrBggcLDw5v8XDiAm2f37t01ZUP1XUeNxZwNAGCAqqoqzZ8/X3v27FFwcLDWrVunrl27mh0L8GgHDhzQm2++qdzc3Dr7UlNTNWfOHJWUlCg0NFQTJkwwISEAAOY6duyYFi9erO+//77WdqfTqW3btmn+/PmSpDFjxmjQoEFNOjd3NrRg+fn5mjJlSs3riooKSdI333yjmJiYmu2PP/64nnjiieaOB3iUTz/9VAkJCZIku92uP/zhDw2OXbRokfr3799c0QCPVVxcrJUrV2rlypUKDg5WSEiIbDab8vPzdfbsWUlXlsRcvXq1fH19TU4LAEDzczgc2rBhgzZs2KAOHTooLCxMNptNP/74oy5cuCBJGjZsmP785z83+dyUDS1YVVWVioqK6mx3OBy1tldPuAPg5qku+yTp5MmTOnnyZINjS0pKmiMS4PGGDh2qBQsWKCkpSRkZGcrOzlZFRYUCAgIUExOjuLg4TZ8+nVnxAQAeKzw8XM8//7xSUlKUmZmpnJwcVVRUqH379oqNjdW9996re++9VzabrcnntrhcLtdNyAwAAAAAADwUczYAAAAAAABDUTYAAAAAAABDUTYAAAAAAABDUTYAAAAAAABDUTYAAAAAAABDUTYAAAAAAABDUTYAAAAAAABDUTYAAAAAAABDUTYAAAAAAABDUTYAAABch8jISEVGRiopKcnsKAAAuB0vswMAAIDWYdWqVXrjjTcaPT49Pf0mpgEAAGaibAAAAIbr1KmT2REAAICJKBsAAIDhEhMTzY4AAABMxJwNAAAAAADAUNzZAAAATBcXF6eTJ08qPj5e99xzj1avXq3PPvtM+fn5atu2raKjo/XUU09p8ODBDZ6jqqpKH330kT7++GOlp6fr4sWLCgwM1NChQzV79mzFxMRcNUN+fr7eeecdJSYmKjc3V5WVlQoJCVGfPn00fvx4TZw4UT4+PvUeW1paqrffflsJCQnKy8tT27ZtNWTIEM2dO/eqmQEAaK0oGwAAgNsoLi7W9OnTlZWVJW9vb/n4+KioqEhffPGFdu3apaVLl2r69Ol1jispKdHcuXN14MABSZLNZpOvr6/Onj2rhIQEJSQk6LHHHtNLL71U7/tu2bJFixcvVnl5uSTJ29tbvr6+ys/P14kTJ7Rz505FRkaqX79+dY49e/aspk2bppycHPn4+MhqtaqoqEi7d+9WYmKi3nrrLY0aNcrArxIAAO6PxygAAIDbeOONN1RYWKgVK1YoJSVFycnJ+uSTTzR8+HA5nU798Y9/VGpqap3jXnnlFR04cEDe3t5auHChkpOTdfDgQe3du1f333+/JGnNmjV677336hy7e/duvfzyyyovL1dUVJTeffddHT58WElJSTp06JDeffddzZw5U97e3vVm/tOf/iRvb2+tW7dOKSkpOnTokDZt2qQePXqosrJSixcvltPpNPYLBQCAm7O4XC6X2SEAAEDL9/OlL6+1GsXEiRO1cOHCmtfVj1FI0tq1a3X77bfXGn/58mX96le/UnZ2tkaPHq2///3vNfu+/fZbzZw5U9KVX/wfeOCBOu83b948JSQkKDAwUHv27Kl5HMLhcGj8+PHKzc1VdHS01q5dK7vd3qjPNzIyUpIUFBSkbdu2qWPHjrX2p6ena/LkyZKk9evXKzo6ulHnBQCgNeDOBgAAYLiCgoKr/ldaWlrvcVFRUXWKBklq06aN5syZI0nau3evSkpKavZ98sknkqTOnTtrxowZ9Z73ueeekySdP3++1koZSUlJys3NlSQtWLCg0UXDz82cObNO0SBdKSMiIiIkXSkeAADwJMzZAAAADHe9v1yPGDHimvucTqdSU1NrXh85ckSSFBMTI6u1/r+j9OrVS6GhoTp9+rSOHDmiuLg4SdKhQ4ckScHBwRo4cOB1Zb7aBJAhISHKzc3VhQsXruvcAAC0VNzZAAAA3EZoaGij9hUWFtZ8fO7cuWseK1258+Hn46UrkztKUlhYWNPD/sTX17fBfV5eV/6u43A4rvv8AAC0RJQNAADAY1ksFrMjAADQKlE2AAAAt3H69OlG7QsKCqr5uHq+hFOnTl313NX7fz6/QvVElnl5eU0PCwAAGkTZAAAA3EZSUtI191mtVvXv379m+2233Vazv6ElJjMzM2vKip/PzRAVFSXpyuMU33333Y2FBwAANSgbAACA20hOTq63cCgvL9eaNWskSaNGjVJAQEDNvkmTJkm6cufDpk2b6j3v3/72N0lSYGCgRo4cWbM9JiZGXbt2lSTFx8eroqLCmE8EAAAPR9kAAADchr+/v+bNm6cdO3bUTKqYmZmpJ598UsePH5fNZtO8efNqHTNo0CCNHz9ekrR06VL9+9//VllZmaQrdywsXLhQO3bskHRlCUwfH5+aY202mxYtWiSLxaLk5GQ9+uij+vrrr2vukKioqFBSUpJefPFFZWRk3PTPHwCA1oKlLwEAgOHuuOOOa45ZtWpVzWMM1X7zm9/o/fff13PPPSe73S4fHx+VlJRIujKZ46uvvlrvEpWvvfaazp8/rwMHDmjp0qWKj4+Xr6+viouL5XK5JEmPPfaYZs2aVefY0aNHa9myZVq0aJGSk5M1e/Zs2e12tWvXTqWlpTWlx5w5c5r8dQAAwFNRNgAAAMMVFBRcc0xlZWWdbQEBAdq8ebNWr16tzz77TPn5+erQoYOGDh2qp556SkOHDq33XP7+/lq7dq0++ugjbd26Venp6bp06ZI6deqkqKgozZ49WzExMQ1mmTJlioYNG6Z//etfSkxMVF5ensrLyxUWFqZbb71V99xzj3r16tX4LwAAAB7O4qqu+wEAAEwSFxenkydPKj4+XtOmTTM7DgAAuEHM2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAxF2QAAAAAAAAzFBJEAAAAAAMBQ3NkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAMRdkAAAAAAAAM9X+Pb8V9DBPU6QAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 1200x600 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "df_stats = pd.read_parquet(f\"{output_dir}/training_stats.parquet\")\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4, 5])\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dXX4z7Wqe8ry"
      },
      "source": [
        "13. Load and test the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mpBJNtYuAY9D",
        "outputId": "76ec7e63-ee22-45c4-b690-60ac501fbbf4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your question: What is the capital of France?\n",
            "Question: 👤 What is the capital of France?\n",
            "\n",
            "Response 1: 🤖 The capital of France is Paris.\n",
            "Response 2: 🤖 The capital of France is Paris.\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "import torch\n",
        "\n",
        "device = torch.device(\"cuda\"  if torch.cuda.is_available() else  \"cpu\")\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(output_dir)\n",
        "aira = AutoModelForCausalLM.from_pretrained(output_dir)\n",
        "\n",
        "aira.eval()\n",
        "aira.to(device)\n",
        "\n",
        "question =  input(\"Enter your question: \")\n",
        "\n",
        "inputs = tokenizer(tokenizer.bos_token + question + tokenizer.eos_token, return_tensors=\"pt\").to(device)\n",
        "\n",
        "responses = aira.generate(**inputs,\n",
        "    bos_token_id=tokenizer.bos_token_id,\n",
        "    pad_token_id=tokenizer.pad_token_id,\n",
        "    eos_token_id=tokenizer.eos_token_id,\n",
        "    do_sample=True,\n",
        "    top_k=50,\n",
        "    max_length=200,\n",
        "    top_p=0.95,\n",
        "    temperature=0.7,\n",
        "    num_return_sequences=2)\n",
        "\n",
        "print(f\"Question: 👤 {question}\\n\")\n",
        "\n",
        "for i, response in  enumerate(responses):\n",
        "    # print only the response and remove the question\n",
        "    print(f'Response {i+1}: 🤖 {tokenizer.decode(response, skip_special_tokens=True).replace(question, \"\")}')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cJXlHi2oe8rz"
      },
      "source": [
        "Done! 🤗"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "A100",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0659624c4b9a4ce8b396b8d45814c580": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ab4d80ebe66546ec92877f3288729bcd",
            "placeholder": "​",
            "style": "IPY_MODEL_e1760e7d52464351ba391e002132a0bf",
            "value": " 456k/456k [00:00&lt;00:00, 1.08MB/s]"
          }
        },
        "0fc39225b0694840b7f421f725c135f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4886d255282b4eefa37e7a1db102a132",
              "IPY_MODEL_7a37f293f6884093ab588c9ded32db07",
              "IPY_MODEL_bd0f1d0070294b6eaebba425e9224150"
            ],
            "layout": "IPY_MODEL_7cccbb84d04b4aaaa09309bb669f349d"
          }
        },
        "10d2880b45814382951aa1015f4f1f5f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "129c206371304edd9eaf34a5b56a00b8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13c74818a5b949438589a0c73007e5a6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1bb5acd98f244616905e0f5cb8752a8a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e2b0f9db8654f85938565d9f694030d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2231293622594eceb80d6481799ddcad": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c1135b0eb85044ab8e47e772e0c71f3c",
              "IPY_MODEL_41b5bf9be3394e728e3f3699026ce885",
              "IPY_MODEL_96552a3dbfa94bb4818b054b2675c6b8"
            ],
            "layout": "IPY_MODEL_13c74818a5b949438589a0c73007e5a6"
          }
        },
        "2cdd431ccdb64e409229913cc43e2544": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2d1b573ef7a242c295ed42d790a71f1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "30ce9a42524446edbf99a1147377ae8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b053f2aac4e42e09c446691cbe67416",
            "placeholder": "​",
            "style": "IPY_MODEL_62ebb5cbcbe94e0bb416896ec8493b5c",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "3181ccb3dde340c1b5c67ed12076c897": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3657a339af424c51ac7534f91fcb5893": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "41b5bf9be3394e728e3f3699026ce885": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b27100866f8b4c08a7931f4d3d84f2b3",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_47472c3a381f4eca9af9d03df1718065",
            "value": 1042301
          }
        },
        "433e467610674781b011918357e68baa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66e3178dae0d4b78ac3e00374ebd8e6b",
            "placeholder": "​",
            "style": "IPY_MODEL_47449e8feed84a5cb428b2ac0006a053",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "47449e8feed84a5cb428b2ac0006a053": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "47472c3a381f4eca9af9d03df1718065": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4886d255282b4eefa37e7a1db102a132": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2cdd431ccdb64e409229913cc43e2544",
            "placeholder": "​",
            "style": "IPY_MODEL_ba5408f14ed54e75a9724d39ac67d5c2",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "4a22e5e3eb354b4cb63bf0aeb9105efa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4cf2d43d7ae24582ba035d9770f8e860": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "57a05e39d49a4b4b853381676bd7ec9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69d7726ac4804227ab1b1c026ca29c2b",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ba9429a433b5479eae6e188f1c5b864f",
            "value": 456318
          }
        },
        "5b053f2aac4e42e09c446691cbe67416": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "601e6dedd4a64ddcbeb5ead5b20c70aa": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62ebb5cbcbe94e0bb416896ec8493b5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6539b916daa747adab05dcff94c2cd3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_433e467610674781b011918357e68baa",
              "IPY_MODEL_57a05e39d49a4b4b853381676bd7ec9d",
              "IPY_MODEL_0659624c4b9a4ce8b396b8d45814c580"
            ],
            "layout": "IPY_MODEL_7ebf18e21686429391b47866e46f8c6d"
          }
        },
        "66e3178dae0d4b78ac3e00374ebd8e6b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69d7726ac4804227ab1b1c026ca29c2b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69e756a204214d689240aef2b9986118": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6bab36d7ee934127a80a53ab87e0e508": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75ef15d0ffba4344bc07a22690a99f13": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_acbc045dd45c41c484e59304a66336f4",
            "placeholder": "​",
            "style": "IPY_MODEL_b8b4e3e26208445e91aa15ee3ea6e6b0",
            "value": " 666/666 [00:00&lt;00:00, 63.5kB/s]"
          }
        },
        "7a37f293f6884093ab588c9ded32db07": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10d2880b45814382951aa1015f4f1f5f",
            "max": 3247159078,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3657a339af424c51ac7534f91fcb5893",
            "value": 3247159078
          }
        },
        "7cccbb84d04b4aaaa09309bb669f349d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ddef1b949f04c81b6e26df3ce68cbc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_89f8e93f887d43ee933fa3b44a44fd72",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e58aaf5a1b9a4a2f85fa2546ee88f24e",
            "value": 124
          }
        },
        "7ebf18e21686429391b47866e46f8c6d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8691d4c5d8d54e2494d2bf149feb9980": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_129c206371304edd9eaf34a5b56a00b8",
            "placeholder": "​",
            "style": "IPY_MODEL_2d1b573ef7a242c295ed42d790a71f1e",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "89f8e93f887d43ee933fa3b44a44fd72": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96552a3dbfa94bb4818b054b2675c6b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9cdd14da170b4b40b029b0150a40afc7",
            "placeholder": "​",
            "style": "IPY_MODEL_4cf2d43d7ae24582ba035d9770f8e860",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 1.66MB/s]"
          }
        },
        "96de7beae5b94e6a92f9f52b9d824035": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9cdd14da170b4b40b029b0150a40afc7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab4d80ebe66546ec92877f3288729bcd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acbc045dd45c41c484e59304a66336f4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b023d77e9b854acd94ac0a91feb2de83": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b27100866f8b4c08a7931f4d3d84f2b3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2d0b131237f49e684451afe60b57593": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8691d4c5d8d54e2494d2bf149feb9980",
              "IPY_MODEL_7ddef1b949f04c81b6e26df3ce68cbc6",
              "IPY_MODEL_e125c896502c4ad48b4bfbdbe3dc7383"
            ],
            "layout": "IPY_MODEL_1e2b0f9db8654f85938565d9f694030d"
          }
        },
        "b8b4e3e26208445e91aa15ee3ea6e6b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba5408f14ed54e75a9724d39ac67d5c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba9429a433b5479eae6e188f1c5b864f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bd0f1d0070294b6eaebba425e9224150": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1bb5acd98f244616905e0f5cb8752a8a",
            "placeholder": "​",
            "style": "IPY_MODEL_b023d77e9b854acd94ac0a91feb2de83",
            "value": " 3.25G/3.25G [00:07&lt;00:00, 507MB/s]"
          }
        },
        "c1135b0eb85044ab8e47e772e0c71f3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6bab36d7ee934127a80a53ab87e0e508",
            "placeholder": "​",
            "style": "IPY_MODEL_69e756a204214d689240aef2b9986118",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "c119c152c0f3485daebb7cbc5992f13a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_96de7beae5b94e6a92f9f52b9d824035",
            "max": 666,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e521d59d193b42a28f29b3ad77c12c96",
            "value": 666
          }
        },
        "e125c896502c4ad48b4bfbdbe3dc7383": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3181ccb3dde340c1b5c67ed12076c897",
            "placeholder": "​",
            "style": "IPY_MODEL_4a22e5e3eb354b4cb63bf0aeb9105efa",
            "value": " 124/124 [00:00&lt;00:00, 10.2kB/s]"
          }
        },
        "e1760e7d52464351ba391e002132a0bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e2ca1209ffe94778a38a383641b09cb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_30ce9a42524446edbf99a1147377ae8e",
              "IPY_MODEL_c119c152c0f3485daebb7cbc5992f13a",
              "IPY_MODEL_75ef15d0ffba4344bc07a22690a99f13"
            ],
            "layout": "IPY_MODEL_601e6dedd4a64ddcbeb5ead5b20c70aa"
          }
        },
        "e521d59d193b42a28f29b3ad77c12c96": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e58aaf5a1b9a4a2f85fa2546ee88f24e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}