{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cellId": "nwmnsa077j0li6frzyvoc"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import json\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "dfsqzm9i1rg0djpmrynj4"
   },
   "source": [
    "Для начала распарсим датасет \"по умолчанию\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cellId": "g133nsvyrtio3fpwpn54m"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "file = open('arxivData.json')\n",
    "data = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "cellId": "04b7jpapxwa243bwo5a9fic"
   },
   "outputs": [],
   "source": [
    "def trl(container):\n",
    "    return tqdm(range(len(container)))\n",
    "\n",
    "def prepared(string):\n",
    "    string = string.replace(\"'\", '\"')\n",
    "    string = string.replace('None', 'null')\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "cellId": "5pr61iqulb4yxea8bjlhw"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78535363d49b48889ed5beaf19af863c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=41000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc1c7b04e9634db295cde22088934053",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=41000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "039e3d572c70452ca0710e6c61d10b80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=41000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "titles = [data[i]['title'] for i in trl(data)]\n",
    "summaries = [data[i]['summary'] for i in trl(data)]\n",
    "tags = [[json.loads(prepared(data[i]['tag']))[j]['term'] for j in range(len(json.loads(prepared(data[i]['tag']))))] for i in trl(data)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "i36xlscxkm9t8ab4hye7c"
   },
   "source": [
    "Теперь соберем данные о тегах со страницы арксива."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "cellId": "h0nt7pj6blgyodi07nt3e"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "page = requests.get(\"https://arxiv.org/category_taxonomy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "cellId": "eqd8ffof4m97mqffttt"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/kernel/lib/python3.8/site-packages/ml_kernel/kernel.py:872: UserWarning: The following variables cannot be serialized: soup\n",
      "  warnings.warn(message)\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "soup = BeautifulSoup(page.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "cellId": "7lwv6ajhf8ewgolxzof1bm"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/kernel/lib/python3.8/site-packages/ml_kernel/kernel.py:872: UserWarning: The following variables cannot be serialized: h4_cases, soup\n",
      "  warnings.warn(message)\n"
     ]
    }
   ],
   "source": [
    "h4_cases = soup.find_all('h4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "cellId": "ybnwppuruulh7ljycj7rh"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/kernel/lib/python3.8/site-packages/ml_kernel/kernel.py:872: UserWarning: The following variables cannot be serialized: h4_cases, html_tag, soup, tags_tags\n",
      "  warnings.warn(message)\n"
     ]
    }
   ],
   "source": [
    "tags_tags = h4_cases[1:]\n",
    "possible_tags = set()\n",
    "tag_to_id = {}\n",
    "id_to_tag = {}\n",
    "id_to_description = {}\n",
    "for i, html_tag in enumerate(tags_tags):\n",
    "    idx = html_tag.get_text().find(' ')\n",
    "    tag = html_tag.get_text()[:idx]\n",
    "    description = html_tag.get_text()[idx+2:-1]\n",
    "    possible_tags.add(tag)\n",
    "    tag_to_id[tag] = i\n",
    "    id_to_tag[i] = tag\n",
    "    id_to_description[i] = description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "cellId": "pplhqb4oesl6pq0w2pr3fe"
   },
   "outputs": [],
   "source": [
    "num_tags = len(possible_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "cellId": "vzlkypl5cyl3nagb85pphc"
   },
   "outputs": [],
   "source": [
    "with open('tags.txt', 'w') as file:\n",
    "    for i in range(num_tags):\n",
    "        file.write(str(id_to_tag[i]) + ' ' + str(id_to_description[i]) + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "vsiuhkjhmqkygl63n2pk9"
   },
   "source": [
    "И переведем теги в индексы."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "cellId": "m26ffr9sr4rc00y2kupo8q"
   },
   "outputs": [],
   "source": [
    "for i, line in enumerate(tags):\n",
    "    line = list(set(line) & possible_tags)\n",
    "    int_line = []\n",
    "    for tag in line:\n",
    "        int_line.append(tag_to_id[tag])\n",
    "    tags[i] = int_line"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "jgvhsk815g9k8wu66ftx"
   },
   "source": [
    "Разобъем данные на трейн и тест..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "cellId": "icw5n8u2se021wsdwlwio2"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "titles_train, titles_test, summaries_train, summaries_test, tags_train, tags_test = train_test_split(titles, summaries, tags, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "rs6f6fyz5nn8x8ycz12m"
   },
   "source": [
    "И создадим датасеты и дадалоадеры."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cellId": "9mji2qz8u8q79ays6y1j6"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7c008df0c254cefbd8d2402ae5020ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=28.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1e94a7ab27e40a9967f12f7011076c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=483.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c676bcfd5d8a49e9bb8b5c35ca64cd33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=231508.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc4baeeb13f04f5da7bc57094aa0828f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=466062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#!g1.1\n",
    "from transformers import AutoTokenizer, AutoModel, pipeline, DistilBertForSequenceClassification\n",
    "\n",
    "base_model_name = 'distilbert-base-uncased'\n",
    "base_tokenizer = AutoTokenizer.from_pretrained(base_model_name)\n",
    "#base_model = AutoModel.from_pretrained(base_model_name).to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cellId": "1cumn09ceki8g3916s4e4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./tokenizer/tokenizer_config.json',\n",
       " './tokenizer/special_tokens_map.json',\n",
       " './tokenizer/vocab.txt',\n",
       " './tokenizer/added_tokens.json',\n",
       " './tokenizer/tokenizer.json')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#!g1.1\n",
    "base_tokenizer.save_pretrained('./tokenizer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "cellId": "3rka3rozlhe1pwg0ewqvfs"
   },
   "outputs": [],
   "source": [
    "def multi_hot(indexes, num_classes):\n",
    "    return torch.zeros((num_classes,)).scatter_(0, torch.tensor(indexes), torch.ones((num_classes,)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "cellId": "ieptlunauzkup3dj55hvnc"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "class ArXivDataset(Dataset):\n",
    "    def __init__(self, titles, abstracts, tags, possible_tags, tokenizer):\n",
    "        super(ArXivDataset, self).__init__()\n",
    "        self. titles = titles\n",
    "        self.abstracts = abstracts\n",
    "        self.tags = tags\n",
    "        self.possible_tags = possible_tags\n",
    "        self.num_classes = len(self.possible_tags)\n",
    "        self.tokenizer = tokenizer\n",
    "        \n",
    "        self.inputs = ['Title: ' + self.titles[idx] + ' Abstract: ' + self.abstracts[idx] for idx in range(len(titles))]\n",
    "        self.inputs = self.tokenizer(self.inputs, truncation=True, padding='max_length', max_length=512, return_tensors='pt')['input_ids']\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.titles)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return (self.inputs[idx], multi_hot(self.tags[idx], self.num_classes) / len(self.tags[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "cellId": "vapsctsu94mjwcjydp4al"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "base_train_ds = ArXivDataset(titles_train, summaries_train, tags_train, possible_tags, base_tokenizer)\n",
    "base_test_ds = ArXivDataset(titles_test, summaries_test, tags_test, possible_tags, base_tokenizer)\n",
    "\n",
    "base_train_dl = DataLoader(base_train_ds, shuffle=True, \n",
    "    batch_size=128, num_workers=0)\n",
    "base_test_dl = DataLoader(base_test_ds, shuffle=True, \n",
    "    batch_size=128, num_workers=0)\n",
    "\n",
    "small_ds = ArXivDataset(titles_train[:256], summaries_train[:256], tags_train[:256], possible_tags, base_tokenizer)\n",
    "small_dl = DataLoader(small_ds, shuffle=True, \n",
    "    batch_size=128, num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "61ne5cjsd7uuxa67duqn1g"
   },
   "source": [
    "Следующий класс в итоге не используется, мы будем использовать встроенную архитектуру головы классификатора."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "cellId": "wmqvly8ahucv5xfnudiaj"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class BaseArXivClassifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BaseArXivClassifier, self).__init__()\n",
    "        self.head = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            nn.Linear(512*768, 155),\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "    \n",
    "    def forward(self, X):\n",
    "        with torch.no_grad():\n",
    "            X = base_model(X).last_hidden_state\n",
    "        X = self.head(X)\n",
    "        return X"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "mavysgsu3gr19liok5m4p3"
   },
   "source": [
    "Определим функции для тренировки и отрисовки графиков"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "cellId": "5dbnf9xl7hif6k1uzavgjr"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_learning_process(train_loss, val_loss):    \n",
    "    plt.figure()\n",
    "    plt.title('loss by epoch')\n",
    "    plt.plot(np.arange(0, len(val_loss))+0.5, train_loss, label='train')\n",
    "    plt.plot(np.arange(0, len(val_loss))+1, val_loss, label='val')\n",
    "    plt.legend()\n",
    "    plt.grid()\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('loss function')\n",
    "    plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "cellId": "441ify9vq4qumr997qqum"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "\n",
    "device = 'cuda'\n",
    "\n",
    "import IPython\n",
    "from math import ceil\n",
    "\n",
    "\n",
    "def train_loop(model, dataloader, loss_fn, optimizer, step=0.05, history_loss=None):\n",
    "    out = display(IPython.display.Pretty('Learning...'), display_id=True)\n",
    "    \n",
    "    size = len(dataloader.dataset) \n",
    "    len_size = len(str(size))\n",
    "    batches = ceil(size / dataloader.batch_size) - 1\n",
    "        \n",
    "    train_loss = []\n",
    "    percentage = 0\n",
    "    for batch, (X, y) in enumerate(tqdm(dataloader, leave=False, desc=\"Batch #\")):\n",
    "        X, y = X.to(device), y.to(device)\n",
    "        # evaluate\n",
    "        output = F.softmax(model(X).logits, dim=1)\n",
    "        loss = loss_fn(output, y)\n",
    "        train_loss.append(loss.item())\n",
    "        \n",
    "        # backpropagation\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        # print info\n",
    "        if batch / batches > percentage or batch == batches: \n",
    "            out.update(f'[{int(percentage * size)}/{size}] Loss: {train_loss[-1]:>8f}')\n",
    "            percentage += step\n",
    "        \n",
    "    if history_loss is not None:\n",
    "        history_loss.append(np.mean(train_loss))\n",
    "\n",
    "    return {'train_loss': np.mean(train_loss)}\n",
    "        \n",
    "def test_loop(model, dataloader, loss_fn, history_loss=None, history_acc=None):\n",
    "\n",
    "    size = len(dataloader.dataset)\n",
    "    test_loss, correct = 0, 0\n",
    "    batches = ceil(size / dataloader.batch_size)\n",
    "\n",
    "    val_loss = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        # evalute and check predictions\n",
    "        for batch, (X, y) in enumerate(tqdm(dataloader, leave=False, desc='Batch #')):\n",
    "            X, y = X.to(device), y.to(device)\n",
    "            output = F.softmax(model(X).logits, dim=1)\n",
    "            loss = loss_fn(output, y)\n",
    "            test_loss += loss.item()\n",
    "                \n",
    "            val_loss.append(loss.item())\n",
    "            \n",
    "    test_loss /= batches\n",
    "    correct /= size\n",
    "    \n",
    "    print(f\"Validation accuracy: {(100*correct):>0.1f}%, Validation loss: {test_loss:>8f} \\n\")\n",
    "    \n",
    "    if history_loss is not None:\n",
    "        history_loss.append(np.mean(val_loss))\n",
    "    \n",
    "    return {'val_loss': np.mean(val_loss)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "eihlww9xq7sp8o0j4al6ir"
   },
   "source": [
    "И натренируем нашу базовую модель."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "cellId": "vspefy8lcknk40w5yxt",
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea9879f7935b4807a7d5bd38c183a3ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=483.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7227056dcac4587b0eac589ee7b0f51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=267967963.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'pre_classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch 1\n",
      "-------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Learning..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "907aa42b0a7746b0898b9bd46a9e0cf2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batch #'), FloatProgress(value=0.0, max=289.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[0/36900] Loss: 0.038880'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[1845/36900] Loss: 0.034217'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[3690/36900] Loss: 0.029051'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[5535/36900] Loss: 0.021646'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[7380/36900] Loss: 0.022256'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[9225/36900] Loss: 0.023418'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[11070/36900] Loss: 0.021545'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[12915/36900] Loss: 0.021974'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[14759/36900] Loss: 0.021328'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[16605/36900] Loss: 0.020796'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-fabbf86ed4e8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Epoch {epoch+1}\\n-------------------------------\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mtrain_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbase_train_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m     \u001b[0mtest_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbase_test_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-1-09a3ee852cb4>\u001b[0m in \u001b[0;36mtrain_loop\u001b[0;34m(model, dataloader, loss_fn, optimizer, step, history_loss)\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mtrain_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0;31m# backpropagation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#!g1.1\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from IPython.display import clear_output\n",
    "\n",
    "classifier = DistilBertForSequenceClassification.from_pretrained(base_model_name, num_labels=num_tags).to('cuda')\n",
    "for param in classifier.base_model.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "classifier.train()\n",
    "loss_fn = F.binary_cross_entropy\n",
    "optimizer = torch.optim.Adam(classifier.parameters(), lr=1e-4)\n",
    "epochs = 30\n",
    "\n",
    "train_loss = []\n",
    "val_loss = []\n",
    "    \n",
    "for epoch in range(epochs):\n",
    "    print(f\"Epoch {epoch+1}\\n-------------------------------\")\n",
    "    \n",
    "    train_loop(classifier, base_train_dl, loss_fn, optimizer, history_loss=train_loss)\n",
    "    test_loop(classifier, base_test_dl, loss_fn, history_loss=val_loss)\n",
    "    \n",
    "    clear_output()\n",
    "    plot_learning_process(train_loss, val_loss)"
   ]
  },
  {
   "attachments": {
    "72b0e3ec-070b-4ffe-a261-f72c4a5740fe.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAgAElEQVR4nO3deXxU5b3H8c9kMlkmkz1kIQsJCTthC5uASEAExAVXRL3a2qu2tXVpbbW1rXpdatVeW/W6XJe6gZHiVURBBE1EAdl3EsAgJAGSQCCBkIQkk7l/HGRRIDnZJpn5vl+veQVmzpz8nonmy3POs1gAFyIiIk3k4+4CRESkc1FwiIiIKQoOERExRcEhIiKmKDhERMQUBYeIiJii4BAREVMUHCIiYoqCQ0RETFFwiIiIKQoOERExRcEhIiKmKDhERMQUBYeIiJii4BAREVMUHCIiYoqCQ0RETFFwiIiIKQoOERExRcEhIiKmKDhERMQUX3cX0B4iIyNJTk5u9LijR48SFBTUDhW1PU9qC6g9HZkntQU8qz0tbcuaNWvO+prL0x8ZGRmupsjOzm7ScZ2BJ7XF5VJ7OjJPaovL5VntaWlbzvY7VZeqRETEFAWHiIiYouAQERFTvOLmuIiIWXV1dRQVFVFTU+PuUpotNDSU3NzcRo8LCAggISEBm83WpPMqOEREzqCoqIjg4GCSk5OxWCzuLqdZjhw5QnBw8DmPcblclJWVUVRUREpKSpPOa8G4S+7RYmJiyMrKavS4yspKHA5HO1TU9jypLaD2dGSe1BY42Z7Q0FBSU1M7bWgAOJ1OrFZro8e5XC7y8/OpqKg47fnMzMyzv8fTHxqO2/mpPR2XJ7XF5TrZnq1bt7q5kpY7fPhwk489U3vP9jtVN8fP4a3lu/how153lyEi0qEoOM7hvVWFfLC2yN1liIgXKi8v54UXXjD9vosvvpjy8vI2qOgkBcc5JIQHUnSo2t1liIgXOltw1NfXn/N98+fPJywsrK3KAjSq6pwSwu0s2X4Al8vVqW+QiUjnc//995Ofn8+gQYOw2WwEBAQQHh5OXl4e27dvZ9q0aRQWFlJTU8Ndd93FbbfdBkBycjKrV6+msrKSSZMmMXbsWJYtW0Z8fDxz584lMDCwxbUpOM4hITyQ6jonB4/WEunwd3c5IuImD8/bwta9h1v1nH27hvDgpf3O+voTTzzB5s2bWb9+PTk5OUydOpXNmzefGDL7+uuvExERQXV1NcOGDeOqq64iMjLytHPk5+fz3nvv8corr3Dttdfy/vvvc+ONN7a4dl2qOoeEcDuALleJiNsNHz78tHkWzz77LAMHDmTkyJEUFhayY8eOH72nW7duDBo0CICMjAx27drVKrWox3EOCeFGl67oUDUDE9v2mqGIdFzn6hm0l1OXR8/JyWHx4sUsX74cu93OuHHjzjjD3d//5JUSq9VKdXXr/CNYPY5ziD8RHFVurkREvE1wcDBHjhw542sVFRWEh4djt9vJy8vjm2++adfa1OM4h5AAG6GBNl2qEpF2FxkZyejRo+nfvz+BgYHExMSceG3y5Mm89NJL9OnTh169ejFy5Mh2rU3B0QhjSK56HCLS/mbNmnXG5/39/VmwYMEZX/v+PkZUVBQrVqw48fy9997banXpUlUjNJdDROR0Co5GJITbKTpUjbFsi4iIKDgacepcDhERUXA0SnM5REROp+BoxKlzOURERMHRKM3lEBE5nYKjEZrLISKdQXvuwqjgaALN5RAROUkTAJsgITyQnfuPursMEfEi999/P4mJidxxxx0APPTQQ/j6+pKdnc2hQ4eoq6vj0Ucf5fLLL2/32hQcTaB9OUS83IL7oXhT654zNh2mPHHWl6dPn87dd999Ijhmz57NwoULufPOOwkJCeHAgQOMHDmSyy67rN1/Lyk4mkD7cohIexs8eDClpaXs3buX/fv3Ex4eTmxsLPfccw9LlizBx8eHPXv2UFJSQmxsbLvWpuBogsRT5nIoOES80Dl6Bm3pmmuuYc6cORQXFzN9+nRmzpzJ/v37WbNmDTabjeTk5DMup97WdHO8CRIijCG5hbpBLiLtaPr06WRlZTFnzhyuueYaKioqiI6OxmazkZ2dze7du91Sl3ocTRAfpkmAItL++vXrx5EjR4iPjycuLo4bbriBSy+9lPT0dIYOHUrv3r3dUpeCowmCA2yE2W0akisi7W7TppM35aOioli+fPkZj6usrGyvknSpqqm0vLqIiEHB0UQJYXYFh4gICo4m+372uPblEPEe3vL/u9l2WgCP/2RiYmLIyspq9LjKysqzrveyaHcdM3NreTbTToh/x58EeK62dEZqT8flSW2Bk+1xOBzExMQQGhraaSf+Op1OrFbrOY9xuVxUVFRQUlLyo/skmZmZZ3yPV9wcT0hIYNy4cY0el5OTc9bj6reWMDN3NUl9BzMoMay1S2x152pLZ6T2dFye1BY42Z66ujqKiorYs2ePu0tqtpqaGgICAho9LiAggIEDB2Kz2Zp0Xq8Ijtbw/VyOokNVnSI4RKRlbDYbKSkp7i6jRXJychg8eHCrn1f3OJpIczlERAwKjibSXA4REYOCwwTN5RARUXCYorkcIiIKDlM0l0NERMFhSkJ4IDV1DZQdrXV3KSIibqPgMCHhlH05RES8lYLDhFPncoiIeCsFhwmayyEiouAwRXM5REQUHKYlhmtIroh4NwWHSZoEKCLeTsFhkuZyiIi3U3CYlBBup6augQOVmsshIt5JwWFSQriG5IqId1NwmKRJgCLi7RQcJsWHay6HiHg3BYdJDn9fwjWXQ0S8mIKjGRI0l0NEvJiCoxm+H5IrIuKNFBzN8P0kQM3lEBFvpOBohoRwO8fqNZdDRLyTgqMZNJdDRLyZgqMZNJdDRLyZgqMZNJdDRLyZgqMZNJdDRLyZgqOZNJdDRLyVBfD4MaUxMTFkZWU1elxlZSUOh6NJ53x+XQ17Khv46/n2lpbXJsy0pTNQezouT2oLeFZ7WtqWzMzMs77m8vRHRkaGqymys7ObdJzL5XI9+vEWV88H5rsaGhqa/J72ZKYtnYHa03F5UltcLs9qT0vbcrbfqbpU1UyayyEi3krB0UyayyEi3krB0UyJEZrLISLeScHRTPFhmsshIt5JwdFMQf6+RAT56VKViHgdBUcLfL9KroiIN1FwtEBCeCCF6nGIiJdRcJjlrIfyQsAYkrtH+3KIiJdRcJiVNQPenQEYPY5j9Q3srzzm5qJERNqPgsOs5DFQsgkqik6Zy6H7HCLiPRQcZvWcbHzd/qn25RARr6TgMCuqJ4SnwLZPT5nLoRvkIuI9FBxmWSzQawp8t4Qgy7HjcznU4xAR76HgaI6ek8B5DHbmaC6HiHgdBUdzJI0C/xDYtuB4cOhSlYh4DwVHc/j6QdoE2PEZCWEBmsshIl5FwdFcPSdDZQkDrd9pLoeIeBUFR3P1uAgsPvQ9sgzQkFwR8R4KjuayR0DiCOJKvgQUHCLiPRQcLdFzEgEHNhNLmW6Qi4jXUHC0RM8pAFwauFE9DhHxGgqOlujSC8KTuch3vYJDRLyGgqMlLBboOZmB9RvYf/CQu6sREWkXCo6W6jkZP1ct3cpXUlPndHc1IiJtTsHRUt1GU+/rYCxr+CKv1N3ViIi0OQVHS/n6Ye0xgYm+65m7ttDd1YiItDkFRyuw9JpCFw6xf8dKKqrq3F2OiEibUnC0hh4X4cLCWNawYPM+d1cjItKmLIDHr84XExNDVlZWo8dVVlbicDia9T0Gr72PfYePcY/9r9w3PLBZ52hNLWlLR6T2dFye1BbwrPa0tC2ZmZlnfc3l6Y+MjAxXU2RnZzfpuDNa8neX68EQ14j733LtK69u/nlaSYva0gGpPR2XJ7XF5fKs9rS0LWf7napLVa2llzGLPNNnHfM27HVzMSIibUfB0Vq69IawJK6wb+LD9XvcXY2ISJtRcLQWiwV6TmGIcwP5e/fzbekRd1ckItImFBytqddkfBuOMca6hbnrdblKRDyTgqM1dRsNfg6uD9vK3PV7tZ2siHgkBUdr8vWH1PGcV7+agoNHWVdY7u6KRERanYKjtfWaQuCxUgb5FvCRLleJiAdScLS2tImAhf+MzuPjjXupdza4uyIRkVal4Ghtji6QMIzzXas5UFnL0vwyd1ckItKqFBxtoddkQg9tITXgMHPXaU6HiHgWBUdb6DkZgF/G57NwSzHVtdrgSUQ8h4KjLUT3hdAkMi1rOVrrZHFuibsrEhFpNQqOtmCxQK/JhBcvo1uwRZMBRcSjKDjaSt9pWOqrua/rOr7cXkp5Va27KxIRaRUKjrbSbRTEZ3DhwSwanPXM31Ts7opERFqFgqOtWCww5h78jhTwk7ANWjFXRDyGgqMt9ZoKkT34ufUjVn5Xxt7yandXJCLSYgqOtuTjA2PupsvR7Yz12chH2uBJRDyAgqOtpV8LwV35XdACja4SEY+g4Ghrvn4w6lek120koHgN20u0wZOIdG4KjvYw5GYaAsL4pW0ec3WTXEQ6OQVHe/B34DPidib6rGbD2hXa4ElEOjUFR3sZfjv11gCmVc1huVbMFZFOTMHRXoIiYfBNTLMu5eV5X2qfDhHptBQc7ch3zK/xsVi4oGw2M1cUuLscEZFmUXC0p7AkLAOu5npbNq9+tooDlcfcXZGIiGkKjnZmGX0PAa5jXONcwBML8txdjoiIaQqO9hbdG3pN5Vb/Rcxf8y1rdh9yd0UiIqZYAI8fGxoTE0NWVlajx1VWVuJwONq8npCKbQxZ93ue5j/4yDaVh0YF4GOxtOr3aK+2tBe1p+PypLaAZ7WnpW3JzMw862suT39kZGS4miI7O7tJx7WK1y92VT3R05V234euN5d91+qnb9e2tAO1p+PypLa4XJ7Vnpa25Wy/U3Wpyl3G3ENgdTH3xm3g6YXbdKNcRDoNBYe7pE2AmHR+6ppLTV0df9ONchHpJBQc7mKxwJi78Sv/lsf7FvHvNUW6US4inYKCw536ToPwZK6ofI/YYH/+MnczzgaPH6sgIp2cgsOdrL4w6k6s+9by4oDtbNlbwawVu91dlYjIOSk43G3QDRCbzuA1f2Be6NN8uHARZbpRLiIdmILD3WwBcGs2THmSvuxktut3bH/9Vjh6wN2ViYickYKjI7DaYMTtWO9az9rYaxhWNo/6fw6CZc9Dfa27qxMROY2CoyOxR9Dnlhe50e8frHf1hM8egBdGQN580OZPItJBKDg6GIe/LzdcMpGrj/yWTwc+Bz6+kDUD3p4GJVvcXZ6IiIKjI7pkQBwX9onhFysj+fC82TDlSdi7Hl4aA0uecnd5IuLlFBwdkMVi4fnrBzMyJZLfzNnKJ4GXwZ3roPclkP047N/m7hJFxIspODqoAJuV134ylIxu4dyVtY7PvquFS/4Bfg5Y/LC7yxMRL6bg6MDsfr68/pNh9I8P5Y5Za8kucsLou2DbJ1DwjbvLExEvpeDo4IIDbLx5y3B6xQbz87fXsDx6OjhiYdFfNNJKRNxCwdEJhAbaePuWEaREBXHLzC18l34nFK6AvE/cXZqIeCEFRycRHuTH2z8bQdewAC5fmkxNaCp8/jA4691dmoh4GQVHJ9Il2J9Zt44kItjO7yuuhAPbYd3b7i5LRLyMgqOTiQkJYNatI1kbcB7r6UXdF49D7VF3lyUiXkTB0Ql1DQvk3dvO4wXbzdiqSilb/A93lyQiXkTB0UklRti5/7abybYMx3/lc3y1PtfdJYmIl1BwdGLduzhIu+5JAjnGjjkP8eDczdTUOd1dloh4OAVHJ5fYazCuQTdys20xX3yzksufX0pe8WF3lyUiHkzB4QF8x/8Rq9XGnJ6fU3a0lsueX8qiXXW4NEFQRNqAgsMThMTBeb8kZvfHLJoRypi0KGbm1fLTN1ax/4i2oRWR1qXg8BSj74LACMKXPsprNw/lxj5+LMsvY8o/l5C9rdTd1YmIB1FweIqAUBj7O9iZgyX/Cy7sZmPer8YQGeTPT/+1ioc+2qIb5yLSKhQcnmTYzyAsCRY/CK4GesUGM/dXo/nJqGTeWLaLaf+zlGX5B9xdpYh0cgoOT+LrD+P/AsWbiC5dAhj7ejx0WT/+9ZNhlFfVcf0rK5jxv9+wetdBNxcrIp2VBfD4oTcxMTFkZWU1elxlZSUOh6MdKmpDrgYy1vwW32OH2JN0JeVh/ah0JIPFSq3TRU5hPR/vrONwrYv0KCtX9rCREmp1d9WN8oifzSk8qT2e1BbwrPa0tC2ZmZlnfN4rgiMjI4PVq1c3elxOTg7jxo1rh4raWNFqqt+5gcCaYuPv/iGQOAKSR0O30VRF9eetlft46ct8yqvqmNg3ht9M7EmfuBD31n0OHvOzOc6T2uNJbQHPak9L22KxWM74vG+zzygdV8JQVox8mXGDe0DBctj1NexeBosXAWD3DeTnicP46aiRfFzRnYc3+jBlawlTB8Rxz4U9SIsOdnMDRKQjU3B4stB4SL/aeABU7jeCZPcy2L0U/6+f4ipcXJY6kRei/sjL35SyYNM+pg2K564Le9AtMsi99YtIh6Tg8CaOLtD3MuMBUF0OG97FtvAB7jq6j5t+/hYvrDvGW8t389GGvcwYnsSvJ6QRHRzg3rpFpEPRqCpvFhgGI38BN86B8kLCZ07mgYHVLPl9JtOHJTJrZQEXPJnD0wu3cbimzt3VikgHoeAQSB0PP/sMbIHwxsXEFH7KY1eks/g3F3Bh3xiez/6WsU9m88qSnZpEKCIKDjkuujfc+gXEDYJ/3wxLniYl0s5zMwbz8a/HMCAhjMfm55L5dA7vrSqg3tng7opFxE0UHHJSUBTcNBfSr4UvHoEPfwn1x+gfH8pbtwxn1q0jiAkJ4L73NzHpH0v4dPM+rcAr4oV0c1xOZwuAK/8XItMg53E4tAumvwNBkYxKjeKDX0by2dYSnlq4jZ+/s5bescFcPiieqelxJEXa3V29iLQD9TjkxywWGHcfXPUa7FkDr06AAzuOv2RhUr9YFt49lqeuHkCAzcrfPs1j7FPZXPrc17z0ZT6FB6vc3AARaUvqccjZpV9tLJr47gwjPK58FXpeBIDVx8I1QxO5ZmgihQerWLB5H59s3McTC/J4YkEeAxNCmTogjovT40gIV09ExJOoxyHnljjcuGke3BVmXQMzr4X9204/JMLObWNTmfurMXz1+0z+MKU3LuDx+XmM+Vs2l//PUl5ZslM9EREPoR6HNC68G9yWAytfhiVPwwvnwdCfwrg/GDfUT5EYYef2C1K5/YJUCsqq+GTTPj7ZtJfH5ufy2Pxc+saFMLl/LJP7x9Ij2nHWtXBEpONScEjT2AKMXQYH3QA5T8Dq12HjbDj/tzDi58brP5AUaecX41L5xTgjRBZuKebTLcX896Lt/Pei7aREBTGpnxEiA+JD8fFRiIh0BgoOMScoCqY+DcNvhUV/MTaNWvUaTHwI+l1p3Fg/g6RIO7eO7c6tY7tTeriGz7aWsHBLMa9+tZOXvswnNiSASf1imNQ/luHJEfhadRVVpKNScEjzdOkF178HO3Ng4Z9gzi3wzYsw6XHjvsg5RIcEcOPIbtw4shsVVXV8nlfCp5uLyVpVyJvLdxMZ5MdF/WK5ZEAcI1IUIiIdjYJDWqb7OLj9S9jwLnz+CLw2EfpcZly+6jbqrD2Q74XabVw5JIErhyRQVVvPl9v288mmfcxdv4d3VxYQEeTHpH6xJDQ4GeNsUIiIdAAKDmk5HysMvhH6ToNlz8E3L0DuRxDVEzJ+AgNngD2i0dPY/XyZkh7HlPQ4qmudfLm9lPkb97Bj/Vfk1Dp4LfdzJvWLYWp6V0Z2V09ExF0UHNJ6/B2Q+QcYfSds+QDWvAEL/wiLH4a+lxsh0lgvxOWCgzsJ3JnN5J05TC5YAj4VOAOsfOOYwoPrJ/HuykIigvwY3zua0WmRnNc9ithQLf0u0l4UHNL6/IKMHsjgG6F4M6x9Eza8B5tmn7kXcrQMvvsSdmZDfg5UFBjPhyYal71SxrJvxYeMLv6MRb6fUpR6Ja9aruDDrSXMWVMEQPeoIM5LNZZFGdk9gkiHv5saL+L5FBzStmL7w8VPwYUPH++F/OtkL6THRKgohH0bARf4h0LK+UaPJXU8RHQ/0Tv59mA0CdOfxvL1MySueYOHXf/Hg4NvYEfP21hSamf5zjI+XLeHmSuM0OkdG3wiSIanRBAaaHPjhyDiWRQc0j787DD4BuPxfS8kdx5EpELmA5CaaSzpbj3Hf5IhXY0QGnMPfP0MPmveoNf6mfQadAO3Xv5b6kIy2LSnguX5ZSzPL2PWigL+tXQXPhYY2i2C8X2imdA7mjRNPBRpEQWHtL/veyEXP9W89/8gQFjzBqyfiW3QDQwZcw9DMtO4IzONY/VO1heU8/W3B/g8t/TEOlqJEYFM6B3D+N7RjOgegb+vtXXbJ+LhFBzSeZ0pQNa+CTH9Ie1C/HtMZES3EYzoHslvL+rFvopqsvP280VeCVmrCnhj2S7sflbGpEUxoU80mb2iiQ455SZ77VEo2QKB4cYy8+qliAAKDvEEpwbIxtnw7WJY/jws/Qf4h0D3CyBtInE9JnL9iCSuH5FETZ2T5fllfJ5Xwhe5pXy5tZA+lgLGhxRxXmAhPep3EHp0JxbX8Z0OQ5MgbTykTjDOFxDa9u0q2QqVJWCPNGbs2yPBVzf9xf0UHOI5QrrCmLuNR81hY6TWjkVGkOTOM46J7gc9LiSg2xgyK4vI9FmHK3wd1OViaaiHY1B2LJQ1zu5sck0j35rGkIgazrdsJHnDv/Fd8wZYrMbs+NQJRpjEDQafVpxT0tAAX/8dvngM+MEOi37BEBQJ9qhTAiUCkkZB74tbrwaRc1BwiGcKCIE+lxoPlwtKc+HbRUaQLH8Blv7TOC4wAkvXwTB6EnQdDF0HExEcR0pZFeUF5ewvOMTsgnIe2TcSH9ctDLZ8y7TgrYwt3URiwaOQ/SiuwEgsqZnHv99lLQuR6kPwf7fDjoWQfo0xdLnqIFQdgKoyY+jy938+ss+4lFZ1wJh4efXr0P+q1vn8RM5BwSGez2KBmL7GY/RdcOwI7F0HYd2Mjap+cO/CAnTv4qB7FwdXZSQAUHmsno2F5awt6MvnBWN5pqgcV81+xvhs4gLnRsZtWUzE5jkUhwxg3+hHSO4/ivAgP3N17l0Ps2+Cw3vh4qdh2H827b5K/TF463Jjj/iwbpAw1Nz3FTFJwSHexz8YUsaaeovD35dRaVGMSjP2H3G5XJQeOcaWvRPYvOcwfyg6RFLhR9xW8SYD508ja14m7wTdTHx8An3iQugbF0zv2BCSIuxnXj5+7Vvwyb3GpaefLoDEYU0vztcfps+EV8fDu9cZG2+FJZlq34/s2wDhKUbPTeQHLPzoIqrniYmJISsrq9HjKisrcTgc7VBR2/OktkDnaU9NVSWx+Vmkl31CtSWQF7mWl6vHU48x5NffCgkOH2IDnaRG+JMcVMfE0tdILFnMwfCB5Pb5LXV+zbvxbj9ayJC191ETEMW6wU/g9G3Glr0uF4mFH5C6800OhQ1gw8D/arTX01l+Nk3lSe1paVsyMzPP+LxXBEdGRgarV69u9LicnBzGjRvXDhW1PU9qC3TC9pTmwYLfw3df0hDdj53DHmK1qzd5xUfYuu8wmwsPEuks4UXbP+nvs4s3fK/hi9hbiI8IJiE88JSHnS4O/6ZvcpWfDe9cBWkTYEaWsQBlUzU4jZpXvQpd+sD+XLjiZRh43Tnf1ul+No3wpPa0tC1nmyirS1UibSG6N9w0F3I/wmfhA6R9cg1p/a+Gix6BkH5snPMk/Xc8j7PBxbye/2BV3UAqDlaxZUsxZUdrTzuVn9WHrmEBJITbSQgPpEdMMMOSw+kbF/LjFYJTM42hyZ/8Bj77E0z+a9Pqra2C938G2+bDqDthwoPwrynw6R8gbaIxkkvkOAWHSFuxWIxVgdMmGnNKvv4HbFsAPSYyYOuHEDsAn+lvc2l4Mpee8raq2nr2HKqmqLyaokPVFB2qouhQNXsOVbM4t4SsVYUABPlZGdItnGHJEQxLjmBwUhgBNisM+xmUfWssbx+ZZvz9XCr3w7vTjQEDFx/f3RHg0n/Cy+cbAXTFi23zGUmnpOAQaWt+dsj8Iwy6HhY+AFs/ZG/cRLre8s4Z92q3+/nSIyaYHjHBZzxdcUUNK3cdZNV3B1m16yDPLN6OywU2q4X0+FCGpUQwPOnXjCndgd/832GJSDEWjTyTA9/CzKvgSAlMfwd6Tz352vej0L76u3G5qvsFrfFpiAdQcIi0l/BkuG4mVJayffVWup4hNJoiNjSAywZ25bKBXQGoqKpj9e6DJ8Lk9a+/42WniyCuY45fHvFv3cDNPo9R6p+Mw9+XIH8rQf6+pDfkcUfxn8Fi4ethr5IYcj49f7jL4tjfGasaf3w3/GIZ2AJb45M4s9qjUFkK1QeNZWM0S77DUnCItDdHNLC11U4XarcxoU8ME/rEAFBd62R9YTl5xYdZWfECV627mdcsT/FM4ouUOO1U1TrpX5HDXYefpsQSyW3O+8nLtkD2VwTarKTHhzIwMZRBieEMTAwlfuozWN6+HJY8DRP+3LwinfXGTP7De4xlVCpLf/y1tvLk8bEDYMa7EJrQCp+QtDYFh4iHCfSzcl5qJOelRgIpMODf8MZUHjn2hHHDftVrsPCvkDCMpBlZLLBHsLusig1F5awrKGdDUTlvLtvNK87vAIhy+POsYyIjvn6GNY5MuvYcQnxYYNOXpq+tgjk/he2fnnwuIAwcMUaIdh1ifHVEG885a+GzP8P/ZsJ1s8zNaZF2oeAQ8XQJQ2Hai8Yv75fGwIHt0PsSuOpVsAViAZKjgkiOCuLyQfEA1NY3kLvvMBuKyllfUM6TBTfxesNyrJ/czfkfPIjdz0ZaTDA9ox34VNbhiiulZ0wwXUMDTg+UqoMwazoUrYJJfzWWZQnqcsZ7O6dJHGlMZnxjKlz2bKNDgqV9KThEvEH/K42RVtmPwYhfwKTHzjnHw8/Xh4GJYQxMDOOm8wAGUbXyMBnz7yBrSC7z/S9mR2kl2dv2c6Cylve2rQKMGfZp0Q7Soh30DzrMVVvvJKh6D9XT/oZ81t8AAAfxSURBVEXQoCuaXm90b2MG/Oyb4IPbjbXGJvzF3LwUaTMKDhFvMfZ3xl7vYYnNert92A2QO5sR+c8x4o7/gJD+AHz8WTbRPQayveQI35ZWsr3kCHu2rebeukdwcYzran/Pyiw/Qud+RnKknaTIILpF2EmKtJMUYSc00Ibdz0qgnxW7ny92m9WY8GiPgP/4ABbcZwxn3r8NrnrFWDJG3ErBIeItLJZmh8aJ91/yDLw4yphhPv1tABx+FoanRDA8JcI4btdSePdhXAF2dk2ZzX+SxMSyKnYfPGrcSyksZ/6mfTgbzr5oRYDNB7ufL4E2K0H+l3Olw59bt79E2TPnkz3kWUK79iAh3E5iuJ2QQF9tBdzOFBwi0nSRqXDB7+Hz/4K8+T/eAyR3Hsz5GYR3w3Lj+6SEJZFyhtPUORvYW15NwcEqKmvqqap1UlXnpLq2nqPHnFTXOamqrafqmJOqWidLa6exy9WVP1T+lYlLZ/CL2ntY4eoDQLC/L/HHl2dJCA8kMcJOcqSdFHsNCbX5+O3fCjUVMPw2cHRp+WdQV22MMNu7zpijYwsyhin7BYHNfvw5+8k/B4RBl97GfjEtDbjKUmMpfXskxKa7bVdKBYeImDPqTtg0B+bfCynnn3x+1WvGc/EZcP1s41LTWdisPnSLDKJbZJCJbzwC14EpNMyaTtahv7Jl8J/5Jvwy9h48Qn3pduwluXTZuYPUht308dlNjKX8xDsbsFC99GVW9PoddX2vpltUEInhdmzH56y4ji/Z53JBrdNFTZ3ztO/s7+tj9GoKvoG5dxj3i2LT4XA91B01Ro7VVRt/PpuAUGMjsZi+EN0XYvpBdJ8z7yZZVwMHthkhUbIFSjYbX4/uP3lMdF/j0uOAayE41sTn2HIKDhExx2ozliN57SJjl8KASZD9OHz5N+g5Ga7+l/Ev7TZgiUrDeuvnMOcW+q99kP5RM+HQbnAeA8Dl64czsicVIRPY5N+dbSSz7lg8FWXF3Hbo74zf+ic+3/Q+P6m7hWLOsf7Wok9P+2tMQAN/DnqfqUc/pCowjpIpM+k6ZIqxxMupXK7jAVJlPGqrjE23Srcaj5ItxvbGxw6ffE9oohECUT1Obs51YAe4joeXb4ARMD0nGRMjo/sawbXhXVj0Z1j8oLEywMAZxsz/tpykeZyCQ0TMSxxurIG18mX6R6yFshUw+Ea45J9gbeNfK4FhRo9myZOwZ40RVrHpENMPS1RPfK02IoFIIB24+vv3NVxL9dcvkLnkMb7yv59l3e9ic+wVYDl9ocjvvttJ9+7dAbBgIaZ8LWO3PkTU0SLedU3k0UPXcfQDCz4ffkpKVBB94kKOP4w9V+JCA7H8MDiTR5/8s8sFFYXGnvKlW45/3Qr5X0BInBEOfS41eiQx/SGi+49Hk3W/wPj8D+wwAmTDe8Yilf4h0G8aDLwekka24od+OgWHiDTPhL9A3idEla2A8++F8X9qv2vuVl9j/S8zfKwEjv019L8En3l3csH2x7mg9iuj9xSZeuKwHEsR48alGb2FLx6B9S8agwqu+4jpyWM572AVufsOk1t85MRcl4837jvx/iA/K6nRDtK6OEiNdtDj+PDkpAi7sZyLxWJstBWWBL0mn6zP5TL/+UX1MH4OmX+CXV8ZIbLpfWNjsPAUuoWOhJrBZ74c1gIKDhFpnoBQuH42m7+eT/8J97u7mqaLSIGbPjJ+uX72J3hxNIx/AEb+8uS/7Hcvh7m/hIM7jS18L3wY/B34cHKy5JT0uBOnPFxTx/biI+QVG0OS8/dXsiy/jP9bt+fEMX5WH5Kj7PSIDiYp0o6zwWUMAKh1Ul3rPPm17uRz1XVO4kIDGRAfSnpCKAMSQukdG4Kf7w+W0/fxMXoh3S8wVjjOnQcbZpFY8BH4/Herf4QKDhFpvrgBHOhy0N1VmGexQMbN0GMifHx875ItH8CUp0jb8SrkfGz0CG6e16RthkMCbAxNjmBo8ukDAo7U1JG//yjfllayo/QI+aWVbNlbwadbirFZLSeGHNv9rCfmsnRx+BvP+1kJsPlQcLCaz7YW895qYzl9P6sPveOCSY83giQ9PoyeMY6Ti1P6O2DQDBg0g+WLP+H8NrjfpOAQEe8V0tVYTHHz+8bclFfHkwDG0N0JDxq/hFsgOMDGoMQwBiWGnfa8y+UyNffE5XJRdKiajUUVbNxTzqaiCj5av5eZKwoAY9TX0ORw3vnZiNPO6/Q1M2qt6RQcIuLdLBZIvxq6j4Nlz7G+MopBF/+6jb+luXsZFouFxAg7iRF2pg4wLpE1NLjYfbCKjUXlbCyqoLa+od0mQio4REQAgqJg4sOU5+S4u5Im8fGxkBIVRMopi1O22/du1+8mIiKdnoJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUyzA2fdvFBER+QH1OERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERM+X/cJo73+3EcRgAAAABJRU5ErkJggg=="
    }
   },
   "cell_type": "markdown",
   "metadata": {
    "cellId": "a3sc3gned3uygxq5ywd529"
   },
   "source": [
    "Прошу прощения, исходный output ячейки потерялся, потому что я случайно перезапустил эту ячейку когда тренировал следующую версию модели. Тем не менее, у меня сохранился график обучения, который я прилагаю здесь.\n",
    "![image.png](attachment:72b0e3ec-070b-4ffe-a261-f72c4a5740fe.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "gkqtjsnon6biela1hxs3z"
   },
   "source": [
    "Сохраним ее чтобы иметь возможность использовать ее в интерфейсе."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "cellId": "pweno0g9r3w2a22ziovd"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "classifier.save_pretrained('./models')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "tiegy2nj0ccm3c8ec8a2jj"
   },
   "source": [
    "Теперь попробуем повторить историю со специализированной моделью той же архитктуры, но предобученной на научных статьях."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {
    "cellId": "my2w2e2vxpnb396jrppeoa"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "516dd347c0544191b3e448f1467ccb70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=385.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70064bb0aef5416db5980f8cca51fdae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=227845.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#!g1.1\n",
    "sci_model_name = 'allenai/scibert_scivocab_uncased'\n",
    "sci_tokenizer = AutoTokenizer.from_pretrained(sci_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {
    "cellId": "sclvuck179m0oe02zkimp5"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "sci_train_ds = ArXivDataset(titles_train, summaries_train, tags_train, possible_tags, sci_tokenizer)\n",
    "sci_test_ds = ArXivDataset(titles_test, summaries_test, tags_test, possible_tags, sci_tokenizer)\n",
    "\n",
    "sci_train_dl = DataLoader(sci_train_ds, shuffle=True, \n",
    "    batch_size=256, num_workers=0)\n",
    "sci_test_dl = DataLoader(sci_test_ds, shuffle=True, \n",
    "    batch_size=256, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {
    "cellId": "44aiscms3qxnuo03p2puv9"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1DklEQVR4nO3deXxV9Z3/8dcnuTf7RtYLBAFBBRIUZHHBKlWRpG6dUavWWqd1ht9SZ3S0M8VOx7YO/Y12puOMv9r6s9VWrXWp1pG6gBuxLqgsohBACQgStkAgkIXsn98f5yTchCw35N57kpvP8/E4j3vvueec+zkuvPl+v+d8j6gqxhhjTKjivC7AGGPM8GLBYYwxZkAsOIwxxgyIBYcxxpgBseAwxhgzIBYcxhhjBsSCw8Q0EdkuIhdH6bd+KyJLovFb4SYi80Wk0us6zPBgwWGMMWZALDiMMcYMiAWHGTFEJFFE/lNEdrvLf4pIovtdroi8KCI1InJQRN4WkTj3u++JyC4RqRWRT0Xkoj5+JldEXnO3fUtExrvHeEBEftatnqUi8ve91DrFPc5B9ze/FvTdb0XkwZ5+x/3+XBFZJSKH3ddzg77LFpHfuOd/SET+u9vv3iEiVSKyR0S+Ffo/XTOSWHCYkeSfgLOBGcAZwFzgB+53dwCVQB5QAHwfUBE5DbgFmKOq6cBCYHsfv3ED8C9ALrAOeMJd/yhwfVAY5QIXA7/vfgARSQVec7/LB64DfiEi0/r7HRHJBl4C7gdygP8AXhKRHHe/x4EUoMg99n1BxwwAmcBY4GbgAREZ1ce5mhHKgsOMJDcAd6tqlaruB34M3Oh+1wKMBsaraouqvq3ORG5tQCIwTUT8qrpdVbf28RsvqeqfVbUJJ6jOEZFxqvohcBjoaK1cB5Sp6r4ejnEZsF1Vf6Oqrar6EfAccE1/vwNcCmxR1cfdfZ8ENgOXi8hooBT4n6p6yD3Pt4KO2eL+82lR1ZeBOuC0fv+pmhHHgsOMJGOAHUGfd7jrAP4NqABeFZFtIrIYQFUrgNuAHwFVIvKUiIyhdzs73qhqHXAw6DceBb7hvv8Gzt/+ezIeOMvtNqsRkRqc0AuE8Dvdz7HjPMcC44CDqnqol9+tVtXWoM8NQFov25oRzILDjCS7cf5Q7nCSuw5VrVXVO1T1ZOAK4PaOsQxV/b2qnufuq8C9ffzGuI43IpIGZHf8BvA74EoROQOYCvx3L8fYCbylqllBS5qq/q8Qfqf7OXac5y73uNkiktVH/cb0y4LDjCRPAj8QkTx3jOEunD/MEZHLRGSyiAhOl1Ib0C4ip4nIhe4geiNwFGjv4ze+IiLniUgCzhjE+6q6E0BVK4FVOC2N51T1aC/HeBE4VURuFBG/u8wRkakh/M7L7r5fFxGfiFwLTANeVNU9wCs44yWj3OOeP9B/iMZYcJiRZAmwGvgEWA+sddcBnAK8jtOvvxL4haquwBnfuAc4AOzFGVC+s4/f+D3wQ5yuo1kc65rq8Cgwnd67qVDVWuASnHGQ3e7v3uvW0ufvqGo1zhjJHUA18I/AZap6wN3vRpyxjM1AFU43nDEDIvYgJ2Oix/0b/u9wBuFP6H8+EfktUKmqP+hvW2MiwVocxkSJiPiBW4Ffn2hoGDMUWHAYEwXu+EQNziW//+lpMcYMknVVGWOMGRBrcRhjjBkQn9cFRENubq5OmDDB6zJCUl9fT2pqqtdlRISd2/AVy+dn59a7NWvWHFDVvO7rR0RwTJgwgdWrV3tdRkjKysqYP3++12VEhJ3b8BXL52fn1jsR6T4LAWBdVcYYYwbIgsMYY8yARDQ4RKTEfZZARcekcd2+TxSRp93vPxCRCe76BSKyRkTWu68XBu2TICIPichnIrJZRK6K5DkYY4zpKmJjHCISDzwALMB5zsEqEVmqqhuDNrsZOKSqk0XkOpxpFa7Fmd7hclXdLSLFwHKc2T3BmUK6SlVPdZ9tkB2pczDGjFwtLS1UVlbS2NjodSknLDMzk02bNvW7XVJSEoWFhfj9/pCOG8nB8blAhapuAxCRp4ArgeDguBJnumqAZ4Gfi4i4zx/oUA4ki0ii++yBbwNTAFS1HSdkjDEmrCorK0lPT2fChAk4c18OP7W1taSnp/e5japSXV1NZWUlEydODOm4kQyOsQQ9MwCn1XFWb9uoaquIHMZ5allwGFwFrFXVpqDpoP9FROYDW4FbenoYjogsAhYBFBQUUFZWNsjTiY66urphU+tA2bkNX7F8fr2dW2ZmJjk5OdTV1UW/qDBpa2ujtra23+0SEhKoqakJ+d/xkL4cV0SKcLqvLnFX+YBC4D1VvV1Ebgf+nWNPceukqg8BDwHMnj1bh8vldnZp4PAUy+cGsX1+vZ3bpk2byMjIiH5BYRRKi6NDUlISM2fODGnbSA6O7yLoYTM4f+Dv6m0bEfHhPO+42v1cCDwPfDPoUZ3VOE8l+6P7+Q/AmZEoXlV5/P0dvPjJ7v43NsaYESSSwbEKOEVEJroPm7kOWNptm6XATe77q4E3VVXdLqmXgMWq+m7Hxu6Mon8C5rurLqLrmEnYiAh/WL2TX/15WyQOb4wxfaqpqeEXv/jFgPf7yle+Qk1NTfgLChKx4HCfXXwLzhVRm4BnVLVcRO4WkSvczR4GckSkArgd6Lhk9xZgMnCXiKxzl3z3u+8BPxKRT3C6qO6I1DmUFAf4uPIwu2p6e1CbMcZERm/B0dra2sPWx7z88stkZWVFqCpHRMc4VPVlnEdZBq+7K+h9I3BND/st4diT2bp/twOIyuMuS4oC/HTZpyzfsJdvnxfa1QbGGBMOixcvZuvWrcyYMQO/309SUhKjRo1i8+bNfPbZZ3z1q19l586dNDY2cuutt7Jo0SLg2BRLdXV1LFy4kPPPP5/33nuPsWPH8sILL5CcnDzo2ob04LjXTs5L47SCdJZZcBgzov34T+Vs3H0krMecNiaDH15e1Ov399xzDxs2bGDdunWUlZVx6aWXsmHDhs5LZh955BGys7M5evQoc+bM4aqrriInJ6fLMbZu3crTTz/Nr371K772ta/x3HPP8Y1vdH+a8cDZlCP9KCkOsGrHQapqh+9NQMaY4W/u3Lld7rO4//77OeOMMzj77LPZuXMnW7ZsOW6f8ePHM2PGDABmzZrF9u3bw1KLtTj6UVIc4L/e2MJrG/dxw1njvS7HGOOBvloG0RI8PXpZWRmvv/46K1euJCUlhfnz5/d4h3tiYmLn+/j4eI4eDc94rbU4+jElkM6EnBSWbdjrdSnGmBEkPT2915v3Dh8+zKhRo0hJSWHz5s28//77Ua3NWhz9EBFKikfz67e3UdPQTFZKgtclGWNGgJycHObNm0dxcTHJyckUFBR0fldSUsKDDz7I1KlTOe200zj77LOjWpsFRwhKigM8+NZWXt9UxdWzCr0uxxgzQvz+97/vcX1iYiKvvPJKj991jGPk5ubywQcfdK7/7ne/G7a6rKsqBGcUZjImM4llG/Z4XYoxxnjOgiMEIsLC4gB/3nKAuqa+b74xxphYZ8ERopKiAM2t7azYXOV1KcYY4ykLjhDNnpBNblqCXV1ljBnxLDhCFB8nXFIUYMWnVTS2tHldjjHGeMaCYwBKigI0NLfx58/2e12KMcZ4xoJjAM6ZlENmst+6q4wxQ05aWlrUfsuCYwD88XFcPLWA1zfto7m13etyjDHGExYcA1RSHOBIYysrt1V7XYoxJoYtXryYBx54oPPzj370I5YsWcJFF13EmWeeyfTp03nhhRc8qc3uHB+gL52SS2pCPMs27OGCU/O8LscYEw2vLIa968N7zMB0KL2n16+vvfZabrvtNr7zne8A8Mwzz7B8+XL+7u/+joyMDA4cOMDZZ5/NFVdcgYiEt7Z+WItjgJL88Xx5Sj6vlu+jrV29LscYE6NmzpxJVVUVu3fv5uOPP2bUqFEEAgG+//3vc/rpp3PxxReza9cu9u3bF/XarMVxAkqKA7z4yR5WbT/I2Sfn9L+DMWZ466NlEEnXXHMNzz77LHv37uXaa6/liSeeYP/+/axZswa/38+ECRN6nE490qzFcQK+fFo+ib44u7rKGBNR1157LU899RTPPvss11xzDYcPHyY/Px+/38+KFSvYsWOHJ3VZcJyA1EQf55+ax7INe2m37ipjTIQUFRVRW1vL2LFjGT16NDfccAOrV69m+vTpPPbYY0yZMsWTuqyr6gSVFAV4beM+Pq6sYeZJo7wuxxgTo9avPzYon5uby8qVK3vcrq6uLlolWYvjRF08tQBfnFh3lTFmxLHgOEGZKX7OnZzLsvK9qFp3lTFm5LDgGISSogA7qhvYtKfn5wIbY4a3kfKXwoGepwXHIFxSVECcYE8GNCYGJSUlUV1dHfPhoapUV1eTlJQU8j42OD4IuWmJzJmQzbLyvdx+yWlel2OMCaPCwkIqKyvZv3/4zobd2NgYUiAkJSVRWFgY8nEtOAappDjAj/+0ka3765iUF73ZKY0xkeX3+5k4caLXZQxKWVkZM2fODPtxratqkEqKAwB2dZUxZsSw4Bik0ZnJzBiXZcFhjBkxLDjCoKQ4wPpdh9l5sMHrUowxJuIsOMKg1O2uWl5urQ5jTOyz4AiD8TmpTB2dYd1VxpgRwYIjTEqKAqz54hBVR6I/xbExxkRTRINDREpE5FMRqRCRxT18nygiT7vffyAiE9z1C0RkjYisd18vDNqnzD3mOnfJj+Q5hKp0egBVWL4x+g9VMcaYaIpYcIhIPPAAUApMA64XkWndNrsZOKSqk4H7gHvd9QeAy1V1OnAT8Hi3/W5Q1RnuUhWpcxiIU/LTODk31e4iN8bEvEi2OOYCFaq6TVWbgaeAK7ttcyXwqPv+WeAiERFV/UhVd7vry4FkEUmMYK2DJiKUFAd4f9tBDtU3e12OMcZETCTvHB8L7Az6XAmc1ds2qtoqIoeBHJwWR4ergLWq2hS07jci0gY8ByzRHiaTEZFFwCKAgoICysrKBnc2IchvaqOtXfn582/xpUL/CR2jrq4uKrV6wc5t+Irl87NzG7ghPeWIiBThdF9dErT6BlXdJSLpOMFxI/BY931V9SHgIYDZs2fr/PnzI16vqvKrTSv4vDWdf54/54SOUVZWRjRq9YKd2/AVy+dn5zZwkeyq2gWMC/pc6K7rcRsR8QGZQLX7uRB4Hvimqm7t2EFVd7mvtcDvcbrEhoSO7qp3thygtrHF63KMMSYiIhkcq4BTRGSiiCQA1wFLu22zFGfwG+Bq4E1VVRHJAl4CFqvqux0bi4hPRHLd937gMmBDBM9hwEqLAzS3tfPm5iExZm+MMWEXseBQ1VbgFmA5sAl4RlXLReRuEbnC3exhIEdEKoDbgY5Ldm8BJgN3dbvsNhFYLiKfAOtwWiy/itQ5nIgzTxpFXnqi3QxojIlZER3jUNWXgZe7rbsr6H0jcE0P+y0BlvRy2FnhrDHc4uKEhUUFPLdmF0eb20hOiPe6JGOMCSu7czwCSotHc7Sljbc+G74PgDHGmN5YcETA3InZZKX47WZAY0xMsuCIAH98HAumFvDGpiqaWtu8LscYY8LKgiNCSqcHqG1q5b2t1V6XYowxYWXBESHzJueSluhj2Xq7usoYE1ssOCIk0RfPhVPyeXXjXlrb2r0uxxhjwsaCI4JKiwMcamjhw+0HvS7FGGPCxoIjgi44LY8kf5zdDGiMiSkWHBGUkuDjglPzWF6+l/b24ybwNcaYYcmCI8JKi0ez70gTH+2s8boUY4wJCwuOCPvylHz88WI3AxpjYoYFR4RlJvuZNzmXZeV76eF5U8YYM+xYcERBaXGAnQePUr77iNelGGPMoFlwRMHFUwuIE+zqKmNMTLDgiIKctETOmpjDsnILDmPM8GfBESWl0wNUVNVRUVXrdSnGGDMoFhxRcsm0AACv2NxVxphhzoIjSgKZSZx5UpZ1Vxljhj0LjigqLR5N+e4jfFHd4HUpxhhzwiw4oqik2OmuWlZuNwMaY4YvC44oGpedQtGYDLss1xgzrFlwRFlpcYC1X9Sw93Cj16UYY8wJseCIso7uquU2SG6MGaYsOKJscn46k/PTrLvKGDNsWXB4oLQ4wAefV1Nd1+R1KcYYM2D9BoeI5InI90XkIRF5pGOJRnGxamFRgHaF1zbu87oUY4wZMF8I27wAvA28DrRFtpyRoWhMBuOyk1lWvpfr5p7kdTnGGDMgoQRHiqp+L+KVjCAiQklRgN++t53DR1vITPZ7XZIxxoQslDGOF0XkKxGvZIQpKR5NS5vy5mbrrjLGDC+hBMetOOHRKCK17mJPJBqkmeOyKMhItKurjDHDTr/Boarpqhqnqknu+3RVzYhGcbEsLk5YWBTgrc/209Dc6nU5xhgTspAuxxWRK0Tk393lskgXNVKUFAdobGnnrU/3e12KMcaELJTLce/B6a7a6C63isi/RrqwkWDuhGyyUxN4xbqrjDHDSCgtjq8AC1T1EVV9BCgBLg3l4CJSIiKfikiFiCzu4ftEEXna/f4DEZngrl8gImtEZL37emEP+y4VkQ2h1DFU+eLjWDC1gDc3V9HUalc6G2OGh1DvHM8Kep8Zyg4iEg88AJQC04DrRWRat81uBg6p6mTgPuBed/0B4HJVnQ7cBDze7dh/CdSFWPuQVjI9QF1TK+9WHPC6FGOMCUkowfGvwEci8lsReRRYA/wkhP3mAhWquk1Vm4GngCu7bXMl8Kj7/lngIhERVf1IVXe768uBZBFJBBCRNOB2YEkINQx58yblkp7os0fKGmOGjX5vAFTVJ0WkDJjjrvqeqobyp9xYYGfQ50rgrN62UdVWETkM5OC0ODpcBaxV1Y6Jnf4F+BkQE4/RS/DFcdHUfF7btI+WtnavyzHGmH71GhwiMkVVN4vIme6qSvd1jIiMUdW1kS5ORIpwuq8ucT/PACap6t93jIf0se8iYBFAQUEBZWVlEa11MMZJKzUNLfzqv1dwUuLRIV3rYNTV1dm5DVOxfH52bgPXV4vjdpw/eH/Ww3cKHDdg3c0uYFzQ50J3XU/bVIqID2f8pBpARAqB54FvqupWd/tzgNkist2tPV9EylR1/nEFqj4EPAQwe/ZsnT//uE2GjLOa2/j1htfYHZ/PtLRqhnKtg1FWVmbnNkzF8vnZuQ1cr8Ghqovct6Wq2uVxdSKSFMKxVwGniMhEnIC4Dvh6t22W4gx+rwSuBt5UVRWRLOAlYLGqvhtU0y+BX7o1TABe7Ck0hpvkhHjmn5bH8vJ9XHhuvNflGGNMn0IZHH8vxHVdqGorcAuwHNgEPKOq5SJyt4hc4W72MJAjIhU4LZyOS3ZvASYDd4nIOnfJD6HWYaukOMD+2ia21tg4hzFmaOtrjCOAM3idLCIzAXG/ygBSQjm4qr4MvNxt3V1B7xuBa3rYbwn9XDWlqtuB4lDqGA4unJJPQnwcq/e28jdeF2OMMX3oa4xjIfBXOGMTP+NYcBwBvh/Zskae9CQ/552Sy+rt+1FVRKT/nYwxxgN9jXE8CjwqIlep6nNRrGnEKikO8ObmKjbsOsL0wpDuszTGmKgLZYxjljtYDYCIjBKRmLj5bqhZMLWAOIFXNuzxuhRjjOlVKMFRqqo1HR9U9RDO/FUmzEalJjAlO45lG/aiql6XY4wxPQolOOI7pvsAEJFkILGP7c0gzC7wse1APVuqYmIqLmNMDAolOJ4A3hCRm0XkZuA1js0vZcLszPx4RLC5q4wxQ1YoTwC8F2dSw6nu8i+q+tNIFzZSZSXFMeukUTbOYYwZsvqd5BBAVV8BXolwLcZVUhxgyUub2H6gngm5qV6XY4wxXYTyBMC/FJEtInJYRI6ISK2IHIlGcSNVSXEAgGXl1l1ljBl6Qhnj+ClwhapmqmqGqqarakakCxvJCkelMH1spj1S1hgzJIUSHPtUdVPEKzFdlBQH+HhnDbtrjnpdijHGdBFKcKx2nwt+vdtt9Zfuo1tNBJW63VXLrbvKGDPEhBIcGThP27sEuNxdLotkUQZOzkvj1II0lll3lTFmiAnl0bHfikYh5nglxaP5+ZtbOFDXRG6a3XNpjBka+g0OEfkNzhP/ulDVb0ekItOptDjA/W9s4dXyfXz9rJO8LscYY4DQuqpexHka30vAGzhdVzYfRhRMCaQzPifFLss1xgwpoXRVdZlSXUSeBN6JWEWmk4hQUhzg4bc/53BDC5kpfq9LMsaYkFoc3Z0CxPRjXIeSkqIAre3K65v2eV2KMcYAod05XuveMX7EvWP8T8D3Il+aATijMIvRmUnWXWWMGTL6eub4PFV9F8hznw1uPBAXJywsCvDkh19Q39RKamJI04sZY0zE9NXiuN99fS8ahZjelRQHaGptZ8WnVV6XYowxfQ6Ot4jIQ0ChiNzf/UtV/bvIlWWCzZmQTW5aAss27OWy08d4XY4xZoTrKzguAy4GFgJrolOO6Ul8nLBgWoCl63bR2NJGkj/e65KMMSNYr8GhqgeAp0Rkk6p+HMWaTA9Kip1xjre3HGDBtAKvyzHGjGChPAHQQmMIOOfkHDKSfDZ3lTHGcydyH4fxQIIvjounFfD6pn20tLV7XY4xZgSz4BhGSooCHD7awsqt1V6XYowZwUK5AfBWEckQx8MislZELolGcaar80/NIyUh3m4GNMZ4KpQWx7dV9QjO8zhGATcC90S0KtOjJH88X56Sz6vle2lrP27CYmOMiYpQgkPc168Aj6tqedA6E2UlRQEO1DWzevtBr0sxxoxQoQTHGhF5FSc4lotIOmCjsx758pR8Enxx1l1ljPFMKMFxM7AYmKOqDYAfsKcCeiQt0cf5p+SxfMNeVK27yhgTfaEExznAp6paIyLfAH4AHI5sWaYvJcUBdh9u5ONK+9dgjIm+UILjl0CDiJwB3AFsBR4L5eAiUiIin4pIhYgs7uH7RBF52v3+AxGZ4K5fICJrRGS9+3ph0D7LRORjESkXkQdFZMTNv7FgagG+OLGbAY0xngglOFrV6RO5Evi5qj4ApPe3k/sH+gNAKTANuF5EpnXb7GbgkKpOBu4D7nXXHwAuV9XpwE3A40H7fE1VzwCKgTzgmhDOIaZkpvg5Z1IOyzbsse4qY0zUhRIctSJyJ85luC+JSBzOOEd/5gIVqrpNVZuBp3DCJ9iVwKPu+2eBi0REVPUjVd3tri8HkkUkEcC9NBicebYSgBH5J2dJcYDt1Q18uq/W61KMMSNMKMFxLdCEcz/HXqAQ+LcQ9hsL7Az6XOmu63EbVW3FGTvJ6bbNVcBaVW3qWCEiy4EqoBYncEacS6YFEIFX1lt3lTEmuvp9nJyq7hWRJ4A5InIZ8KGqhjTGMVgiUoTTfdXlTnVVXSgiScATwIXAaz3suwhYBFBQUEBZWVnE6w2Hurq6kGs9NSuO5z6oYKZ/d/8bDwEDObfhJpbPDWL7/OzcToCq9rkAXwN24HQpPQZ8Dlwdwn7nAMuDPt8J3Nltm+XAOe57H87YhrifC4HPgHl9/MY3ccZd+qxl1qxZOlysWLEi5G0ffnubjv/ei7q1qjZyBYXRQM5tuInlc1ON7fOzc+sdsFp7+DM1lK6qf8K5h+MmVf0mztjFP4ew3yrgFBGZKCIJwHXA0m7bLMUZ/Aa4GnhTVVVEsoCXgMXqPPccABFJE5HR7nsfcCmwOYRaYlJJcQDAbgY0xkRVKMERp6rBD7uuDmU/dcYsbsFpVWwCnlHVchG5W0SucDd7GMgRkQrgdpwbDXH3mwzcJSLr3CUfSAWWisgnwDqccY4HQziHmDQmK5kzxmXZZbnGmKjqd4wDWOYORj/pfr4WeDmUg6vqy923VdW7gt430sPltKq6BFjSy2HnhPLbI0VJUYB7l21mV81RxmYle12OMWYECKXl8A/AQ8Dp7vKQqn4v0oWZ0JR2dFdZq8MYEyWhtDhQ1eeA5yJcizkBE3JTmRJIZ9mGPdx83kSvyzHGjAC9tjhEpFZEjvSw1IrIkd72M9FXUhxg9Y5DVNU2el2KMWYE6DU4VDVdVTN6WNJVNSOaRZq+lRaPRhVeLd/ndSnGmBHAnjkeA04tSOPk3FQb5zDGRIUFRwwQERYWB1i5rZqahmavyzHGxDgLjhhRWhygrV15baN1VxljIsuCI0ZMH5vJ2Kxk664yxkScBUeMEBEWFgV4e8sB6ppavS7HGBPDLDhiSOn0AM1t7by5uar/jY0x5gRZcMSQM08aRW5aIss27PG6FGNMDLPgiCHxccLCogJWbN5PY0ub1+UYY2KUBUeMKS0ezdGWNt76bL/XpRhjYpQFR4w56+RsMpP9dnWVMSZiLDhijD8+jgXTCnh90z6aW9u9LscYE4MsOGJQaXGA2sZW3tt6wOtSjDExyIIjBs2bnEtqQjzL7ZGyxpgIsOCIQUn+eC6cWsCr5ftoa1evyzHGxBgLjhhVWhygur6ZDz8/6HUpxpgYY8ERoy44NY9EX5x1Vxljws6CI0alJvq44NQ8lm3YS7t1VxljwsiCI4aVTg+w90gj6yprvC7FGBNDLDhi2IVTCvDHC8vtZkBjTBhZcMSwzGQ/507K5ZUNe1G17ipjTHhYcMS40uIAXxxsYOOeI16XYoyJERYcMW7BtALiBOuuMsaEjQVHjMtJS2TuxGxeseAwxoSJBccIUFo8mi1VdVRU1XldijEmBlhwjAALiwIAdjOgMSYsLDhGgEBmEjNPyuIVe6SsMSYMLDhGiNLiABt2HWHnwQavSzHGDHMWHCNESdFowLqrjDGDZ8ExQpyUk8K00Rl2dZUxZtAiGhwiUiIin4pIhYgs7uH7RBF52v3+AxGZ4K5fICJrRGS9+3qhuz5FRF4Skc0iUi4i90Sy/lhTWhxgzY5D7DvS6HUpxphhLGLBISLxwANAKTANuF5EpnXb7GbgkKpOBu4D7nXXHwAuV9XpwE3A40H7/LuqTgFmAvNEpDRS5xBVLY3w3N8w6uBaaI/Ms8JLip2rq1617ipjzCBEssUxF6hQ1W2q2gw8BVzZbZsrgUfd988CF4mIqOpHqrrbXV8OJItIoqo2qOoKAPeYa4HCCJ5D9Bz4DLa+yRmf/Bj+75nw7n9BfXVYf+KUgnQm5aVad5UxZlAiGRxjgZ1BnyvddT1uo6qtwGEgp9s2VwFrVbUpeKWIZAGXA2+Er2QPjT4dbt/Ixql3QPpoeO0u+I+p8MdF8MUHEKZJCkuLR/PB5wc5WN8cluMZY0Yen9cF9EVEinC6ry7ptt4HPAncr6rbetl3EbAIoKCggLKyssgWGyZ1qWdSVXA+qfk7GLN7GQXlS/F98jR1qRPYPaaEfQUX0OZLOeHj5za20dau/Pz5tzi/0B/GyvtXV1c3bP49DFQsnxvE9vnZuZ0AVY3IApwDLA/6fCdwZ7dtlgPnuO99OGMb4n4uBD4D5vVw7EdwQiOkWmbNmqXDxYoVK7quaKxVXfWI6i/nqf4wQ/UnY1T/9Peqezec0PHb29t13j1v6F898sHgix2g484thsTyuanG9vnZufUOWK09/Jkaya6qVcApIjJRRBKA64Cl3bZZijP4DXA18KaqqtsN9RKwWFXfDd5BRJYAmcBtEax96EhMg9nfgv/xNtz8Oky9HD76HfzyXHh4IXzyjDOwHiIRobQ4wLsV1RxpbIlg4caYWBWx4FBnzOIWnFbFJuAZVS0XkbtF5Ap3s4eBHBGpAG4HOi7ZvQWYDNwlIuvcJV9ECoF/wrlKa627/q8jdQ5DigiMmwN/8SDcsRkuWQL1VfDHv4H7psGr/wwHe+y1O05JcYDmtnZWbK6KcNHGmFgU0TEOVX0ZeLnburuC3jcC1/Sw3xJgSS+HlXDWOCylZMO5fwtnfwc+L4NVD8PKB+C9+2HSRTDnZjhlIcT3/K935rhR5Kcn8vL6PVw5o/v1CsYY07chPThu+hEXB5MudJYju2HNo7D2UXjq65BRCLNugjO/CemBbrsJJcUBHlu5g5l3v8qkvDQm56d1vk7OT2NsVjJxcZbRxpjjWXDEiowx8OU74fzvwqevwOqHYcVP4K17YcqlMPtmmHi+0+UF3HbxqYzPSaWiqo6tVXW8tnEfT9Ufu3o60RfHyXlpTMpL7QyTSXlpTMxNJckf79VZGmOGAAuOWBPvh2lXOEv1Vlj9iDOYvvEFyDkFZn8bZlxPduoobj5vYpddD9U3s3W/88Cniqo6tu6v4+PKGl5av6fzNpI4gXHZKUGtFDdY8tLJTInu5b3GGG9YcMSynEmw8Cdw4Q+g/HknRJbfCW/cDcVXwZxvw9hZnZuPSk1gdmo2sydkdzlMY0sb2/bXU7HfaZ10vL5TcYDm1mPTo+SmJTApL41J+WlM7giW/DTGZCYhYt1exsQKC46RwJ8MM77uLHs+dgLkkz/Aut/B6BnOYHrx1ZDQ842FSf54po3JYNqYjC7r29qVykMN3Vop9bz0yR4OHz12qW9KQjyT8tJIa29kQ/uWztbK+JxUEnw2QbMxw40Fx0gz+gy4/L9gwd3OPSCrfg1L/xaW/wBmXO90ZeWdFtKh4uOE8TmpjM9J5cIpBZ3rVZXq+uYuXV4VVXVs3NnOylc/67p/dgqTug3MT8pLJT3Jur2MGaosOEaqpEyY+zcw569hx3vOYPqqh+GDB2HCl5wAmXIZ+BIGfGgRITctkdy0RM4++djUY2VlZcw55zy27a/vDJOO17JPq2hpOzYfV0FGYrcwcV7z0xOt28sYj1lwjHQiMGGes9RVwUePw+rfwrPfgtR853LeWX8FWePC8nOpiT6mF2YyvTCzy/qWtnZ2Hmzo7O6qcMdSnl+7i9qm1s7t0hN9nOyOoUzKT+0cSzkpOwVfvHV7GRMNFhzmmLR8+NIdMO82qHjdaYG8/TN45z+cGwrn3OzcYBgX/j+g/fHO5b8n56V1Wa+qVNU2dQ7Kd7RS3qnYz3NrK4P2FybkpB672is/lYL0JLLTEshJTWRUit+CxRzT3gb7NsDODxlbuRm2tEDOZMg6CeLscvP+WHCY48XFw6kLneXQDuemwrWPwWevQNZ4Z+6smTdCam7ESxERCjKSKMhI4tzJXX+vtrGls3XS0eX1WVUtr23aR1u7djsOZCX7yUlLJDs1gdy0BLJTnVDJccMleH1WSgLxdgNk7Giqg12r4Yv3naVyFTTXAXAKQMWvne3iE2DURCdEcia5r+6Slt95H9RIZ8Fh+jZqPFx0F1ywGDb/CVY9Aq//CFb8H5h2pXNj4Ulne/I/VHqSnxnjspgxLqvL+ubWdr442MCBuiaq65o5WN/EgbpmDtY3U13vrPtsXx3VdU3UHG3p8VEncQKjUhLI6QiYtERyUxPI7gyahC4h1B6m56WYMDmyB3a+7zzL5ouVsHc9aBsgUFAEp18LJ50DJ53Fux+uZd5p+VBd4S5bndeK16At6Lk1CenHh0nH56SMXkuJRRYcJjS+BOfej+KroGqzc0nvx0/C+j9A/jRnMP30a4fE/0AJvrjOQfX+tLa1c6ihxQmVuiaq3deD9c0cqG/mYJ0TNpt2H6G6vrnLZcbB4gRy3nvdDRQ3YDpbMMeHTUaSzwb5w6W9HQ586gRER1DU7HC+8yVD4Ww47++doCicDclZXXZvSdgG4891li7HbYPDO7uGSXUFVH4IG54Dgv6ykJoPuaccHyyjJoAvMZJn7wkLDjNw+VPgKz+Fi3/oBMeqh+Hl78JrP4TTv+aMhQSme11lSHzxceSlJ5KXngik97t9S1s7h+qbj2vBrN24hfTc/M716ytrqK5r7jKwH8wfL926yhK6daN1XZ+aEG9B06GlEXavPRYUOz+Axhrnu9Q8pwU8d5ETFIHpJ3RlIOB02Y6a4CyTLz6+hkOfB7VS3HD59BWo339sO4lzxk16aqVkFEZkvDAaLDjMiUtIda64OvMm2LXGCZCPn4Q1v4HCuU6ATPsq+JO8rjRs/PFx5GckkZ/R9ZxObt3B/PmnH7d9U2ub25ppprre6TbreN/ZsqlrZkd1A9V1TdQ3t/X4uwm+OKerzB2PyUj2k5oQT3JCPKkJPpIT4kkJep+aGE+y30dqorM+JcHX+TrsbrqsP+CEQ0dQ7P4I2t2WX+6pzvQ64852AiP75Oh0m/qTIH+qs3R3tAYObnWC5MCWY8GyYyW01B/bzpfk1Htc99dkSMkZ0uMpFhxm8EScLoDC2c4UJx8/6XRlPf8/YNmdMPMGmPUtr6v0RKIvntGZyYzOTA5p+8aWts5QqXYDp3vYVNc388XBBhqaW2loaqOhpe24iwH64ouTrmGSGE+K3+e8dguZFDeQKne2UPPRrmPrE7sFVYKPJH/c4FtFqs5zZb5YeSwoqrc438UnwJiZcM7/doJi3FmQmtP38byQnOVM5RM0nQ/gnFvt3uNbKVWbnZZKe1DrNCnTmVuueyslZ5LzFzaPWXCY8ErJhnO+A2f/b/j8LfdZIb+A9/4vs9ImQe2XnGlOxsxwxkZisP93MJL88YzNSmZsVmhBA84ly02t7RxtbqO+udV9besSLEebW6lvauNoSxv1Ta00dHzf3Nb5/mB9M5WH2mhoaqWhpY2Gpjaa247NRUb5uj7rEIEUfzzJCT63xeOGS6KPZL/7mhDvtpR8pCbEk+ZrZ/TRLYw+/BG5h9aRdWAN/sZqANqTstDCs4ibcQNy0tlOaAzn1qsIZIx2lolf6vpdW6szLhM8llJdAdvfgU+e6rpt+pieWymjxjuTnEaBBYeJDBE4eb6zHNkDH/2O1o+WOpMtrvmts02cHwqmuUEy08LkBIkISf54kvzxjEo9wf78XrS0tdPQ3Mabb73N6bPmctQNms6Aamp1w8gNp6Ag6nita2ql6kgTDS2txDUe4bSWTZyumzgj7jNmSAVJ4nQ77WjP5886jdXtp7Kq/TS2No5Ba+LwbRTSkmpJTXiPtEQfaUk+UhN9pCXGk5bovE93X1MTfaQn+UhNCHqf6HP2SwxTqygS4n1uGEwCLun6XXOD0wrrftXXxv+Go4eObSfumExQKyXrUD20nnvi4zy9sOAwkZcxGi74Bz7WOcy/4AI4tB32rHP6qnevc/4HWPuos62FyZDij48jMzmOUUlxTMrr/yq1LlSh5ouu4xMNGyFeUYmnvWA6TWO+zf6CWdTknMkRfw75zW1c0NTKrCYnlOrcpb6plbpG931zK4cbmtl1yGlFdawL5Yro+DhxWjpuyKQl+WiuP8rTlWu6BExnMLkhFBxYHSGVEq0LFhJSIFDsLN01HOzW9eUGy+d/htajzAC49K8BCw4znIlA9kRnKfoLZ51qUJiscwKl1zCZ4QSKhcnQ09bq3o0dFBS1u53vEtJh3Bwo+iqMOwspnE18QiopQAqQN8ifbm9XGlqCwqax9bjgqXVf65vaqHW/r29uZddhqKiq67JtKENGIpCW0NHSiSctyd+lFdQ1hI69P70wkzED6IrsU0o2pMyFcXO7/wOB2t18tOIFZvYy6/VgWHAY74UjTEbPcG7ssjCJni53Y6+EytWdd2OTMRbGn+NcEjvuLOffTQSn8oiLk84/nAv637yLsrIy5s+/oPOzqnK0xW3JNLV1tnQ6Wz7d3zc6AdQRRgdqmzu3qWtqPe7ChfuuPYO/mFkYhrPuQ1wcZBZyOKsoIoe34DBDUyhhsmed82RDC5Po6Lwb21263I1dDGdcdywowjQpphdExL2izBfKrT196rhwobPV09g6oAsfhioLDjN8WJhETmuTc+Na/X7nvon6/c5syfX7oXYPZ1W8A2X7nG077sb+0u3OvROFc5zLR81xgi9cyE2Lnf/mLDjM8GZh0jNV54qbjhCorwp63xEKB44FRdPhno/jS4a0fOrSTib5gludoAicHrXLPs3QZMFhYk9vYVKz49iVXD2FSf7UY1dyDcUwaWk8vlXQ01K3HxoOdL2hrJM4dyWn5jmzG4+Zcex9ap4z51Lw54RUEKG8rIz558yP8gmbocqCw4wMIsfmHRpwmMxwAiXcYdLe7syx1GMroPtyAJqO9HwcXzKkuX/oZxQ6dabmOdOAdwmFPCc07HkTZpAsOMzI1WeYrHMCZc862LjUeR4J9B4mHVqO9twqqOsWAh1dR9rD3FQS161VMNNtCQQFQFp+11aBMVFkwWFMsC5h8lVnXQhhclbCKHjvKDTX9nxcf+qxP+izxsHYmcdCoPuSkm2tAjOkWXAY05/+wmTPOo58tpbkidPcLqPuS661CkxMseAw5kR0C5NNvjIK5s/3uChjomOYTcxvjDHGaxYcxhhjBsSCwxhjzIBYcBhjjBmQiAaHiJSIyKciUiEii3v4PlFEnna//0BEJrjrF4jIGhFZ775eGLTPT0Rkp4jURbJ2Y4wxPYtYcIhIPPAAUApMA64XkWndNrsZOKSqk4H7gHvd9QeAy1V1OnAT8HjQPn8Cuk0+b4wxJloi2eKYC1So6jZVbQaeAq7sts2VgDu/A88CF4mIqOpHquo+AYZyIFlEEgFU9X1V3RPBuo0xxvQhksExFtgZ9LnSXdfjNqraChwGcrptcxWwVlWbIlSnMcaYARjSNwCKSBFO99Ul/W3bw76LgEXuxzoR+TSctUVQLk5XXSyycxu+Yvn87Nx6N76nlZEMjl1A8GPACt11PW1TKSI+IBOoBhCRQuB54JuqunWgP66qDwEPnUDdnhKR1ao62+s6IsHObfiK5fOzcxu4SHZVrQJOEZGJIpIAXAcs7bbNUpzBb4CrgTdVVUUkC3gJWKyq70awRmOMMQMUseBwxyxuAZYDm4BnVLVcRO4WkSvczR4GckSkArgd6Lhk9xZgMnCXiKxzl3wAEfmpiFQCKSJSKSI/itQ5GGOMOZ6oqtc1mCAissjtZos5dm7DVyyfn53bCRzXgsMYY8xA2JQjxhhjBsSCwxhjzIBYcAwRIvKIiFSJyAavawknERknIitEZKOIlIvIrV7XFE4ikiQiH4rIx+75/djrmsJNROJF5CMRedHrWsJNRLa7c+KtE5HVXtcTTiKSJSLPishmEdkkIueE7dg2xjE0iMj5QB3wmKoWe11PuIjIaGC0qq4VkXRgDfBVVd3ocWlhISICpKpqnYj4gXeAW1X1fY9LCxsRuR2YDWSo6mVe1xNOIrIdmK2qMXcDoIg8Crytqr92b4lIUdWacBzbWhxDhKr+GTjodR3hpqp7VHWt+74W59Ls7lPPDFvq6Jip2e8uMfO3MfdG3EuBX3tdiwmdiGQC5+Pc8oCqNocrNMCCw0SRO23+TOADj0sJK7crZx1QBbymqrF0fv8J/CPQ7nEdkaLAq+7jGxb1u/XwMRHYD/zG7Wb8tYikhuvgFhwmKkQkDXgOuE1Vj3hdTzipapuqzsCZVmeuiMREV6OIXAZUqeoar2uJoPNU9Uycxz98x+0yjgU+4Ezgl6o6E6jn2A3Wg2bBYSLO7ft/DnhCVf/odT2R4nYFrABKPC4lXOYBV7jjAE8BF4rI77wtKbxUdZf7WoUzN16sPOunEqgMav0+ixMkYWHBYSLKHTx+GNikqv/hdT3hJiJ57txqiEgysADY7GlRYaKqd6pqoapOwJlr7k1V/YbHZYWNiKS6F2zgduNcAsTEVY2quhfYKSKnuasuAsJ2QcqQnlZ9JBGRJ4H5QK47F9cPVfVhb6sKi3nAjcB6dxwA4Puq+rJ3JYXVaOBR94mXcThzssXcZasxqgB43vm7DT7g96q6zNuSwupvgSfcK6q2Ad8K14HtclxjjDEDYl1VxhhjBsSCwxhjzIBYcBhjjBkQCw5jjDEDYsFhjDFmQCw4jBnCRGR+LM5Ka4Y3Cw5jjDEDYsFhTBiIyDfc53KsE5H/5058WCci97nP6XhDRPLcbWeIyPsi8omIPC8io9z1k0XkdffZHmtFZJJ7+LSg5yo84d6Nb4xnLDiMGSQRmQpcC8xzJztsA24AUoHVqloEvAX80N3lMeB7qno6sD5o/RPAA6p6BnAusMddPxO4DZgGnIxzN74xnrEpR4wZvIuAWcAqtzGQjDPFejvwtLvN74A/us9JyFLVt9z1jwJ/cOdMGquqzwOoaiOAe7wPVbXS/bwOmIDzwChjPGHBYczgCfCoqt7ZZaXIP3fb7kTn92kKet+G/X9rPGZdVcYM3hvA1SKSDyAi2SIyHuf/r6vdbb4OvKOqh4FDIvIld/2NwFvu0xErReSr7jESRSQlmidhTKjsby7GDJKqbhSRH+A8SS4OaAG+g/PwnLnud1U44yAANwEPusEQPGvpjcD/E5G73WNcE8XTMCZkNjuuMREiInWqmuZ1HcaEm3VVGWOMGRBrcRhjjBkQa3EYY4wZEAsOY4wxA2LBYYwxZkAsOIwxxgyIBYcxxpgB+f+AhoZOxJyX4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7\n",
      "-------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Learning..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59aed5898485404a8a18eb9f6300eadc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batch #'), FloatProgress(value=0.0, max=145.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[0/36900] Loss: 0.021270'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[1845/36900] Loss: 0.021709'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[3690/36900] Loss: 0.021914'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[5535/36900] Loss: 0.021512'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[7380/36900] Loss: 0.020658'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[9225/36900] Loss: 0.021273'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[11070/36900] Loss: 0.021641'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[12915/36900] Loss: 0.020861'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[14759/36900] Loss: 0.021167'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[16605/36900] Loss: 0.021825'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[18449/36900] Loss: 0.021072'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[20294/36900] Loss: 0.021985'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[22140/36900] Loss: 0.020836'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[23985/36900] Loss: 0.022309'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[25830/36900] Loss: 0.021078'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'[27675/36900] Loss: 0.021097'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-5ec6961c37c7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Epoch {epoch+1}\\n-------------------------------\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mtrain_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msci_classifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msci_train_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0mtest_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msci_classifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msci_test_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-21-09a3ee852cb4>\u001b[0m in \u001b[0;36mtrain_loop\u001b[0;34m(model, dataloader, loss_fn, optimizer, step, history_loss)\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#!g1.1\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "sci_classifier = AutoModelForSequenceClassification.from_pretrained(sci_model_name, num_labels=num_tags).to('cuda')\n",
    "for param in sci_classifier.base_model.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "sci_classifier.train()\n",
    "loss_fn = F.binary_cross_entropy\n",
    "optimizer = torch.optim.Adam(sci_classifier.parameters(), lr=1e-4)\n",
    "epochs = 30\n",
    "\n",
    "train_loss = []\n",
    "val_loss = []\n",
    "    \n",
    "for epoch in range(epochs):\n",
    "    print(f\"Epoch {epoch+1}\\n-------------------------------\")\n",
    "    \n",
    "    train_loop(sci_classifier, sci_train_dl, loss_fn, optimizer, history_loss=train_loss)\n",
    "    test_loop(sci_classifier, sci_test_dl, loss_fn, history_loss=val_loss)\n",
    "    \n",
    "    clear_output()\n",
    "    plot_learning_process(train_loss, val_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "4tyrvht7r3rk4dny2oyk5"
   },
   "source": [
    "Видим, что разницы в результатах не заметно, поэтому остановим обучение."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "cmyx82lx7eixgamzj6jbza"
   },
   "source": [
    "Теперь попробуем датасет побольше (существенно побольше). Он взят [отсюда](https://www.kaggle.com/datasets/Cornell-University/arxiv?select=arxiv-metadata-oai-snapshot.json). Мы используем только часть датасета (100'000 статей)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "cellId": "4p7b0qtd94vy281idav2o"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1df6c68fe89c4f9aab024a5508bfb48a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "big_data = []\n",
    "with open('arxiv-metadata-oai-snapshot.json') as big_file:\n",
    "    for line in tqdm(big_file):\n",
    "        big_data.append(json.loads(line))\n",
    "        if len(big_data) >= 10**5:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "cellId": "dxhbuir0c2iux8wfptpc"
   },
   "outputs": [],
   "source": [
    "def trl(container):\n",
    "    return tqdm(range(len(container)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "cellId": "dyijqg6hqaoorj7jqdj3hj"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ee2a9096f28450d9a0da04c0455678c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=100000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "741143f6b10b4d2898bca8a4caf91227",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=100000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8777fbb3bc8c4d85851d55fa44dea200",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=100000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "titles = [big_data[i]['title'] for i in trl(big_data)]\n",
    "summaries = [big_data[i]['abstract'] for i in trl(big_data)]\n",
    "tags = [big_data[i]['categories'].split() for i in trl(big_data)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "ne6lc68j6kry5of6x805d"
   },
   "source": [
    "По каким-то причинам небольшая доля статей в этих данных не имеет ни одного тега из списка из 155 тегов, вытащенных нами с официальной страницы арксива. Чтобы избавиться от ситуации, когда у примера \"нет правильных ответов\" мы добавляем особый тег None, отвечающий за эту ситуацию. Разумеется, нам придется позже отфильтровать эту \"заглушку\" в пользовательской выдаче результатов. На практике, модель почти не пользуется этим тегом и предсказывает его очень редко и очень глубоко в списке тегов."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "cellId": "175cqj7ibqujnrz43e7vhq"
   },
   "outputs": [],
   "source": [
    "tag_to_id['None'] = 155\n",
    "id_to_tag[155] = 'None'\n",
    "id_to_description[155] = 'No tag'\n",
    "possible_tags.add('None')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "cellId": "z8tqaxe8t33akw7spxi3o"
   },
   "outputs": [],
   "source": [
    "for i, line in enumerate(tags):\n",
    "    line = list(set(line) & possible_tags)\n",
    "    int_line = []\n",
    "    for tag in line:\n",
    "        int_line.append(tag_to_id[tag])\n",
    "    if len(int_line) == 0:\n",
    "        int_line.append(155)\n",
    "    tags[i] = int_line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "cellId": "5iqci44rvbpzh2vnulsd1s"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "titles_train, titles_test, summaries_train, summaries_test, tags_train, tags_test = train_test_split(titles, summaries, tags, test_size=0.01, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "cellId": "od5nv2zwd3t0mjtp4qhdl6d"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "big_train_ds = ArXivDataset(titles_train, summaries_train, tags_train, possible_tags, base_tokenizer)\n",
    "big_test_ds = ArXivDataset(titles_test, summaries_test, tags_test, possible_tags, base_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "cellId": "wu358amm229d41o53r1ys"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "big_train_dl = DataLoader(big_train_ds, shuffle=True, \n",
    "    batch_size=128, num_workers=0)\n",
    "big_test_dl = DataLoader(big_test_ds, shuffle=True, \n",
    "    batch_size=128, num_workers=0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "y0rmkjb6ebf70313pjon"
   },
   "source": [
    "Натренируем классификатор:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "cellId": "q6v6a9sq9xl3fobim9swpf"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_transform.weight', 'vocab_projector.weight', 'vocab_layer_norm.weight', 'vocab_projector.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'classifier.weight', 'pre_classifier.bias', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "#!g1.1\n",
    "classifier = DistilBertForSequenceClassification.from_pretrained(base_model_name, num_labels=num_tags + 1).to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "cellId": "mb6xu490zup1awasmer9t3"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABAbklEQVR4nO3deXgW1fXA8e/JTkhIIJAASSBAEAiyh90lgiAugAsILoCtlWpL3WuxWmqVtq6t+hO11A1xAQW1VBCsSFxQkVX2XZawL2EJECDJ+f0xE3iNCbyBvJmQnM/zzJN37tyZ91yUHObeO3dEVTHGGGP8FeR1AMYYY84tljiMMcaUiiUOY4wxpWKJwxhjTKlY4jDGGFMqljiMMcaUiiUOU6mJyAYRubScvusNERldHt9V1kQkQ0SyvI7DnBsscRhjjCkVSxzGGGNKxRKHqTJEJFxEnhWRre72rIiEu8dqi8jHIrJPRPaKyFciEuQe+4OIbBGRgyKySkR6nuJraovI/9y6X4hIQ/caY0TkmSLxTBGRe0qItbl7nb3ud17vc+wNEXm5uO9xj3cTkbkist/92c3nWC0Red1tf7aIfFTke+8TkZ0isk1EfuH/n66pSixxmKrkIaAL0BZoA3QCHnaP3QdkAXWABOCPgIpIM2AE0FFVo4HLgA2n+I6bgMeA2sAi4G23fBxwg08yqg1cCrxT9AIiUh34n3ssHhgMvCgiaaf7HhGpBUwFngfigH8AU0Ukzj1vPBAJtHSv/U+fa9YFYoBE4FZgjIjUPEVbTRVlicNUJTcBj6rqTlXdBfwFGOIeOw7UAxqq6nFV/UqdhdzygXAgTURCVXWDqq47xXdMVdUvVfUoTqLqKiLJqvo9sB8ovFsZDGSq6o5irnEVsEFVX1fVPFVdCEwGBp7ue4ArgTWqOt49911gJdBXROoBlwO3q2q2284vfK553P3zOa6q04AcoNlp/1RNlWOJw1Ql9YGNPvsb3TKAp4C1wKcisl5ERgKo6lrgbuARYKeITBCR+pRsc+EHVc0B9vp8xzjgZvfzzTj/+i9OQ6Cz2222T0T24SS9un58T9E2FrYzEUgG9qpqdgnfu0dV83z2DwNRJdQ1VZglDlOVbMX5pVyogVuGqh5U1ftUtTHQD7i3cCxDVd9R1QvccxV44hTfkVz4QUSigFqF3wG8BfQXkTZAC+CjEq6xGfhCVWN9tihVvcOP7ynaxsJ2bnGvW0tEYk8RvzGnZYnDVCXvAg+LSB13jGEUzi9zROQqEUkVEcHpUsoHCkSkmYj0cAfRc4EjQMEpvuMKEblARMJwxiC+U9XNAKqaBczFudOYrKpHSrjGx8B5IjJERELdraOItPDje6a5594oIiEiMghIAz5W1W3AJzjjJTXd615U2j9EYyxxmKpkNDAPWAwsARa4ZQBNgc9w+vW/BV5U1Vk44xuPA7uB7TgDyg+e4jveAf6M03XUgZNdU4XGAa0ouZsKVT0I9MYZB9nqfu8Tbiyn/B5V3YMzRnIfsAd4ALhKVXe75w3BGctYCezE6YYzplTEXuRkTPlx/4X/Fs4g/Bn95RORN4AsVX34dHWNCQS74zCmnIhIKHAX8MqZJg1jKgJLHMaUA3d8Yh/OlN9nPQ3GmLNkXVXGGGNKJaB3HCLSx10uYW3hvPgix8NFZKJ7fI6IpLjlvURkvogscX/28DlnkIgsFpFlInKqaZHGGGMCIGB3HCISDKwGeuEs5TAXuEFVl/vU+Q3QWlVvF5HBwDWqOkhE2gE7VHWriJwPzFDVRHfZhIVAB1XdJSLjgDdVdeapYqldu7ampKQEpJ0V3aFDh6hevbrXYXjG2m/tt/afefvnz5+/W1XrFC0POauoTq0TsFZV1wOIyASgP7Dcp05/nCdyASYBL4iIuEssFFoGVHPn0TfGWU5hl3vsM+A64JSJIyUlhXnz5p1lc85NmZmZZGRkeB2GZ6z91n5rf8YZny8iRVchAAKbOBLxWRYB566jc0l1VDVPRPbjLMy226fOdcACVT0qImuBZm6XVhZwNRBW3JeLyHBgOEBCQgKZmZln2ZxzU05OTpVtO1j7rf3W/kC0P5CJ46yJSEucB596A6hqtojcAUzEeXr3G6BJceeq6lhgLEB6erpW1X912L+4rP3W/gyvw/BMoNofyMHxLfispwMkuWXF1hGREJwlnfe4+0nAh8BQ39VIVfW/qtpZVbsCq3DGUYwxxpSTQN5xzAWaikgjnAQxGLixSJ0pwDCcJR4GAJ+rqrqLsE0FRqrqbN8TRCReVXe67wn4DXA9xhhTxo4fP05WVha5ubleh3LGYmJiWLFixWnrRUREkJSURGhoqF/XDVjicMcsRgAzgGDgNVVdJiKPAvNUdQrwKjDeHbvYi5NcwHlxTiowSkRGuWW9VXUn8Jy7uig47w6wOw5jTJnLysoiOjqalJQUnLUvzz0HDx4kOjr6lHVUlT179pCVlUWjRo38um5Axzjcl8FMK1I2yudzLj99OU1h+WhOLj5X9NgNZRymMcb8TG5u7jmdNPwlIsTFxbFr167TV3bZkiPGGFOCyp40CpW2nZY4SlBQoLwzZxPTlmzzOhRjjKlQLHGUQAQmzN3EP/63GlvPyxhT3vbt28eLL75Y6vOuuOIK9u3bV/YB+bDEUQIRYVjXFNbuzGH22j1eh2OMqWJKShx5eXnF1D5p2rRpxMbGBigqhyWOU7iqTT3iqofxxjc/eh2KMaaKGTlyJOvWraNt27Z07NiRCy+8kH79+pGWlgbA1VdfTYcOHWjZsiVjx449cV5KSgq7d+9mw4YNpKenc9ttt9GyZUt69+7NkSMlva24dCr0k+NeCw8J5oZODRiTuZZNew7TIC7S65CMMR74y3+XsXzrgTK9Zlr9Gvy5b8sSjz/++OMsXbqURYsWkZmZyZVXXsnSpUtPTJl97bXXqFWrFkeOHKFjx45cd911xMXF/eQa69atY+LEifz73//m+uuvZ/Lkydx8c9G3GZee3XGcxk1dGhAkwvjvNngdijGmCuvUqdNPnrN4/vnnadOmDV26dGHz5s2sWbPmZ+c0bNiQtm3bAtChQwc2bNhQJrHYHcdp1IupRp+WdZk4dzP39DqPyDD7IzOmqjnVnUF58V0ePTMzk88++4xvv/2WyMhIMjIyin3CPTw8/MTn4ODgMuuqsjsOPwzrlsKB3Dw+WrjV61CMMVVEdHQ0Bw8eLPbY/v37qVmzJpGRkaxcuZLvvvuuXGOzfz77oWNKTVrUq8G4bzZwQ6fkKvNQkDHGO3FxcXTv3p3zzz+fatWqkZCQcOJYnz59ePnll2nRogXNmjWjS5cu5RqbJQ4/iAi/6JbCA5MX8936vXRtEnf6k4wx5iy98847xZaHh4fzySefFHuscByjdu3azJkz50T5/fffX2ZxWVeVn/q1rU/NyFDGfbPB61CMMcZTljj8FBEazKCODfh0+Xa27CubASZjjDkXWeIohZu7NADgre+KfQ2vMcZUCZY4SiGpZiS90hKY8P0mco/nex2OMcZ4whJHKQ3rlkL24eNM+cGm5hpjqiZLHKXUtXEczRKiGffNBls11xhTJVniKCURYWi3hizbeoD5G7O9DscYYwCIiooqt+8KaOIQkT4iskpE1orIyGKOh4vIRPf4HBFJcct7ich8EVni/uzhc84NbvliEZkuIrUD2YbiXNMukRoRIbxhU3ONMVVQwBKHiAQDY4DLgTTgBhFJK1LtViBbVVOBfwJPuOW7gb6q2goYBox3rxkCPAdcoqqtgcXAiEC1oSSRYSFcn57M9KXb2XHg5+vDGGPM2Ro5ciRjxow5sf/II48wevRoevbsSfv27WnVqhX/+c9/PIktkE+OdwLWqup6ABGZAPQHlvvU6Q884n6eBLwgIqKqC33qLAOqiUg4UAAIUF1E9gA1gLUBbEOJhnZN4dXZP/L2dxu5t3czL0IwxpSXT0bC9iVle826reDyx0s8PGjQIO6++25++9vfAvDee+8xY8YM7rzzTmrUqMHu3bvp0qUL/fr1K/dlkAKZOBKBzT77WUDnkuqoap6I7AficO44Cl0HLFDVowAicgewBDgErAF+W9yXi8hwYDhAQkICmZmZZ9mcn2tdO5g3vl5Lq5CthAZVzPWrcnJyAtL2c4W139p/pu2PiYk5schg+PFjBOWf+s17pVVw/BhHS1jEECA1NZXt27ezevVqdu/eTY0aNahevTr3338/33zzDUFBQWzZsoV169adWMeq6KKI+fn5JS6UWFRubq7ff1YVeq0qEWmJ033V290PBe4A2gHrgf8DHgRGFz1XVccCYwHS09M1IyOjzOMLqr+Loa99z8HYplzbPqnMr18WMjMzCUTbzxXWfmv/mbZ/xYoVREdHOzv9/lF2QfkIO83xQYMGMX36dLZv386NN97IlClT2L9/PwsXLiQ0NJSUlBRCQkJOxHkiXtfBgwd/VlaSiIgI2rVr51fdQA6ObwGSffaT3LJi67jjFzHAHnc/CfgQGKqq69z6bQFUdZ06c2HfA7oFKP7TuiC1No3rVLf1q4wxATFo0CAmTJjApEmTGDhwIPv37yc+Pp7Q0FBmzZrFxo3erGIRyMQxF2gqIo1EJAwYDEwpUmcKzuA3wADgc1VVEYkFpgIjVXW2T/0tQJqI1HH3ewErAtWA0wkKEoZ1TeGHrP0s3GRTc40xZatly5YcPHiQxMRE6tWrx0033cS8efNo1aoVb775Js2bN/ckroB1VbljFiOAGUAw8JqqLhORR4F5qjoFeBUYLyJrgb04yQWcmVKpwCgRGeWW9VbVrSLyF+BLETkObARuCVQb/HFdhySemrGKcd9soF2Dml6GYoyphJYsOTkoX7t2bb799tti6+Xk5JRXSIEd41DVacC0ImWjfD7nAgOLOW80xYxbuMdeBl4u20jPXFR4CAM6JPH2nI388coWxEdHeB2SMcYElD05XgaGdm3I8Xzl3TmbT1/ZGGPOcZY4ykDjOlFcdF4d3p6zkWN5BV6HY4wpI1VlPbrSttMSRxm5pVtDdh48yvRl270OxRhTBiIiItizZ0+lTx6qyp49e4iI8L+bvUI/x3EuyTgvnoZxkYz7ZgP92tT3OhxjzFlKSkoiKyuLXbt2eR3KGcvNzfUrIURERJCU5P+zaJY4ykhQkDC0awqPfbycpVv2c35ijNchGWPOQmhoKI0aNfI6jLOSmZnp90N9pWFdVWVoYHoSkWHBtmquMaZSs8RRhmpEhHJt+0Sm/LCVPTlHvQ7HGGMCwhJHGRvWNYVjeQVMmGtTc40xlZMljjLWNCGa7qlxvP3dRvLybWquMabyscQRAMO6prB1fy7/W77D61CMMabMWeIIgJ4tEkiqWc0GyY0xlZIljgAIDhKGdGnInB/3smLbAa/DMcaYMmWJI0AGdUwmIjTI3tVhjKl0LHEESGxkGFe3TeSjRVvYd/iY1+EYY0yZscQRQMO6pZB7vICJNjXXGFOJWOIIoBb1atCpUS3Gf7eR/ILKvVCaMabqsMQRYLd0SyEr+wgzV9jUXGNM5WCJI8B6pyVQLyaCcd9u8DoUY4wpEwFNHCLSR0RWichaERlZzPFwEZnoHp8jIilueS8RmS8iS9yfPdzyaBFZ5LPtFpFnA9mGsxUSHMTNXRoye+0e1uw46HU4xhhz1gKWOEQkGBgDXA6kATeISFqRarcC2aqaCvwTeMIt3w30VdVWwDBgPICqHlTVtoUbsBH4IFBtOGuqUJDP4I7JhIUE2V2HMaZSCOQdRydgraquV9VjwASgf5E6/YFx7udJQE8REVVdqKpb3fJlQDURCfc9UUTOA+KBrwLWgrNRkA/vD4PP/kxcVDh9W9fngwVbOJB73OvIjDHmrATyRU6JgO881Cygc0l1VDVPRPYDcTh3HIWuAxaoatF1ygcDE7WE9zqKyHBgOEBCQgKZmZln2Iwz13TfUeovf4GFRxJpFX4ek4/l8/cJmVyWElpuMeTk5HjS9orC2m/tt/Znlvl1K/QbAEWkJU73Ve9iDg8GhpR0rqqOBcYCpKena0ZGRiBCPLWuHeClbrTf+G/a3/41U7Ys5Ntdx/jr0IsJCpJyCSEzMxNP2l5BWPut/db+jDK/biC7qrYAyT77SW5ZsXVEJASIAfa4+0nAh8BQVV3ne5KItAFCVHV+YEIvI+HR0H8M7F0Hn49mWLcUNuw5zBerz913GBtjTCATx1ygqYg0EpEwnDuEKUXqTMEZ/AYYAHyuqioiscBUYKSqzi7m2jcA7wYm7DLW6CLoeBt89yJX1PiR+OhwWzXXGHNOC1jiUNU8YAQwA1gBvKeqy0TkURHp51Z7FYgTkbXAvUDhlN0RQCowymfqbbzP5a/nXEkcAJc+AjUbEvrfEQxLj+eL1btYvyvH66iMMeaMBPQ5DlWdpqrnqWoTVf2rWzZKVae4n3NVdaCqpqpqJ1Vd75aPVtXqvlNvVXWnz3Ubq+rKQMZepsKjnC6r7B/5Re44QoOFN7/d6HVUxhhzRuzJ8fKScgF0vp3Iha9wZ5MdTJqfRc7RPK+jMsaYUrPEUZ56joJajRm+9xkKjubwwYIsryMyxphSs8RRnsKqQ/8XCc/J4unYyYz7ZgMFtmquMeYcY4mjvDXsCl1+wxW5U4nf8z1fr919+nOMMaYCscThhR4PU1ArlWfCxjLx6+VeR2OMMaViicMLYZEEXfMSddlDtx+fY9Oew15HZIwxfrPE4ZXkThzpcDs3Bc/kq+nveR2NMcb4zRKHh6r3GcX2sAb0WPMohw/u9TocY4zxiyUOL4VWY2+vZ4nXvWydeJ/X0RhjjF8scXisRXoPJkdcR2rWB+ia/3kdjjHGnJYlDo+JCEGXPMjqgkSOfTgCjuzzOiRjjDklSxwVwFUdGvGX4BGEHN4JMx7yOhxjjDklSxwVQERoMK069eDlvL6w6C1YPcPrkIwxpkSWOCqIm7s04P/yr2VntSYw5U44ku11SMYYUyxLHBVEUs1ILk5L4s7c4eihXTD9Qa9DMsaYYlniqECGdUvhuyPJLE+9DX54F1ZO8zokY4z5GUscFUjXxnE0S4jmwV2XoQnnw8d3w2F7MNAYU7FY4qhARISh3RqyePsRlnV6Ag7vgU/+4HVYxhjzE6dNHCJSR0T+KCJjReS1ws2fi4tIHxFZJSJrRWRkMcfDRWSie3yOiKS45b1EZL6ILHF/9vA5J8yNZbWIrBSR60rR3grvmnaJ1IgI4aVVkXDRA7DkPVjxX6/DMsaYE/y54/gPEAN8Bkz12U5JRIKBMcDlQBpwg4ikFal2K5CtqqnAP4En3PLdQF9VbQUMA8b7nPMQsFNVz3Ov+4UfbThnRIaFMKhjMtOXbmdb6zugbmv4+B44tMfr0IwxBvAvcUSq6h9U9T1VnVy4+XFeJ2Ctqq5X1WPABKB/kTr9gXHu50lATxERVV2oqlvd8mVANREJd/d/CfwdQFULVLXSvQlpSJcUClR5e+42uPol52nyafd7HZYxxgAQ4kedj0XkClUt7RSfRGCzz34W0LmkOqqaJyL7gTicO45C1wELVPWoiMS6ZY+JSAawDhihqjuKfrmIDAeGAyQkJJCZmVnK8L3VpnYw42avpXVIJKkNr6fxsrdZpqnsiu9equvk5OScc20vS9Z+a7+1P7PsL6yqp9yAg0ABkOt+Pggc8OO8AcArPvtDgBeK1FkKJPnsrwNq++y3dMuauPu1AQUGuPv3AuNPF0uHDh30XPPl6p3a8A8f66R5m1Xzjqu+fJHqE41UD+4s1XVmzZoVmADPEdb+WV6H4Clr/6yzOh+Yp8X8Tj1tV5WqRqtqkKpGuJ+jVbWGHzlpC5Dss5/klhVbR0RCcMZS9rj7ScCHwFBVXefW3wMcBj5w998H2vsRyznngtTaNI2P4plPV7HrcL7TZXX0IEy9F5ykaYwxnvBrOq6I9BORp93tKj+vPRdoKiKNRCQMGAxMKVJnCs7gNzh3KJ+rqrpdUlOBkao6u7CymwH/C2S4RT2BSvnSbhHhH9e3Ze/hY/x6/DyOxjWDjAdhxRRY9sHpL2CMMQHiz3Tcx4G7cH5BLwfuEpG/n+48Vc0DRgAzgBXAe6q6TEQeFZF+brVXgTgRWYvT7VQ4ZXcEkAqMEpFF7hbvHvsD8IiILMbp/qq0b0BqlRTDMwPbsmDTPh78YAna7XeQ2AGm3g85O70OzxhTRfkzOH4F0FZVCwBEZBywEDjtYkrqDKhPK1I2yudzLjCwmPNGA6NLuOZG4CI/4q4UrmxdjzU7m/LsZ2tolhDNr69+CV6+0JmiO+gtEPE6RGNMFePvk+OxPp9jAhCHOYU7ezTlylb1eHz6SmbujoUeD8HKj2HJJK9DM8ZUQf4kjr8DC0XkDfduYz7w18CGZXwFBQlPD2xDy/o1uPPdhaxqNAySOjrPdhzc7nV4xpgqxp9ZVe8CXXBmMk0GuqrqxEAHZn6qWlgw/x6aTmR4CLeOX0B27+cgLxf+e7fNsjLGlKsSE4eINHd/tgfq4TzAlwXUd8tMOasXU42xQzqw8+BRfv3JAfIyHobVn8Biy+PGmPJzqsHxe3GevH6mmGMK9Cim3ARYuwY1eWpAa+6asIiHa13A35O7IJ88AI0uhhr1vA7PGFMFlJg4VHW4+/Fyd/bTCSISEdCozCn1b5vI6h0HGTNrHek9HmTAtsHw37vgxok2y8oYE3D+DI5/42eZKUf39WpG77QEHph1iDWt74U1M2DRO16HZYypAk41xlFXRDrgrEzbTkTau1sGEFleAZriBQUJ/xzUlmZ1a3Dd/FYcqdcZpo+E/UVXdTHGmLJ1qjuOy4CncdaYesZnuwf4Y+BDM6dTPTyEfw/tQFhoCLfuuwUtyIMpv7NZVsaYgCoxcajqOFW9BLhFVXuo6iXu1l9VbbGkCiKpZiT/GtKBeQdqMi7yFlg3ExaOP+15xhhzpvwZ4+jg8x4MRKSmiBS7HIjxRoeGtfjbta34y45urI9qB9P/CPs2n/5EY4w5A/4kjstVdV/hjqpm46xfZSqQAR2SGH5RKkP3DON4vnVZGWMCx5/EEezz2lZEpBoQfor6xiMP9GlOs2bn8+jRG2D9LJj/htchGWMqIX8Sx9vATBG5VURuBf7HyfeEmwokOEh4dnBbvq/Vj+9oRcGMh4g48rO36hpjzFnxZ62qJ3AWNWzhbo+p6pOBDsycmeiIUP49rDOPyh3kHs+n+bIn4Ui212EZYyoRv5ZVV9VPVPV+d5sR6KDM2WkQF8mfh/Th7uMjqJ6zAX2jLxza7XVYxphKwp83AF4rImtEZL+IHBCRgyJyoDyCM2euc+M4evQfxq3H7iNv5yp440o4aN1Wxpiz588dx5NAP1WNUdUaqhqtqjUCHZg5e4M7NSA8uQNDj/6e43s3wuuXw/4sr8Myxpzj/EkcO1R1xZlcXET6iMgqEVkrIiOLOR4uIhPd43NEJMUt7yUi80Vkifuzh885me41i76L3BRjcLMwQlMzuDF3JHkHdzjJI3uD12EZY85h/iSOee4v9xvcbqtrReTa050kIsHAGOByIA24QUTSilS7FchW1VTgn8ATbvluoK+qtgKGAUUfhb5JVdu6204/2lBlBQcJ/3dDO/bUasuwvIfJP3IAXr8Cdq/1OjRjzDnKn8RRAzgM9Ab6uttVfpzXCVirqutV9RgwAehfpE5/Tk7tnQT0FBFR1YWqutUtX4az0KI9O3KGYqqF8uqwjiylCXeE/IWCvKPOnceO5V6HZow5B4kG6OliERkA9FHVX7n7Q4DOqjrCp85St06Wu7/OrbO7yHVuV9VL3f1MIA7Ix3mV7WgtphEiMhznRVQkJCR0mDBhQkDaWdHl5OQQFRUFwPI9+Tw9L5fLa27jH3mjCSrI44c2j5IT3djjKAPHt/1VkbXf2n827b/kkkvmq2p60fJTvQEQABF5HeeNfz+hqr8842j8JCItcbqvevsU36SqW0QkGidxDAHeLCa+scBYgPT0dM3IyAh0uBVSZmYmhW3PAKITN/Knj+D8TmO5Y+O9pC97BG7+AJJ+9v9GpeDb/qrI2m/tD0T7/emq+hiY6m4zcbqucvw4bwuQ7LOf5JYVW0dEQoAYYI+7nwR8CAxV1XWFJ6jqFvfnQeAdnC4x46chXRoypEtDnvj+OFM7vArVasGb/WGjvZvLGOMff54cn+yzvQ1cD/jzz9O5QFMRaSQiYcBgYEqROlNwBr8BBgCfq6q6q/FOBUaq6uzCyiISIiK13c+hOGMtS/2IxfgY1TeN7qlx3DNjLz9c+g7UqA9vXQfrZnkdmjHmHODXk+NFNAVOOwVWVfOAEcAMYAXwnqouE5FHRaSfW+1VIE5E1gL3AoVTdkcAqcCoItNuw4EZIrIYWIRzx/LvM2hDlRYaHMSYG9tTPzaCX36wha3XTIZajeGdQbDaFgYwxpyaP2McB/npGMd24A/+XFxVpwHTipSN8vmcCwws5rzRQEnv/Ojgz3ebU4uNDOOVYR255sXZ/PL9DUy+5SOqTxwAE26CAa9BWr/TX8QYUyWd6p3j3d2Pddwnxgu381R1cjnFZwIoNT6KF25sz+odB7lnykYKhvwHEtvD+7fA4ve9Ds8YU0GdqqvqefenjZpWYhefV4eHr0zj0+U7eOar7c4Mq4bd4IPbYIG9gtYY83On6qo6LiJjgSQReb7oQVW9M3BhmfL0i+4prNl5kDGz1tE0Ppqrb3wPJt4MU0ZAXi50us3rEI0xFcipEsdVwKXAZcD88gnHeEFE+Eu/81m/6xAPTF5Mw7gutLvhXXj/FzDtfid5dPud12EaYyqIEruqVHW3qk7AWRl3XNGtHGM05SAsJIiXbu5AQo1who+fz7ZDBXD9OGh5DXz6MHzxlNchGmMqCH+e4/ihPAIx3qtVPYxXh3Xk8NE8bntzHofzBa59BdrcALNGw8xHIUBL1Bhjzh1n8hyHqcTOS4jm+RvasWzrAe5//wcKJBj6vwgdboGvnoEZf7TkYUwVZ4nD/EzPFgk8eHlzpi3Zzt8/WUEBAlc9C51vh+9ehKn3QkGB12EaYzziz6tj7xKRGuJ4VUQWiEjv051nzm23XdiYm7s04N9f/cgtb8xl7+Hj0OdxuOAemPeaM+OqIN/rMI0xHvDnjuOXqnoAZ4Xamjir0T4e0KiM50SEx/qfz9+uacV36/Zw5fNfsWDzPuj5Z8j4Iyx6Gyb/CvKPex2qMaac+ZM4xP15BTBeVZf5lJlKTES4sXMDJt/RjZBg4fqXv+X1bzagFz8AvR6FZR84T5nnHfU6VGNMOfInccwXkU9xEscM9z0Y1sFdhbRKiuHjEReS0awOf/nvcka8s5CDHX4Dlz8FKz+GCTfC8SNeh2mMKSf+JI5bcVat7aiqh4FQ4BcBjcpUODGRoYwdks7Iy5szfdl2+r8wm5UNB0Pf52HtTHh7IBz15zUtxphznT+JoyuwSlX3icjNwMPA/sCGZSqioCDh9oub8PavOnPwaB5Xj5nNZHrCNf+CjbOdd3rk2v8axlR2/iSOl4DDItIGuA9YRzGvajVVR5fGcUy98wLaJsdy3/s/8OC6Fhy75lXYMg/evBoO7/U6RGNMAPmTOPJUVYH+wAuqOgaIDmxYpqKLj47grVs785uMJrz7/WauyazDzitehR1LYVxfyNnldYjGmADxJ3EcFJEHcabhThWRIJxxDlPFhQQH8UCf5rw6LJ2s7CP0/Lga87q/DHvWwRtXwoFtXodojAkAfxLHIOAozvMc24EkwFa8Myf0bJHAx7+7gEa1qzPg0wjeSn0GPbAF3rgC9m32OjxjTBnzZ5HD7cDbQIyIXAXkqqpfYxwi0kdEVonIWhEZWczxcBGZ6B6fIyIpbnkvEZkvIkvcnz2KOXeKiCz1Jw4TeMm1Inn/9q7c3KUBDy+K5eGoxyg4tBtevgDmjIX8PK9DNMaUEX+WHLke+B7n3eDXA3NEZIAf5wUDY4DLgTTgBhFJK1LtViBbVVOBfwJPuOW7gb6q2goYBvzkVXQici1gcz8rmPCQYEZf3YrnBrflg131GZj3GPti0+CT38O/LoQfv/Q6RGNMGfCnq+ohnGc4hqnqUKAT8Cc/zusErFXV9ap6DJiAM8Duqz9Q+G6PSUBPERFVXaiqW93yZUA1EQkHEJEo4F5gtB8xGA/0b5vIlBHd2ReZQvuNv2Vay6fQYznOoPnEIZC90esQjTFn4VRvACwUpKo7ffb34F/CSQR8O7izgM4l1VHVPBHZD8Th3HEUug5YoKqF61o8BjwDHD7Vl4vIcGA4QEJCApmZmX6EXPnk5OR41vbft1HeWBbCb+Ynkl77r4xO/oSmqybDqulsTr6GTQ2uoyA4PKAxeNn+isDab+0PRPv9SRzTRWQG8K67PwiYVuaRFENEWuJ0X/V299sCTVT1nsLxkJKo6lhgLEB6erpmZGQENNaKKjMzEy/bfllP5a05m3jsv8u5NW8gY6+/k5ZLnyZl6URSsmdD70eh5bUggVn+zOv2e83ab+0PRPv9GRz/Pc4v4NbuNlZV/+DHtbcAyT77SW5ZsXVEJASIwbmjQUSSgA+Boaq6zq3fFUgXkQ3A18B5IpLpRyzGIyLCkC4Nef/2rgBcPf5Hxif+Cb1lGkTWhEm/dKbubl/icaTGGH/59SInVZ2sqve624d+Xnsu0FREGolIGDAYmFKkzhScwW+AAcDnqqoiEgtMBUaq6myfOF5S1fqqmgJcAKxW1Qw/4zEeapMcy9Q7L+DCpnX403+Wcee3kRwaNtN5QdSulfCvi+Dje+DQHq9DNcacRomJQ0QOisiBYraDInLgdBdW1TxgBDADWAG8p6rLRORREennVnsViBORtTgD3oVTdkcAqcAoEVnkbvFn0U5TAcRGhvHK0HR+f1kzpi7eSr8Xv2VN8gD43Xzo9GuYPw7+r71N3zWmgisxcahqtKrWKGaLVtUa/lxcVaep6nmq2kRV/+qWjVLVKe7nXFUdqKqpqtpJVde75aNVtbqqtvXZdha59gZVPf/Mm268EBQk/PaSVN76VWf2HzlOvxdm89HKw3D543DHbKjf9uT03fVfeB2uMaYY9s5x44luTWoz9c4LaZUYw90TF/HQh0vIrXkeDPkIBr0Nxw7Bm/1s+q4xFZAlDuOZhBoRvHNbZ359cWPenrOJgS9/y+bsI9DiKvjt99DjYVj7GYzpBJ//FY6dcga2MaacWOIwngoJDuLBy1swdkgHNuw5xFX/9zUzV+yA0Ai46PcwYh606AtfPgkvdISlk0HV67CNqdIscZgKoXfLukz93YUk1azGrePm8cT0leTlF0BMIlz3CvxiOkTWOjl9d9tir0M2psqyxGEqjAZxkUy+oxs3dGrAS5nruOmVOew8mOscbNgVhmdC3+ec6btjL7bpu8Z4xBKHqVAiQoP5+7WteGZgG37I2kfPZ75gzKy1HDmWD0HB0OGWItN328Gcf9n0XWPKkSUOUyFd1yGJj393IZ0b1eKpGavIeHoW736/yem+qlbTnb77DdRvD588YNN3jSlHljhMhZUaH8Urwzry3q+7khhbjQc/WELvZ79k+tLtqCrEN4chH8Lgd+D4YXf67s2QvcHr0I2p1CxxmAqvU6NaTL6jG/8a0gEBbn9rPte+9A1z1u9xFkdsfiX8Zg70+BOsnQkvdILPRzvPghhjypwlDnNOEBEua1mXGXdfxBPXtWLbvlwGjf2OX74xl5XbD7jTd+93xj/S+sOXT8ELHam77TM4fsTr8I2pVCxxmHNKSHAQgzo2YNb9GfyhT3PmbdjL5c99xb3vLSIr+zDUqA/X/Rt+OQOq16b5qv+DZ5rDJyNh50qvwzemUrDEYc5J1cKCuSOjCV8+cAnDL2zMx4u30ePpLxj98XKyDx2DBl1g+BcsavMYpPaEua/Ai53htT7ww0Q4nut1E4w5Z1niMOe02MgwHryiBZn3Z3B1u/q8NvtHLnpyFmNmreXw8Xz21WwNA16D+1ZCr0chZwd8OBz+0RymPwi7VnndBGPOOZY4TKVQP7YaTw5ow/S7L6Jz4zhnCu9TmWRuPu5M4a1eG7rfBSPmw9Ap0DgDvv+3sw7W61fA4vfsLsQYP1niMJXKeQnRvDIsnfdv70pyrUjeWHbMncK7zZnCGxQEjS+GgW/AvSvg0r/Aga3wwW3wjxYw4yHYtdrrZhhToVniMJVSx5RaTLq9K3e1DydYhNvfWsA1L37Dd+t9liiJqgMX3A2/W+As597oIpjzMozpCK9fCUsmQd5Rr5pgTIVlicNUWiJCu/gQPrnrQp68rjU7DuQyeOx3/OL171mxzecllkFB0OQSuH6ccxfS889wIAsm3+rMyJrxEOxe611DjKlgLHGYSi8kOIjrOyYz6/4MHry8OfM3ZnPF819x78RFbN5b5B0fUfFw4b3wu4XOU+kpFzh3IS90gDeusrsQYwhw4hCRPiKySkTWisjIYo6Hi8hE9/gcEUlxy3uJyHwRWeL+7OFzznQR+UFElonIyyISHMg2mMojIjSYX1/chK8e6MHwixozdck2ej7zBY99vJy9h479tHJQEDTpAYPGwz3Loeco2LfJuQv5Rwv49E+wZ503DTHGYwFLHO4v9DHA5UAacIOIpBWpdiuQraqpwD+BJ9zy3UBfVW0FDAPG+5xzvaq2Ac4H6gADA9UGUznFRIby4OUtyPx9Bte0S+T12T9y8ZOzeOHzNRw+Vswqu9EJcOF9cOciuPkDaNgNvh0D/9cexvWFpR9A3rGfn2dMJRXIO45OwFpVXa+qx4AJQP8idfoD49zPk4CeIiKqulBVt7rly4BqIhIOoKqFndMhQBhgr4MzZ6ReTDWeGNCaGXdfRJcmcTz96WoufiqTt+ds5Hh+wc9PCApyHiYc9Bbcu9x5te3eDTDpF85dyP9G2V2IqRJEA/QaThEZAPRR1V+5+0OAzqo6wqfOUrdOlru/zq2zu8h1blfVS33KZuAkpk+AIaqaX8z3DweGAyQkJHSYMGFCAFpZ8eXk5BAVFeV1GJ4pTfvXZOfz/upjrM4uoG6kcN15YaQnBCMiJZ+k+dTa+wP1tk2n9u65CAXsrdmGbfUuY3ftTmhQaBm15MzYf39r/9m0/5JLLpmvqulFy0POKqoAE5GWON1XvX3LVfUyEYkA3gZ6AP8req6qjgXGAqSnp2tGRkbA462IMjMzqapth9K1PwP4lSozV+zkiekrGbMohxb1anBjp2T6tU0kplpJSaAncC8c2AYL36LWgnHUWv4kVK8D7YdC+q3OK3A9YP/9rf2BaH8gu6q2AMk++0luWbF1RCQEiAH2uPtJwIfAUFX92f2/quYC/+Hn3V/GnDER4dK0BKbffRFPDWiNAH/6zzI6/+0z7n1vEd//uJcS79Jr1IOLfw93/QA3TYKkTvD1P+G51s670jfPLde2GBMogbzjmAs0FZFGOAliMHBjkTpTcAa/vwUGAJ+rqopILDAVGKmqswsri0gUEK2q29xEcyXwVQDbYKqo4CBhYHoyA9OTWZK1nwlzN/GfRVv5YMEWGtepzuCOyVzbPonaUeE/PzkoGJr2crbsjfD9WFgwHpZOhsQO0OU3ztLvwd52YxlzpgJ2x6GqecAIYAawAnhPVZeJyKMi0s+t9ioQJyJrgXuBwim7I4BUYJSILHK3eKA6MEVEFgOLgJ3Ay4FqgzEArZJi+Os1rfj+oZ48NaA1tSLD+Nu0lXT520zueGs+X6zeRX5BCXchNRvCZX91BtOveBpy9ztTep9t5bwz5NCe4s8zpgIL6BiHqk4DphUpG+XzOZdiptOq6mhgdAmX7ViWMRrjr8iwkBN3IWt3HmTi3M1MXrCFT5ZuJzG2GgPTkxiYnkxibLWfnxweBZ1uc8Y71n4Gc15y3lL45dPQaiB0uQMSWpZ/o4w5AxV6cNyYiio1PpqHrkzj/sua8dnynUyYu4nnZq7huZlruPi8OgzumEzPFgmEBhe5qQ8KgvN6O9vOlc5T6T9MgIXjnbWyuvwGml7m1DOmgrLEYcxZCA8J5srW9biydT027z3M+/M28968LG5/awG1o8K4rn0Sgzom07hOMVMi45tD32edp9IXjHOWeX93MNRsBJ1/DW1vgoga5d4mY07H/lljTBlJrhXJvb2bMXtkD16/pSPtG9Tkla9/pMczX3D9v77lgwVZ5B7/2SNHEFkLLrgH7lrsLPceFQ/TR8I/0pxX3u5dX+5tMeZU7I7DmDIWHCRc0jyeS5rHs/NgLpPnb2Hi3E3c+94P/HnKMq5pl8igjsm0rB9T5MQQaHmNs21Z4HRjzX3F+dnscuh8u9OddaoHEo0pB5Y4jAmg+OgI7showu0XN+a79XuZOHcTE+Zu5s1vN9I6KYZBHZPp16Y+0RFFpuYmtodrxzqvu537Ksx7FVZNg/iW0OV2Z0A9tJhBeGPKgXVVGVMORISuTeJ4dnA75v7xUh7pm8axvAIe+nApnf46k9+//wPzNxbzcGF0XejxkLNCb/8xIEEw5XdON9bMx5yn1Y0pZ3bHYUw5i4kM5ZbujRjWLYXFWfuZMHczUxZt4f35WaTGR514uLBW9bCTJ4VGQLubnQHzDV873VdfPQOzn4W0q53ZWEkdvGqSqWIscRjjERGhTXIsbZJjefjKFkxdvI0JczcxeuoKnpi+kt4t6zKwQxLdU2ufnNYrAo0udLa9PzozsRaOh6WTIKmj8zxIi372VLoJKEscxlQA1cNDuL5jMtd3TGbVdufhwg8WZjF18TZiI0O5LK0uV7SuR7cmcSeTSK1G0OdvcMmDsOgd5y5k0i8hur7zsGGHWzxtk6m8LHEYU8E0qxvNqL5p/OHyZny5ejfTlmxj6pJtTJy3mdjIUHqnJXBl6/onk0h4tPPcR8fbYM2nzlPpM/8CXzzJeXUuhOZxULeV180ylYglDmMqqPCQYHqlJdArLYHc4/l8tcZJIp8s2c5787JOJJErWtU72Z3VrI+z7VgOc14mYdG78PKnkNzFuQtp0Q9Cwk7/5cacgiUOY84BEaGnTiIx1UK5rKVPEklIg37P82213lxQfYPzPMjkWyEqwenC6nAL1KjvcavMucoShzHnGN8kcjQvn69W72ZqCUkkL7g6dBvhzLpaN9MZTP/iSWdxxRZ9nbuQht3toUJTKpY4jDmHhYcEc2laApf6JBHfO5HqoXDFnh+4snU9uqf2JLRpL2c21rxXnXeELP8I4tOg46+g9SBnFV9jTsMShzGVRHFJ5LXPFjF96Xben+/cifROS+CK1vXo3uNRwjL+6Lxc6vuxMPVe+OwRaHujk0RqN/W6OaYCs8RhTCVUmERCdobT9YILT9yJ/DyJXEb3W28kbPt8pxtr7qvOtN7GlzjdWOf1cd5oaIwPSxzGVHJF70S+XrObqYtPJpEaESFc1rIuV7T+K917PkbY4vEw73WYcCPEJEP6L6H9UKhe2+ummArCEocxVUh4SDA9WyTQs0XJSaR3y95cdflNdM+fS+j8V5xnQjL/Di2vhU7DbWkTE9jEISJ9gOeAYOAVVX28yPFw4E2gA7AHGKSqG0SkF/A4EAYcA36vqp+LSCTwPtAEyAf+q6ojMcaUWrFJZMk2ZizdzqT5WdSIiOCS5o/S84JsLsj+iJorJyOLJ0D9dk4CaXmts4aWqXICljhEJBgYA/QCsoC5IjJFVZf7VLsVyFbVVBEZDDwBDAJ2A31VdauInA/MABLdc55W1VkiEgbMFJHLVfWTQLXDmKqgpCTy5erd/GfRUeAy6oZfwh0153FV9lTiProDnfEQ0n6o05VVs6HXTTDlKJB3HJ2Ataq6HkBEJgD9Ad/E0R94xP08CXhBRERVF/rUWQZUE5FwVT0MzAJQ1WMisgBICmAbjKlyfJOIqrJ57xHmb9rLvA3ZvLsxjkf2daWLLGdY/qf0mv08Mvs5dtTNILjzcOq0uQyxwfRKT362/n9ZXVhkANBHVX/l7g8BOqvqCJ86S906We7+OrfO7iLXuV1VLy1y/VhgAXBpYXIqcnw4MBwgISGhw4QJE8q4heeGnJwcoqKq7tx8a3/Zt//wcWX9/nzWZBewb+8OuuR8xoCgz6kjB9iodZlZ7TI21+lBclw0DWoEERLk3cOF9t//7Np/ySWXzFfV9KLlFXpwXERa4nRf9S5SHgK8CzxfXNIAUNWxwFiA9PR0zcjICGywFVRmZiZVte1g7S+P9ucX3MyqLbtY+f37JK19m18eGcfhTRP46MfuvC1dqRsfz3n142iWXIeWDeKJiYp2xkZCIiAoJKBPrdt//8C0P5CJYwuQ7LOf5JYVVyfLTQYxOIPkiEgS8CEwVFXXFTlvLLBGVZ8NQNzGmFIIDhLSkuMh+bfAb2HrIpj9MoNWfMCNBZ87f6P3AEt+fq5KEIREICHhTiI58TOiyH4xP0OrFan/82tEHtoEqrakShkLZOKYCzQVkUY4CWIwcGOROlOAYcC3wADgc1VVtxtqKjBSVWf7niAio3ESzK8CGLsx5kzVb0vkwJfh8N9g+xLIy+Vo7mE279zLpp3ZbN29jx3Z+yk4lku4HCNG80mOCKJudYiPhJphBQTnH4O8XMg7Crn7nZ95uT7bUTh+BDh1V3sngBWjnQcam/SAJpdAVHx5/ClUagFLHKqaJyIjcGZEBQOvqeoyEXkUmKeqU4BXgfEishbYi5NcAEYAqcAoERnllvXGmZ77ELASWCDOvyJeUNVXAtUOY8wZiqwFjS8GIBznL3Sqe6igQFm/+xDzN+5l/sZsxm3MZv2GQwCEBAktE2NIb1iT9g1q0jophqSa1ZCidw2qUJB3MpH4JhR3f+W302keuhXW/g8Wu+OcdVu5SaQHNOjq3KGYUgnoGIeqTgOmFSkb5fM5FxhYzHmjgdElXNbuOY05xwUFCanxUaTGRzGoYwMA9h46xoKN2czflM38Ddm89d1GXv36RwBqRobSKimW1okxtE6KoXVSLAk1wpHgUOc1ueHRxX7P9g3HaZ6RAQUFsH2xs0Lwulnw7Ysw+zkIqQYpFzhJJLUn1D7PurX8UKEHx40xVUet6mEnlkYBOJZXwMrtB1ictZ8lWftZvGU/L32xjvwCp3uqTnQ4rRNjaJUUQ5ukWFolxVA7qoS7h6AgqN/W2S68D47mwIavYd3nzjbjQadvpEai053VpCc0znDumszPWOIwxlRIYSFBtE6KpXVS7Imy3OP5LN92gMWb97F4i5NQPl+1k8KnCurHRNDKvSNpnRRDzrESxkDCo06+LREgeyOsn+UkkRX/hYVvAeI8JZ/a07kjSero3N0YSxzGmHNHRGgw7Rs4Yx+FDh3NY9nWAyzO2ufcnWzZz4xlO04cf3LRLCeZJDoJ5fzEGkRHFEkANRuefDNifh5sXejejcyEr/4BXz4FYdHQ6CL3jqQHxDUpn0ZXQJY4jDHntOrhIXRqVItOjU52K+0/cpylW/bz0ZcLOBRegx8272Pq4m0njjeuU93t5oqlTVIMafVrEBnm/joMDoHkjs6W8Qc4sg82fAVrZzqJZNVUp17NlJOD7I0ugoiY8mu0xyxxGGMqnZhqoXRPrc3xrDAyMpzVfPceOsbirH0nxku+W7+XjxZtBSBIoGl8tNvN5dyZNK8bTURoMFSLdV6z26KvM5Nr7/qTYyOL34N5r4EEO11ZhYkksX2lfo+JJQ5jTJVQq3oYGc3iyWh28jmOnQdyWewmkiVZ+5i1cieT5mcBzrTgwplfTepE0SQ+itQ6UTSqnUK1Trc5L7rKPw6bvz+ZSDL/Dpl/c+4+GmdAwwsgrjHEpkBscqWZ+muJwxhTZcXXiODStIgTM7lUla37c1nijpcs3+bM6pq6ZNuJAXgRSIyt5iSTOlE0iU+iSaPf0KTTA9QOOoj8+IWTRNZ+Dsv/4/Nt4szaqtnQ6eaKdX/WTHHKohLOmanAljiMMcYlIiTGViMxthp9zq93ojz3eD4b9hxi3c5DrNuVc2L7/se9HDmef6JejYgQmsTXpUmd4TRpfw9p0YdIDd1DQv52QvZvhH0bIXuDk1gObvvpl4dUcxJI0YRSmGTCK85ijZY4jDHmNCJCg2letwbN69b4SXlBgbL9QK6TSHbmsG7XIdbuzOHL1btOdHkBhATF0jCuPk3q9KJJ/SiatIkitWYwqeHZRB3KOplQsjc4U4M3fgPHDv40iMjaP00mvnctNRKdQf1yYonDGGPOUFCQUD+2GvVjq3Fh0zo/OXYg9zjrdx1yE0rhdojPV+4kr+Dk8yV1osNpUud8UuO70CQ5iibto2hSpzr1Qo8QtH/DyWRSmFi2zIdlH4Hm+wQSAjFJP+sCiz6wB/IvKPOkYonDGGMCoEZEKG2TY2mbHPuT8uP5BWzee5h1uw753KnkMGXRVg7k5p2oVy00mMZ1qvPer/tTPbzIr+r8PDiQdTKh+N6xrJwKh51XGnUAuOwGCC7bbi5LHMYYU45Cg4NoXCeKxnWi6EXCiXJVZc+hYye6vNbtymHrviNEhhUzrTc45GR3FRf//PjRg5C9kcWzp9M6AGMjljiMMaYCEBFqR4VTOyqczo3jzu5i4dFQ93z2xu0+fd0zEBSQqxpjjKm0LHEYY4wpFUscxhhjSsUShzHGmFKxxGGMMaZULHEYY4wpFUscxhhjSsUShzHGmFIR1RLeyVuJiMguYKPXcXikNhCYp4DODdZ+a7+1/8w1VNU6RQurROKoykRknqqmex2HV6z91n5rf9m337qqjDHGlIolDmOMMaViiaPyG+t1AB6z9ldt1v4AsDEOY4wxpWJ3HMYYY0rFEocxxphSscRRSYlIsojMEpHlIrJMRO7yOqbyJiLBIrJQRD72OhYviEisiEwSkZUiskJEunodU3kSkXvc//eXisi7IhLhdUyBJCKvichOEVnqU1ZLRP4nImvcnzXL4rsscVReecB9qpoGdAF+KyJpHsdU3u4CVngdhIeeA6aranOgDVXoz0JEEoE7gXRVPR8IBgZ7G1XAvQH0KVI2Epipqk2Bme7+WbPEUUmp6jZVXeB+PojzSyPR26jKj4gkAVcCr3gdixdEJAa4CHgVQFWPqeo+T4MqfyFANREJASKBrR7HE1Cq+iWwt0hxf2Cc+3kccHVZfJcljipARFKAdsAcj0MpT88CDwAFHsfhlUbALuB1t7vuFRGp7nVQ5UVVtwBPA5uAbcB+Vf3U26g8kaCq29zP24GEsrioJY5KTkSigMnA3ap6wOt4yoOIXAXsVNX5XsfioRCgPfCSqrYDDlFG3RTnArcvvz9OAq0PVBeRm72NylvqPHtRJs9fWOKoxEQkFCdpvK2qH3gdTznqDvQTkQ3ABKCHiLzlbUjlLgvIUtXCu8xJOImkqrgU+FFVd6nqceADoJvHMXlhh4jUA3B/7iyLi1riqKRERHD6t1eo6j+8jqc8qeqDqpqkqik4A6Kfq2qV+temqm4HNotIM7eoJ7Dcw5DK2yagi4hEun8XelKFJgf4mAIMcz8PA/5TFhe1xFF5dQeG4Pxre5G7XeF1UKZc/Q54W0QWA22Bv3kbTvlx77QmAQuAJTi/6yr18iMi8i7wLdBMRLJE5FbgcaCXiKzBuQt7vEy+y5YcMcYYUxp2x2GMMaZULHEYY4wpFUscxhhjSsUShzHGmFKxxGGMMaZULHEYU4GJSEZVXd3XVFyWOIwxxpSKJQ5jyoCI3Cwi37sPWv7LfRdIjoj8030nxEwRqePWbSsi34nIYhH5sPAdCSKSKiKficgPIrJARJq4l4/yea/G2+6T0MZ4xhKHMWdJRFoAg4DuqtoWyAduAqoD81S1JfAF8Gf3lDeBP6hqa5ynmgvL3wbGqGobnHWVClc1bQfcDaQBjXFWBTDGMyFeB2BMJdAT6ADMdW8GquEsJlcATHTrvAV84L4nI1ZVv3DLxwHvi0g0kKiqHwKoai6Ae73vVTXL3V8EpABfB7xVxpTAEocxZ0+Acar64E8KRf5UpN6Zru9z1OdzPvb31njMuqqMOXszgQEiEg8n3vPcEOfv1wC3zo3A16q6H8gWkQvd8iHAF+5bGrNE5Gr3GuEiElmejTDGX/YvF2POkqouF5GHgU9FJAg4DvwW5+VJndxjO3HGQcBZ3vplNzGsB37hlg8B/iUij7rXGFiOzTDGb7Y6rjEBIiI5qhrldRzGlDXrqjLGGFMqdsdhjDGmVOyOwxhjTKlY4jDGGFMqljiMMcaUiiUOY4wxpWKJwxhjTKn8P6ZNnGDdWTWLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#!g1.1\n",
    "\n",
    "from IPython.display import clear_output\n",
    "\n",
    "for param in classifier.base_model.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "classifier.train()\n",
    "loss_fn = F.binary_cross_entropy\n",
    "optimizer = torch.optim.Adam(classifier.parameters(), lr=1e-4)\n",
    "epochs = 10\n",
    "\n",
    "train_loss = []\n",
    "val_loss = []\n",
    "    \n",
    "for epoch in range(epochs):\n",
    "    print(f\"Epoch {epoch+1}\\n-------------------------------\")\n",
    "    \n",
    "    train_loop(classifier, big_train_dl, loss_fn, optimizer, history_loss=train_loss)\n",
    "    test_loop(classifier, big_test_dl, loss_fn, history_loss=val_loss)\n",
    "    \n",
    "    clear_output()\n",
    "    plot_learning_process(train_loss, val_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "tr49elk25f4hlsochz07w"
   },
   "source": [
    "Видно, что 10 эпох недостаточно и хотелось бы потренироваться еще немного. Так и сделаем (предварительно сохранив классификатор на всякий случай):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "cellId": "xwtyveu2c3zuzq5eacqs"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "classifier.save_pretrained('./big_models')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "cellId": "wscjyljcn4xakf6wku95s"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8A0lEQVR4nO3deXwV5dXA8d/Jvq9AAgmQsBN2QpBFNlnEBbFWBVTcpdraV19tq7ZWW2tbbd9qW+uGS1VEwV1UFEUBN5Ad2fct7AmQBQhkOe8fM2CISbgJufdCcr6fz3zunZlnZs7Mvbkn8zwzz4iqYowxxngqwN8BGGOMObtY4jDGGFMjljiMMcbUiCUOY4wxNWKJwxhjTI1Y4jDGGFMjljhMvSYiW0RkmI+29ZKIPOyLbdU1ERksItn+jsOcHSxxGGOMqRFLHMYYY2rEEodpMEQkVET+KSI73eGfIhLqzmskIh+KyEER2S8iX4lIgDvvHhHZISIFIrJWRIZWs5lGIvKZW3aOiLR01/GkiPyjQjzTROR/q4i1g7ue/e42ryw37yUReaay7bjz+4nIAhHJc1/7lZuXICL/dff/gIi8V2G7d4vIXhHZJSI3eH50TUNiicM0JL8D+gDdgW5Ab+B+d97dQDbQGEgCfguoiLQHbgeyVDUaOB/YUs02rgb+BDQClgKT3ekvA+PKJaNGwDDgtYorEJFI4DN3XhNgLPCUiGScajsikgB8BPwbSAQeAz4SkUR3uUlABNDJXffj5daZDMQCKcBNwJMiEl/NvpoGyhKHaUiuBh5S1b2qug/4IzDenVcMNAVaqmqxqn6lTkdupUAokCEiwaq6RVU3VrONj1T1S1U9ipOo+opIc1WdD+QBx89WxgKzVXVPJeu4GNiiqv9V1RJVXQK8DVxxqu0AFwHrVXWSu+zrwBpglIg0BS4AblXVA+5+zim3zmL3+BSr6nSgEGh/yqNqGhxLHKYhaQZsLTe+1Z0G8HdgA/CpiGwSkXsBVHUDcCfwB2CviEwRkWZUbfvxN6paCOwvt42XgWvc99fg/PdfmZbAOW612UEROYiT9JI92E7FfTy+nylAc2C/qh6oYru5qlpSbvwwEFVFWdOAWeIwDclOnB/l41q401DVAlW9W1VbAZcAdx1vy1DV11T1XHdZBR6tZhvNj78RkSgg4fg2gFeB0SLSDegIvFfFOrYDc1Q1rtwQpaq3ebCdivt4fD93uOtNEJG4auI35pQscZiG5HXgfhFp7LYxPIDzY46IXCwibUREcKqUSoEyEWkvIue5jehFwBGgrJptXCgi54pICE4bxDxV3Q6gqtnAApwzjbdV9UgV6/gQaCci40Uk2B2yRKSjB9uZ7i57lYgEicgYIAP4UFV3AR/jtJfEu+sdWNODaIwlDtOQPAwsBL4HlgOL3WkAbYGZOPX6c4GnVHUWTvvGI0AOsBunQfm+arbxGvAgTtVRJj9UTR33MtCFqqupUNUCYAROO8hOd7uPurFUux1VzcVpI7kbyAV+A1ysqjnucuNx2jLWAHtxquGMqRGxBzkZ4zvuf/iv4jTC1+qPT0ReArJV9f5TlTXGG+yMwxgfEZFg4A7g+domDWPOBJY4jPEBt33iIM4lv//0azDGnCarqjLGGFMjdsZhjDGmRoL8HYAvNGrUSNPS0iqdd+jQISIjI30bkIcsttqx2GrHYqud+hzbokWLclS18Y9mqGq9HzIzM7Uqs2bNqnKev1lstWOx1Y7FVjv1OTZgoVbym2pVVcYYY2rEEocxxpgascRhjDGmRhpE47gxxtRUcXEx2dnZFBUVVVsuNjaW1atX+yiqmvE0trCwMFJTUwkODvZovZY4jDGmEtnZ2URHR5OWlobT92XlCgoKiI6O9mFknvMkNlUlNzeX7Oxs0tPTPVqvVVUZY0wlioqKSExMrDZp1AciQmJi4inPrMqzxGGMMVWo70njuJrupyWOakyau4UPlu08dUFjjGlALHFU481F2bw6r+JTOI0xxvsOHjzIU089VePlLrzwQg4ePFj3AZVjiaMaWWkJLN1+kKMlpf4OxRjTwFSVOEpKSiop/YPp06cTFxfnpagcljiqkZWWwNGSMlbsyPN3KMaYBubee+9l48aNdO/enaysLAYMGMAll1xCRkYGAJdeeimZmZl06tSJiRMnnlguLS2NnJwctmzZQq9evbjlllvo1KkTI0aM4MiRqp5WXDN2OW41stLiAZi/+QCZLRP8HI0xxl/++MFKVu3Mr3ReaWkpgYGBNV5nRrMYHhzVqcr5jzzyCCtWrGDp0qXMnj2biy66iBUrVpy4ZPbFF18kISGBI0eOkJWVxU9/+lMSExNPWsfGjRuZOnUqzz33HFdeeSVvv/0211xT8WnGNWdnHNVIjAqldeNIFmzZ7+9QjDENXO/evU+6z+Lf//433bp1o0+fPmzfvp3169f/aJmWLVvSvXt3ADIzM9myZUudxGJnHKfQOz2BD7/fRWmZEhjQMC7NM8acrLozA1/dAFi+e/TZs2czc+ZM5s6dS0REBIMHD670PozQ0NAT7wMDA+usqsrOOE4hKy2BgqIS1u4u8HcoxpgGJDo6moKCyn938vLyiI+PJyIigjVr1jBv3jyfxmZnHKfQO91p21iwZT8ZzWL8HI0xpqFITEykf//+dO7cmfDwcJKSkk7MGzlyJM888wwdO3akffv29OnTx6exWeI4hdT4CJrFhjF/y36u65fm73CMMQ3Ia6+9Vun00NBQPv7440rnHW/HaNSoEd99992J6b/61a/qLC6rqvJAVnoCCzbvx3kgljHGNGyWODyQlZbA3oKjbNt/2N+hGGOM31ni8MDxdo75m+2yXGOMscThgTaNo4iLCLb7OYwxBkscHgkIEHq1TGDBlgP+DsUYY/zOq4lDREaKyFoR2SAi91YyP1REprrzvxORNHf6cBFZJCLL3dfzyi0zzp3+vYh8IiKNvLkPx/VOj2dzziH2Fnj+sBNjjKmPvJY4RCQQeBK4AMgAxolIRoViNwEHVLUN8DjwqDs9Bxilql2A64BJ7jqDgH8BQ1S1K/A9cLu39qG8rDT3fo7NdtZhjDnzREVF+Wxb3jzj6A1sUNVNqnoMmAKMrlBmNPCy+/4tYKiIiKouUdXjT1BaCYSLSCgg7hApziOrYgCfPGmpc0os4cGB1s5hjGnwvHkDYAqwvdx4NnBOVWVUtURE8oBEnDOO434KLFbVowAichuwHDgErAd+UdnGRWQCMAEgKSmJ2bNnVxpkYWFhlfMqSo9Rvli+jcEx+zwqf7pqEpuvWWy1Y7HVjj9ii42NrbLLj/JKS0s9KldTDz74ICkpKUyYMAGAv/zlLwQFBfHVV19x8OBBiouL+f3vf89FF110YpmKcdQktqKiIo+P8Rl957iIdMKpvhrhjgcDtwE9gE3AE8B9wMMVl1XVicBEgF69eungwYMr3cbs2bOpal5FS0vW8a/P19OzT39iwoJruDc1V5PYfM1iqx2LrXb8Edvq1at/6Lzw43th9/JKy5WUlhAUWIuf0uQucMEjVc4eP348d955J3fffTcA77//PjNmzODXv/41MTEx5OTk0KdPH8aMGXPimeEVO1usSQeMYWFh9OjRw6Oy3qyq2gE0Lzee6k6rtIzbfhEL5LrjqcC7wLWqutEt3x1AVTeqcxv3G0A/L8X/I73TElCFRVutncMY4109evRg79697Ny5k2XLlhEfH09ycjK//e1v6dq1K8OGDWPHjh3s2bPH57F584xjAdBWRNJxEsRY4KoKZabhNH7PBS4HvlBVFZE44CPgXlX9plz5HUCGiDRW1X3AcGC1F/fhJD1axBMUICzYvJ8h7Zv4arPGGH+r5szgiBe7Vb/iiit466232L17N2PGjGHy5Mns27ePRYsWERwcTFpaWqXdqXub1xKH22ZxOzADCAReVNWVIvIQsFBVpwEvAJNEZAOwHye5gHOlVBvgARF5wJ02QlV3isgfgS9FpBjYClzvrX2oKDwkkM4psdZAbozxiTFjxnDLLbeQk5PDnDlzeOONN2jSpAnBwcHMmjWLrVu3+iUur7ZxqOp0YHqFaQ+Ue18EXFHJcg9TSbuFO+8Z4Jm6jdRzvdMTeOmbLRQVlxIWXPPHRRpjjKc6depEQUEBKSkpNG3alKuvvppRo0bRpUsXevXqRYcOHfwS1xndOH4mykpLYOKXm1i2/SDntEo89QLGGHMali//oVG+UaNGzJ07t9JyhYWFvgrJuhypqV4t4wGsusoY02BZ4qih+MgQ2idFM9/6rTLGNFCWOGohKz2exVsPUFpmD3Yypj5rKA9vq+l+WuKohay0BAqPlrB6V76/QzHGeElYWBi5ubn1PnmoKrm5uYSFhXm8jDWO10L5Bzt1Ton1czTGGG9ITU0lOzubffuq72KoqKioRj+6vuRpbGFhYaSmpnq8XksctdA0NpzU+HAWbNnPjeem+zscY4wXBAcHk55+6r/v2bNne9xVh695Kzarqqql3mkJzN+8v96fxhpjTEWWOGopKz2B3EPH2JRzyN+hGGOMT1niqKUfHuxk93MYYxoWSxy11LpxJImRIcy3GwGNMQ2MJY5aEhGy0hLsDnJjTINjieM0ZKUnsH3/EXbn+b5bY2OM8RdLHKeht9vOYdVVxpiGxBLHaejYNJrIkEBrIDfGNCiWOE5DUGAAPVvEMd8ShzGmAbHEUVuF++DFC7g6ciFr9xRw8PAxf0dkjDE+YYmjtiISIT+bPvmfALDQulk3xjQQljhqKyAAul1F7K6vaR643y7LNcY0GJY4Tke3sQjKbfEL7MoqY0yDYYnjdCSkQ8tzGVnyBcuzD3LkWKm/IzLGGK+zxHG6ul9FQtF2uuo6lmy3dg5jTP1nieN0ZYxGgyO5MmgOCzZb4jDG1H+WOE5XaBTS6VIuCZzHsk07/R2NMcZ4nSWOutD9KiI4QmL2pxSXlvk7GmOM8SpLHHWhRT8ORaRyic5m5c58f0djjDFeZYmjLgQEoN2uon/ASlatXunvaIwxxqsscdSRqN7XECBKxOo3/R2KMcZ4lSWOuhLfkg2RPeh5YDpl1s5hjKnHLHHUody2l9OCPez4/gt/h2KMMV5jiaMOJZ8zhkIN4+iiV/0dijHGeI0ljjrUIrkRnwf0I3XnJ3DskL/DMcYYr7DEUYdEhI0powkrOwKrP/B3OMYY4xWWOOpYQoeBbClLomjBK/4OxRhjvMISRx3LapXIW6UDCcv+Bg5s9Xc4xhhT5yxx1LEOyTHMCBpCGQLLpvg7HGOMqXOWOOpYYICQktaWpYFdYelkKLN7Oowx9YslDi/onZ7AK0f6w8GtsO1bf4djjDF1yquJQ0RGishaEdkgIvdWMj9URKa6878TkTR3+nARWSQiy93X88otEyIiE0VknYisEZGfenMfaqN3WgKflGVREhQJS1/zdzjGGFOnvJY4RCQQeBK4AMgAxolIRoViNwEHVLUN8DjwqDs9Bxilql2A64BJ5Zb5HbBXVdu5653jrX2orS6psZQFhbM8biisfA+OFvo7JGOMqTPePOPoDWxQ1U2qegyYAoyuUGY08LL7/i1gqIiIqi5R1eNPRVoJhItIqDt+I/BXAFUtU9UcL+5DrYQGBdK9eRxTigdA8SFYPc3fIRljTJ0RVfXOikUuB0aq6s3u+HjgHFW9vVyZFW6ZbHd8o1smp8J6blXVYSISBywH3gQGAxuB21V1TyXbnwBMAEhKSsqcMqXyK5wKCwuJioo6/R2u4O11x/ho8zGWx/6a4tAElvb4c43X4a3Y6oLFVjsWW+1YbLVzurENGTJkkar2+tEMVfXKAFwOPF9ufDzwnwplVgCp5cY3Ao3KjXdyp7V2xxsBClzujt8FTDpVLJmZmVqVWbNmVTnvdMxZu1db3vOhrn3jQdUHY1RzN9V4Hd6KrS5YbLVjsdWOxVY7pxsbsFAr+U31ZlXVDqB5ufFUd1qlZUQkCIgFct3xVOBd4FpV3eiWzwUOA++4428CPb0R/Onq2zqRlLhwHtvbA+yeDmNMPeLNxLEAaCsi6SISAowFKlb2T8Np/AbnDOULVVW3Suoj4F5V/eZ4YTcDfoBTTQUwFFjltT04DcGBAdw8IJ1PtgeR16w/LHvN7ukwxtQLXkscqloC3A7MAFYDb6jqShF5SEQucYu9ACSKyAacaqfjl+zeDrQBHhCRpe7QxJ13D/AHEfkep/rrbm/tw+kak9Wc+IhgphYPgIPbYOvX/g7JGGNOW5A3V66q04HpFaY9UO59EXBFJcs9DDxcxTq3AgPrNlLviAgJ4rp+aTw2s5CbYqIJXPoapJ8VoRtjTJVOecYhIo1F5LfuTXcvHh98EVx9cF3fNCQ4gvkRg2HV+3C0wN8hGWPMafGkqup9nEbrmTjtDscH44H4yBDGZDXnsX29oPiwkzyMMeYs5klVVYSq3uP1SOqxmwekM2heW3JDW5C4ZDL0uMbfIRljTK15csbxoYhc6PVI6rHU+Agu6ZbCK0f6OZ0e5m489ULGGHOG8iRx3IGTPIpEpMAd8r0dWH3zs0GtmHLsXMoIsHs6jDFntVMmDlWNVtUAVQ1z30eraowvgqtPOiTHkNG+PfPoii61ezqMMWcvj+7jEJFLROT/3OFibwdVX906qDWvHzsXyc+GLV/6OxxjjKkVTy7HfQSnumqVO9whIn/1dmD1Ue/0BPamDKWACMqWTPZ3OMYYUyuenHFcCAxX1RdV9UVgJHCRd8Oqn0SEGwdn8H5JX8pWTYMiayoyxpx9PO1yJK7c+1gvxNFgDO+YxNyY8wkqLUJXvuvvcIwxpsY8SRx/BZaIyEsi8jKwCKj5wyUMAAEBwqDBF7ChrBn5814+9QLGGHOG8eSqqteBPjhdmb8N9FXVqd4OrD4b3TOFj4OHErtvEWxf4O9wjDGmRqpMHCLSwX3tCTQFst2hmTvN1FJoUCCR/X/GHo3j8Pt326W5xpizSnVdjtyF8+jVf1QyT4HzvBJRA3FFvw785ctr+GvOf2DZ69Djan+HZIwxHqkycajqBPftBW735yeISJhXo2oAosOCiT/nahbP/YSunz5IUMdREGb3VRpjznyeNI5/6+E0U0M3nNuaP5fdQNCRffDl3/0djjHGeKS6No5kEckEwkWkh4j0dIfBQISvAqzPGkeH0j5zEG+VDUbnPQ05G/wdkjHGnFJ1bRznA9cDqTjtHOJOzwd+692wGo4JA1px5fwrGRU0n9AZv4Wr3/B3SMYYU60qzzhU9WVVHQJcr6rnqeoQdxitqu/4MMZ6La1RJFmdO/JE6WWwfgas/8zfIRljTLU8aePIFJG44yMiEi8ilT4P3NTOrYNa8+zRERwMbwGf3Aslx/wdkjHGVMmTxHGBqh48PqKqB3D6rzJ1pEtqLOe0SeaPx66B3A0w/1l/h2SMMVXyJHEEikjo8RERCQdCqylvauHWQa1591BndjY+F+b8DQr3+jskY4yplCeJYzLwuYjcJCI3AZ8B1slSHevfJpHOKTHce2gcWnwYPn/I3yEZY0ylPOmr6lGcTg07usOfVPVv3g6soRERbh3Umi/3x7O59XhY8ipRBXZ5rjHmzONRt+qq+rGq/sodZng7qIbqgs5NaZkYwe9yL0AjG9F2/XOg6u+wjDHmJJ48AfAyEVkvInkiki8iBSJiTyDygsAA4ZYBrZi7s4SNXe8mNn8NLH/T32EZY8xJPDnj+BtwiarGqmqMqkarqnWq5CWXZ6bSKCqEP2X3oCCqNXz2ABwt9HdYxhhzgieJY4+qrvZ6JAaAsOBAbuifzpz1uXzV7CYo2AVfP+bvsIwx5gRPEsdCEZkqIuPcaqvLROQyr0fWgF1zTksiQwJ5aW9r6HIlfPsf2L/Z32EZYwzgWeKIAQ4DI4BR7nCxN4Nq6GIjgrm+fxrzd5fyZcvbISAIPr3f32EZYwxQfSeHAKjqDb4IxJzsf4a25cNFm/mf6Xv4su//EPPNX2HTbGg12N+hGWMaOE+uqvqviLxYcfBFcA1ZaFAgt3UL5VhJGbdt6IvGp8HH90Jpib9DM8Y0cJ5UVX0IfOQOn+NUXdllPj6QHBnAw5d25puthXyQ9AvYtxoWvuDvsIwxDZwnVVVvlx8XkdeBr70WkTnJZT1T+Xp9DncuVQa17EfsrD9D58shMtHfoRljGiiP7hyvoC3QpK4DMVV76NLOtEyM4tacK9GjhTDLerU3xviPJ20cBe4d4/nuHeMfAPd4PzRzXFRoEE+M68HCI0l8ET0KXfQS7F7u77CMMQ1Udc8c7+++bezeMX58aFex+sp4X+eUWO67oCP/u/dCjgbFwMf3WD9Wxhi/qO6M49/u67e+CMSc2g3908jq0Iq/FP0Utn4DK+0JvsYY36sucRSLyEQgVUT+XXHwZOUiMlJE1orIBhG5t5L5oe5d6RtE5DsRSXOnDxeRRSKy3H09r5Jlp4nICg/3s14QEf5+RTc+CzufdQGt0Wm/hOyF/g7LGNPAVJc4Lga+AI4AiyoZqiUigcCTwAVABjBORDIqFLsJOKCqbYDHgUfd6TnAKFXtAlwHTKqw7stooJcEJ0SG8NjYXlx75H/J1Vh49TLY9b2/wzLGNCBVJg5VzVHVKTg9475ccfBg3b2BDaq6SVWPAVOA0RXKjOaHpwm+BQwVEVHVJaq6052+Egg//vhaEYkC7gIa7KVFfVsncuWQ3lxaeA+HJRwmXQp71/g7LGNMAyHqpQZWEbkcGKmqN7vj44FzVPX2cmVWuGWy3fGNbpmcCuu5VVWHueOPA18CS4APVbVzFdufAEwASEpKypwyZUqlcRYWFhIVFXW6u+sV1cVWWqY8uqAIyd/BexEPEyCwtPtfOBLRzO+x+ZvFVjsWW+3U59iGDBmySFV7/WiGqnplAC4Hni83Ph74T4UyK4DUcuMbgUblxju501q7492Bae77NGCFJ7FkZmZqVWbNmlXlPH87VWw7DhzWrn+Yobc9NlnLHk1X/UeG6oGtZ0Rs/mSx1Y7FVjv1OTZgoVbym1qbGwA9tQNoXm481Z1WaRkRCQJigVx3PBV4F7hWVTe65fsCvURkC87d6+1EZLaX4j/jNYsL52+Xd2X6nlieT/sHHCuAl0dB/s5TL2yMMbXkyQ2Ad4hIjDheEJHFIjLCg3UvANqKSLqIhABjgWkVykzDafwG5wzlC1VVEYnD6RvrXlX95nhhVX1aVZupahpwLrBOVQd7EEu9dX6nZK7t25I/Lw5h0YDn4VAOvDIaCvf5OzRjTD3lyRnHjaqaj/M8jnicKqdHTrWQqpYAtwMzgNXAG6q6UkQeEpFL3GIvAIkisgGnwfv4Jbu3A22AB0RkqTtYNydV+O2FHemQHM0tXwj7L30VDm53GswP7/d3aMaYesiTxCHu64XAJFVdWW5atVR1ujp3mrdW1T+70x5Q1Wnu+yJVvUJV26hqb1Xd5E5/WFUjVbV7uWFvhXVv0SoaxhuasOBA/nNVT44cK+UXX4dTOuY1yFnvXKpblOfv8Iwx9YwniWORiHyKkzhmiEg0UObdsExNtWkSxR9Hd2Luplye3t4crnzF6c9q8pVw7JC/wzPG1COeJI6bcKqQslT1MBAM2FMBz0BXZKZySbdmPD5zPQtDe8NPX4Ds+fD6WCg+4u/wjDH1hCeJoy+wVlUPisg1wP2A1X+cgUSEP/+kMylx4dwxZSl56RfBpc/A5q/gjWuh5Ji/QzTG1AOeJI6ngcMi0g24G+e+ile8GpWpteiwYJ4Y14O9BUXc8NJ88tpdBhc/Dus/hbdvtEfPGmNOmyeJo8S9EWQ0zg18TwLR3g3LnI5uzeN4YlwPlu/I4+rn57G/49Uw8hFY/QG8dyuUlfo7RGPMWcyTxFEgIvfhXIb7kYgE4LRzmDPYyM5NmTi+F+v2FDJ24lz2droBhj4Iy9+ED++EMru+wRhTO54kjjHAUZz7OXbj3AH+d69GZerEkA5N+O/1WWzff4Qxz85jZ5fbYOBvYPEr8Ik9CMoYUzunTBxuspgMxIrIxUCRqlobx1mif5tGTLqpNzkFR7ny2bls63on9L0d5k+EKVfDoVx/h2iMOct40uXIlcB84ArgSuA7t8dac5bolZbA5FvOofBoCVdOnMeG7vfCiD/Dhs/g6X6wcZa/QzTGnEU8qar6Hc49HNep6rU4z9n4vXfDMnWta2ocr9/Sh5KyMsY+N481ra6Fmz+HsBine5JP77fLdY0xHvEkcQRU6O4j18PlzBmmY9MYpkzoS2CAMHbiPL4vbQET5kDmDfDtE/D8UKerEmOMqYYnCeATEZkhIteLyPU4vdZO925YxlvaNInizZ/1Iyo0iKuf+46FO4tg1D9hzGTIy4ZnB8Kil6zh3BhTJU8ax38NTAS6usNEVb3H24EZ72mRGMEbP+tLo+hQxr8wn2835EDHi+G2byE1Cz64A6ZeY73rGmMq5VGVk6q+rap3ucO73g7KeF+zuHCm/qwPzRPCueGlBcxauxdimsL492D4n2DdDHi6P2ya4+9QjTFnmCoTh4gUiEh+JUOBiOT7MkjjHU2iw5gyoS9tk6KY8MpCPlmxCwICoP//wM0zISTSeSjUZw9Yw7kx5oQqE4eqRqtqTCVDtKrG+DJI4z0JkSFMvrkPnVNi+cVrS3h/qft032bd4WdzoOe18M2/4IXhkLPBr7EaY84MdnWUITY8mEk3nUNWWjx3Tl3K1AXbnBkhkXDJv2HMq3BwKzw7wLnr3BrOjWnQLHEYAKJCg/jv9b0Z0LYx97y9nJe+2fzDzI6j3IbzXjDtl/DGtQQVF/gvWGOMX1niMCeEhwTy3LWZDM9I4g8frOJ/py4lv6jYmRnTDMa/D8P+CGun02vhnZC9yK/xGmP8wxKHOUloUCBPX92TO4e1ZdqynVzwz6+Yt8ntzyogAM69E276DJUAePF8WPC8VV0Z08BY4jA/EhQYwJ3D2vHWrX0JDhTGPTePv05fzdES9zkeKT1ZlPkYtBoMH90N794Kxw77NWZjjO9Y4jBV6tEinul3DGBc7xY8++UmRv/nG9bsdq7ELgmOhqvegMG/he+nwvPDIHejnyM2xviCJQ5TrYiQIP7yky68cF0vcgqPcskT3/D8V5soU3WqrgbfA1e/BQU7YeJgWPORv0M2xniZJQ7jkaEdk/jkzoEMbNeYhz9azd8XFLHz4BFnZtthTmeJCa1gylUw8w/2bHNj6jFLHMZjjaJCee7aTB65rAub8soY+c8vmbZspzMzviXcOAMyr4evH4dXfwKF+/warzHGOyxxmBoREcb2bsGf+ofTpkkU//P6Eu6YsoS8w8UQHAaj/gWjn4Lt852edrcv8HfIxpg6ZonD1EqTiADe+Flf7h7ejo++38XIf33p9LIL0ONquOkzCAyG/14A3020S3aNqUcscZhaCwoM4JdD2/LOz/sRHhLIVc9/x8MfrqKouBSadnX6ump9Hnz8a3hnAhw75O+QjTF1wBKHOW1dU+P46JcDGN+nJc9/vZlLn/yGlTvzIDwexk2B8+6H5W86l+xaR4nGnPUscZg6ER4SyJ8u7cx/b8gi99AxRj3xNX+YtpK8o6Uw8Ncw/h0o2O1csrv6A3+Ha4w5DZY4TJ0a0r4JM/93ENf0ackrc7dw3v/N5o0F2ylLHwI/+xIatXWeLjjjd1Ba7O9wjTG1YInD1LnYiGAeGt2ZD355LumNIvnN29/zk6e/5fvCaLjxE8i6Geb+x2k4P7jN3+EaY2rIEofxmk7NYnnz1r48dmU3dhw4wugnv+G+aevYP/ivcMVLsHcNPDMA1n7s71CNMTVgicN4lYhwWc9UZv1qEDf1T+eNhdsZ8n+zmVTQk9IJc5wbB18f61Rd2eNpjTkrWOIwPhEdFsz9F2fw8R0DyGgaw+/fW8GoyTtZPHwq9J5gVVfGnEUscRifapcUzWu3nMN/rurBgcPHuGziYu4qvJq8Uc9Dzjp45lzrKNGYM5wlDuNzIsLFXZsx865B/Hxwaz5YtpP+02KY2vNVyuLTnY4SP7nPqq6MOUNZ4jB+ExkaxG9GdmDGnQPJbBnPPbMKubjw9+xqfy3Me8p5wuCBLf4O0xhTgVcTh4iMFJG1IrJBRO6tZH6oiEx1538nImnu9OEiskhElruv57nTI0TkIxFZIyIrReQRb8ZvfKNV4yheuiGLieMzyS8JoO+ykfwt9n6O7V2PPjPAbhg05gzjtcQhIoHAk8AFQAYwTkQyKhS7CTigqm2Ax4FH3ek5wChV7QJcB0wqt8z/qWoHoAfQX0Qu8NY+GN8REUZ0SmbmXYO4/6KOfFKaxdBDD7HqaGOYeg15795tVVfGnCG8ecbRG9igqptU9RgwBRhdocxo4GX3/VvAUBERVV2iqu6DHlgJhItIqKoeVtVZAO46FwOpXtwH42NhwYHcPKAVn989iEdvuoSJrZ/iv6UjiV32PJse7c/sufMpLi3zd5jGNGiiXuruWkQuB0aq6s3u+HjgHFW9vVyZFW6ZbHd8o1smp8J6blXVYRXWH4eTOIap6qZKtj8BmACQlJSUOWXKlErjLCwsJCoq6nR21WssNsfBojIOrP+Gy3OfQhUe5Xoik9rQvVk00VFRlAaGgfzwP5Adt9qx2GqnPsc2ZMiQRaraq+L0oNOKystEpBNO9dWICtODgNeBf1eWNABUdSIwEaBXr146ePDgSrcxe/ZsqprnbxZbOSPPozT3Ggonj+fP+5+CvTgDoAiERiOhMRAWQ15RGbFNmkNoNITFOK+hsT+MN+sBTTr6LvZy7DOtHYutdrwVmzcTxw6gebnxVHdaZWWy3WQQC+QCiEgq8C5wrapurLDcRGC9qv7TC3GbM1RgYjqxP/8Ctn5Dzr7dLFq3ldVbdiDHCmgWeIzOcQG0ii6l7Fg2HM6BA5uhKB+OFkDJkZNX1uFip9feZt39si/GnM28mTgWAG1FJB0nQYwFrqpQZhpO4/dc4HLgC1VVtxrqI+BeVf2m/AIi8jBOgrnZi7GbM1VQCLQeQqPWcH4fGFJSxmer9vDqvK38elMuwYFC98YB3DqgBwPbNSY40K3CKjkGxwrhyAHn2SDznoI1H0Lb82HQbyD1R2fjxpgqeC1xqGqJiNwOzAACgRdVdaWIPAQsVNVpwAvAJBHZAOzHSS4AtwNtgAdE5AF32gggBPgdsAZYLCIA/1HV5721H+bMFhIUwEVdm3JR16Zs2FvIa99t4835m7np5YUkRoYwqlszLuuZQpeUWCQiASISYPC90Oc2mP8czH0Snh8KrYbAoHugZV9/75IxZzyvtnGo6nRgeoVpD5R7XwRcUclyDwMPV7FaqcsYTf3RpkkUD4zKoF/kHkjO4J0l2bw2fxsvfbuF1o0juaxnKqO7NyM1PgLCYmHgr+CcW2HhC/DtE/DfkZA2wDkDSRsAYl81YypzRjeOG1MbQQHC4IwkhmUkkXekmOnLd/Hu4h38fcZa/j5jLeekJ/DTnqlc0CWZ6LAo6H8HZN0Ci16Cb/4FL4+CFn2dNpDW51kCMaYCSxymXosND2Zc7xaM692C7fsP896SHby7ZAe/eft7fv/+CoZnJHFZzxQGtG1McN+fQ68bYckk+PpxePUySOnlnIG0HWEJxBiXJQ7TYDRPiOCXQ9ty+3ltWLr9IO8u2cEHy3by4fe7Tm4PyboZ6XktLH0Nvn4MXrsSmnaDgb+B9hdCgHXxZho2SxymwRERerSIp0eLeO6/KIM56/bxbrn2kJS4cAa2a8SgdiPpd8sYYta9A1/9A6ZeDU06QcdR0KIPpGZB6Jl545cx3mSJwzRoIUEBDM9IYrjbHvLJil3MWrOPD5ft4vX52wkMEHq2aMWgjNcYFTiXFutfQb78G2gZSCAkd3HaQ1r0cYboZH/vkjFeZ4nDGFdseDBjslowJqsFxaVlLN1+kDlr9/Hl+n384/NN/J8mkRD5O4a1DmdU4g56yhoidy9wGtW/e9pZSXx6uUTSFxq1tbYRU+9Y4jCmEsGBAWSlJZCVlsCvzm9PbuFRvt6Qw5x1+/hiXQ5vrIgCetGx6XkM6RHPyEZ7yCheRVD2PFj/KSx7zVlReMJJiSSw5LBzE2JpMZQec4ficuMVp5d7X1YMaedCQiu/HhtjLHEY44HEqFBGd09hdPcUysqU1bvzmbNuH1+u28dz327jqVIlIqQt/Vr3YfjgJoxIPkR8ziLYNg+2zYW1zuNwBwB8fRqBBARD71ucS4UjEupi14ypMUscxtRQQIDQqVksnZrF8vPBbSg8WsLcjbl8uW4fX6zZy8zVe7lPoFfLNozo1J/zBybTPKQQts1j48KZtG7XEQKDITDEHdz3AcGVTz/+vrQY5j4B3z3jXPE16B7IutnphsUYH7LEYcxpigoNOtHA/pAqq3blM2PlHj5duZuHP1rNwx+tpmPTGM7v1IGEuEhanXMeUtt2j0uegN4/g0/vhxn3wYLnYPifoMNF1pZifMYShzF1SOSHs5G7hrdja+4hPl25h09X7eZfn69HFZ5bM4sRGcmc3ymZzJbxBAbU8Ac/uTOMfxc2zIQZv3MuE27ZH87/s9NlvDFeZonDGC9qmRjJLQNbccvAVuwrOMpT733JlpIoJs3dygtfbyYxMoRhHZM4v3MS/Vo3Iiw40LMVi0Db4U7njItfgll/hYmDoetYGPoAxKZ4c7dMA2eJwxgfaRwdyqDmwTw4uDeFR0uYvXYvn67cw/Tlu5i6cDuRIYH0bZ1Iz5bx9GwRT9fUWCJCTvEnGhjktHN0uQK+egzmPQ2r3od+v3T64LIbFI0XWOIwxg+iQoO4uGszLu7ajKMlpczbtJ8ZK3czb1MuM1c7jzYMDBA6No2mZ4v4E0PzhPDK20fCYmH4H52+tj7/I3z5N1j8Mpx3P3S/GgI8PJMxxgOWOIzxs9CgQAa1a8ygdo0BOHDoGEu2H2Dx1oMs3naAtxdl88rcrQA0igqhx4lEEkfX1DjCQ8olhfiWcPmLcM5tMOO3MO2X8N2zMOJhaD3EH7tn6iFLHMacYeIjQzivQxLndUgCoLRMWbu7gMXbDrB42wGWbDvIZ6v2AE4X8h2bxtCzRRw9W8bTOSWWtMRIAptnwU2fwsp3YeYfYNKl0Ga4cw9I66FOFZcxtWTfHmPOcIEBQkazGDKaxXBNn5YA7D90jCVuIlm89SBvLsrmZfesJDw4kPbJ0c4yTTPJuORTOmVPIXTev52efiMbQ5crodtYaNrVn7tmzlKWOIw5CyVEhjC0YxJDOzpnJSWlZazbU8iqXfms2pnPql15fPT9Ll77bhsAIm1pm/AsP222mmHHZpE+fyIB855Ekzoh3cY5icQYD1niMKYeCAoMOHFWQqYzTVXZmVfkJJKd+azelc/kXV346/7WxDGGiwPnMXbPV3T+9H7KPn2A1IhubAu4icaZPyE80q7GMlWzxGFMPSUipMSFkxIXzvCMpBPT84uKWbOrgNW7+jBp543kZ6+k2/6PueTQVzT74nbyP/81nwefy4rGFyIt+tIuOZq2TaJp3Tjq5Ib4ioqLIG87HNjy4yEvGxLSnScqpmZBai+ns0a72/2sZInDmAYmJiyY3ukJ9E4/3kliV0pKxzB1+he0iSggeu2bDN/3ORfv+oxtO5vwTum5PFY6gGya0C2+mN6x+XSJPEDroH00LdtDzJEdBOZthfydgP6woaBwiE9zhtQsyN3g9LG14DlnfniCk0BSermvmRAe59NjYWrHEocxhqDAAFKigzhn8E9g6E/gaCGs/oDUZa9zx+Z3uTPoHYoDwgg+XASHf1hulyawXpuQG9KOY/HDCGnUiriUtqSkZ5CS2pLAwAqP2S0rhb2rYcdCyF4A2Ytg/WecSDiN2v2QSFKzoEmGXQF2BrJPxBjzY6FR0H0cAd3HwcHtsPxNgg/tO3EGURzbkm2lCazNLWXdngLW7ylk7Z4CNq0opGw5wCrCg9fSLjmajsnRdEiOpkPTGDokRxOX3NnpbyvzemdbRXmwc0m5RFLueSbBEdCsB+naFFqFOwnFbmb0O0scxpjqxTWHAXedNCkYaA20bgYXdml6YnpRcSnr9xSyenc+a3YVsGZ3PjNW7mbKgu0nyjSNDTspkWQ0jSG95UCCWg12Cqg67SI7FrnJZCHNd74LL74FEYnQbiS0v8Dpp6u+d6lyKBeCwyEkwt+RnMQShzGmzoQFB9IlNZYuqbEnpqkq+wqOsnp3AWt25bNmdwGrd+Xz9YYcikudKqqgACEpJoymsWEkxzqvSTE9aZrSn+SMMHZ8/zUXtTxK4PpPYM2HsHQyBIZC+kAnibQbWX86diwthjUfOY8k3jTLmRaV7FxcEJ/mPJ44Ps0dT4fIRj6/yMAShzHGq0SEJjFhNIkJO9GtCsCxkjI25RSyZlcB6/YUsDuviF15Razcmc/M1XsoKi4rt5YA7vw2nMbRY0mJGU//hPX0K5lPxo5viN3wGXx0F8VNuhLY8UIC2l8ATbudfVdsHdgCi16GJa/Cob0QkwoDf+M8yOvAZmf+pjlQ8PrJy4VE/XARwomEkgbx6UhZiVdCtcRhjPGLkKAAOiTH0CE55kfzVJW8I8XszneSyZz5y4hJbsnuvCPsyivik7y2vJTfnIKiS2gjOxgWsJhhuxfTc8+jMOcR9gc2ZmN8f3JThhLUehCpTeJpHh9BZOgZ9pNXWgxrP3bOLjZ+4XaXf77T/tN2eOXtOcVFcHCrk0j2uwnlwGbnqrUNM6Gk6ETRgQRAvy1OJ5h16Aw7isYY45ylxEWEEBcRQofkGGRXMIMHt/tRucKjJezOO8KOg5exev9hvt6TTXz2bNoc/Ioe+z4mIuc9Di0NZY22YJYmciCoMUcjmyGxqYQmNie6STpNklNpGhdOUkxY9fep1KUDW2HxK87ZReFuiG7mPAq453iITa1+2eAwaNzeGSoqK4PCPU4i2b+ZLcu+Jr2OkwZY4jDGnMWiQoNo0ySaNk2i3Sktgf4AaPERCtbM4uiq6bTIWU/rwh1EFi0muPAYFAI7nCWOajC7NIElmsi+wEYUhiRzNCKZkugUAuObE56QSnxCY5rEhpMUE0qT6DBCggIqC6d6pSWw7hNY9F/Y8LlzdtFmOPT6p/NaF5cdBwRATFNnaNmPrXkppJ/+Wn/EEocxpl6S4HCiu1xIdJcLf5ioCodzIW87JQeyyd+zhcP7thKYl0164U4yjqwl+tjXBB4rg4OAezHYMQ0kjyj2azRLiKIwIJajIXGUhcWTXxLM57sWE52QRGKjpjRJbkp0fBKExTk/5Ae3weJJsGQSFOyC6KYw6DfQY7xzxdpZyBKHMabhEHGuQopsRFCzHiR0goSKZUpLnOqjvB2U5WVzJGcbR/L2UVyQQ8ShXKIO7yfo2F7CitcRmZ9PEKWw9sebKiWAIwFRRJYVAHAwZRA65K/Edb2YgKBgr++qN1niMMaY8gKDnHaG2FQCOIdIILKqsqp89fl0Mju3Z8+eHeTu3U3B/t0czsuhuDAHObyfLcWRvFE8kB0bG8NGCHlnJs3jw2mZGEmLhAhS48NPXIqcFOMMtaoK8yFLHMYYU1silAZFEpHchvTkNpW2J5SUlvGTg0Vs3X+IrbmH2bb/MFtzD7Ft/xG+25TLoWOlP1qmUVTIScnkh9dwkmNDSYoJIzrMf2ctljiMMcaLggIDaJEYQYvECAa0PXmeqpJ/pMS97PgIe/KL2J13lN35R9idV0T2gSMs2nqAA4eLf7TeqNAgkmJCeXxMd7qmxvlmZ1yWOIwxxk9EhNiIYGIjgmmfHF1luaLiUjepFLHbfd2VV8Se/CJiw31/5mGJwxhjznBhwYG0TIykZWKVrS0+dWa3wBhjjDnjeDVxiMhIEVkrIhtE5N5K5oeKyFR3/ncikuZOHy4ii0Rkuft6XrllMt3pG0Tk3yJnW4c0xhhzdvNa4hCRQOBJ4AIgAxgnIhkVit0EHFDVNsDjwKPu9BxglKp2Aa4DJpVb5mngFqCtO4z01j4YY4z5MW+ecfQGNqjqJlU9BkwBRlcoMxp42X3/FjBURERVl6jqTnf6SiDcPTtpCsSo6jxVVeAV4FIv7oMxxpgKvJk4Ujhxwz4A2e60SsuoagmQByRWKPNTYLGqHnXLZ59incYYY7zojL6qSkQ64VRfjajFshOACQBJSUnMnj270nKFhYVVzvM3i612LLbasdhqp0HGpqpeGYC+wIxy4/cB91UoMwPo674PwmnbEHc8FVgH9C9Xvimwptz4OODZU8WSmZmpVZk1a1aV8/zNYqsdi612LLbaqc+xAQu1kt9Ub1ZVLQDaiki6iIQAY4FpFcpMw2n8Brgc+EJVVUTigI+Ae1X1m+OFVXUXkC8ifdyrqa4F3vfiPhhjjKng+H/33lm5yIXAP4FA4EVV/bOIPISTxaaJSBjOFVM9gP3AWFXdJCL345yhrC+3uhGquldEegEvAeHAx8Av9RQ7ISL7gK1VzG6Ec6ZzJrLYasdiqx2LrXbqc2wtVbVxxYleTRxnAxFZqKq9/B1HZSy22rHYasdiq52GGJvdOW6MMaZGLHEYY4ypEUscMNHfAVTDYqsdi612LLbaaXCxNfg2DmOMMTVjZxzGGGNqxBKHMcaYGmkwiaO2Xbz7IK7mIjJLRFaJyEoRuaOSMoNFJE9ElrrDA76Izd32Frcb+6UisrCS+eJ2b79BRL4XkZ4+iqt9ueOxVETyReTOCmV8dtxE5EUR2SsiK8pNSxCRz0RkvfsaX8Wy17ll1ovIdZWV8UJsfxeRNe5n9q57021ly1b7+Xsptj+IyI5yn9uFVSxb7d+0l2KbWi6uLSKytIplvX3cKv3d8Nl3rrLbyevbgHMD4kagFRACLAMyKpT5OfCM+34sMNVHsTUFerrvo3G6WakY22DgQz8duy1Ao2rmX4hzI6YAfYDv/PT57sa5Wckvxw0YCPQEVpSb9jec3g8A7gUerWS5BGCT+xrvvo/3QWwjgCD3/aOVxebJ5++l2P4A/MqDz7zav2lvxFZh/j+AB/x03Cr93fDVd66hnHHUuot3bwemqrtUdbH7vgBYzdnV4+9o4BV1zAPixOn+3peGAhtVtareAbxOVb/E6f2gvPLfqZep/BEA5wOfqep+VT0AfEYdP2OmsthU9VN1eqQGmIfTN5zPVXHcPOHJ37TXYnN/G64EXq/LbXqqmt8Nn3znGkriqKsu3r3KrR7rAXxXyey+IrJMRD4Wp9dgX1HgU3GexDihkvmeHFtvG0vVf8D+Om4ASer0rwbOGVFSJWXOhON3I85ZY2VO9fl7y+1uNdqLVVS3+Pu4DQD2qOr6Kub77LhV+N3wyXeuoSSOM56IRAFvA3eqan6F2YtxqmG6AU8A7/kwtHNVtSfOkxx/ISIDfbjtUxKnA81LgDcrme3P43YSdeoIzrhr30Xkd0AJMLmKIv74/J8GWgPdgV04VUJnmnFUf7bhk+NW3e+GN79zDSVx7ACalxtPdadVWkZEgoBYINcXwYlIMM6HP1lV36k4X1XzVbXQfT8dCBaRRr6ITVV3uK97gXdxqgjK8+TYetMFOA/62lNxhj+Pm2vP8Wo793VvJWX8dvxE5HrgYuBq90fmRzz4/Oucqu5R1VJVLQOeq2Kb/jxuQcBlwNSqyvjiuFXxu+GT71xDSRy17uLd24G5daUvAKtV9bEqyiQfb28Rkd44n5vXk5qIRIpI9PH3OA2qKyoUmwZcK44+QF65U2VfqPI/P38dt3LKf6euo/JHAMwARohIvFslM8Kd5lUiMhL4DXCJqh6uoownn783YivfRvaTKrbpyd+0twzDeS5QdmUzfXHcqvnd8M13zlut/mfagHP1zzqcKzF+5057COcPByAMp7pjAzAfaOWjuM7FOZ38HljqDhcCtwK3umVux3n2+jKchsx+PoqtlbvNZe72jx+38rEJ8KR7XJcDvXz4mUbiJILYctP8ctxwktcuoBinzvgmnDayz3EeDzATSHDL9gKeL7fsje73bgNwg49i24BTz338O3f8isJmwPTqPn8fxDbJ/S59j/ND2LRibO74j/6mvR2bO/2l49+xcmV9fdyq+t3wyXfOuhwxxhhTIw2lqsoYY0wdscRhjDGmRixxGGOMqRFLHMYYY2rEEocxxpgascRhzBlMnB5+P/R3HMaUZ4nDGGNMjVjiMKYOiMg1IjLfff7CsyISKCKFIvK4+7yEz0WksVu2u4jMkx+ehRHvTm8jIjPdThkXi0hrd/VRIvKWOM/PmOyLXpuNqY4lDmNOk4h0BMYA/VW1O1AKXI1zZ/tCVe0EzAEedBd5BbhHVbvi3CF9fPpk4El1OmXsh3PXMjg9n96J87yFVkB/L++SMdUK8ncAxtQDQ4FMYIF7MhCO07lcGT90hPcq8I6IxAJxqjrHnf4y8Kbbt1GKqr4LoKpFAO765qvbL5I4T5xLA772+l4ZUwVLHMacPgFeVtX7Tpoo8vsK5Wrbv8/Rcu9Lsb9b42dWVWXM6fscuFxEmsCJ5z63xPn7utwtcxXwtarmAQdEZIA7fTwwR52nuGWLyKXuOkJFJMKXO2GMp+w/F2NOk6quEpH7cZ74FoDTm+ovgENAb3feXpx2EHC6u37GTQybgBvc6eOBZ0XkIXcdV/hwN4zxmPWOa4yXiEihqkb5Ow5j6ppVVRljjKkRO+MwxhhTI3bGYYwxpkYscRhjjKkRSxzGGGNqxBKHMcaYGrHEYYwxpkb+H8h79Qa2wBpPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#!g1.1\n",
    "epochs = 10\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    print(f\"Epoch {epoch+1}\\n-------------------------------\")\n",
    "    \n",
    "    train_loop(classifier, big_train_dl, loss_fn, optimizer, history_loss=train_loss)\n",
    "    test_loop(classifier, big_test_dl, loss_fn, history_loss=val_loss)\n",
    "    \n",
    "    clear_output()\n",
    "    plot_learning_process(train_loss, val_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cellId": "92tr8y80qhn03ukqzs2f3so"
   },
   "source": [
    "Эта версия используется в финальном сервисе, присланном в качестве решения.\n",
    "\n",
    "Далее мы вводим несколько функции для обработки результатов и для визуального контроля качества модели."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "cellId": "kbw34ornw4l2h32kbi3g"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "\n",
    "def top_xx(preds, xx=95):\n",
    "    tops = torch.argsort(preds, 1, descending=True)\n",
    "    total = 0\n",
    "    index = 0\n",
    "    result = []\n",
    "    while total < xx / 100:\n",
    "        next_id = tops[0, index].item()\n",
    "        if next_id == 155:\n",
    "            continue\n",
    "        total += preds[0, next_id]\n",
    "        index += 1\n",
    "        result.append({'tag': id_to_tag[next_id], 'description': id_to_description[next_id]})\n",
    "    return result\n",
    "\n",
    "def print_data(idx):\n",
    "    print('Title:')\n",
    "    print(big_test_ds.titles[idx])\n",
    "    print('\\nAbstract:')\n",
    "    print(big_test_ds.abstracts[idx])\n",
    "\n",
    "def correct_classes(idx):\n",
    "    return [id_to_tag[i] for i in big_test_ds.tags[idx]]\n",
    "\n",
    "def top_xx_guesses(model, idx, xx):\n",
    "    print(type(model(big_test_ds[idx][0].to('cuda').reshape(1, -1))))\n",
    "    preds = F.softmax(model(big_test_ds[idx][0].to('cuda').reshape(1, -1)).logits, dim=1)\n",
    "    return [tag['tag'] for tag in top_xx(preds, xx)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "cellId": "h5gkji1otgnc2l8qytswhf"
   },
   "outputs": [],
   "source": [
    "#!g1.1\n",
    "def press_x_to_win(idx, xx):\n",
    "    print_data(idx)\n",
    "    print('CORRECT:', correct_classes(idx))\n",
    "    print('GUESSED:', top_xx_guesses(classifier, idx, xx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "cellId": "jligt6ypw4g9xq4m4odur"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title:\n",
      "Light Bosons of Electromagnetic Field and Breakdown of Relativistic\n",
      "  Theory\n",
      "\n",
      "Abstract:\n",
      "  In our analysis, a quantisation scheme for local electromagnetic waves in\n",
      "vacuum is introduced by the model of nonideal Bose-gas consisting of\n",
      "Bose-light-particles (which are no photons) with spin one and a finite mass.\n",
      "This fact destroys the Relativistic Theory of Einstein as well as displays a\n",
      "wrong sound of so-called a spontaneous breakdown of symmetry because the light\n",
      "boson can be moved by speed of wave in vacuum.\n",
      "\n",
      "CORRECT: ['physics.gen-ph']\n",
      "<class 'transformers.modeling_outputs.SequenceClassifierOutput'>\n",
      "GUESSED: ['quant-ph', 'hep-th', 'hep-ph', 'cond-mat.mes-hall', 'gr-qc', 'physics.gen-ph', 'math-ph', 'math.MP', 'math.DG', 'math.AG']\n"
     ]
    }
   ],
   "source": [
    "#!g1.1\n",
    "press_x_to_win(3, 70)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Yandex DataSphere Kernel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "notebookId": "23a7be32-1075-460e-a712-a538c2e12f04",
  "notebookPath": "Transformers DevOps/models.ipynb"
 },
 "nbformat": 4,
 "nbformat_minor": 4
}