{ "cells": [ { "cell_type": "markdown", "id": "22fce411", "metadata": { "papermill": { "duration": 0.010561, "end_time": "2023-07-20T14:28:01.754512", "exception": false, "start_time": "2023-07-20T14:28:01.743951", "status": "completed" }, "tags": [] }, "source": [ "# Setup" ] }, { "cell_type": "code", "execution_count": 1, "id": "9731dbdc", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:01.776525Z", "iopub.status.busy": "2023-07-20T14:28:01.775654Z", "iopub.status.idle": "2023-07-20T14:28:01.788643Z", "shell.execute_reply": "2023-07-20T14:28:01.787784Z" }, "papermill": { "duration": 0.026316, "end_time": "2023-07-20T14:28:01.790737", "exception": false, "start_time": "2023-07-20T14:28:01.764421", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "78a4d10e", "metadata": { "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "execution": { "iopub.execute_input": "2023-07-20T14:28:01.812708Z", "iopub.status.busy": "2023-07-20T14:28:01.811212Z", "iopub.status.idle": "2023-07-20T14:28:06.771346Z", "shell.execute_reply": "2023-07-20T14:28:06.770400Z" }, "papermill": { "duration": 4.973439, "end_time": "2023-07-20T14:28:06.773932", "exception": false, "start_time": "2023-07-20T14:28:01.800493", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n", " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n" ] } ], "source": [ "import random\n", "import csv\n", "from pathlib import Path\n", "from kaggle_secrets import UserSecretsClient\n", "import copy\n", "\n", "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from tqdm.auto import tqdm\n", "from sklearn.model_selection import train_test_split\n", "import wandb\n", "import torch\n", "from torch import nn\n", "from torch.utils.data import Dataset, DataLoader\n", "\n", "dataset_path = Path('/kaggle/input/myanimelist-dataset-animes-profiles-reviews')\n", "output_path = Path('/kaggle/working')" ] }, { "cell_type": "code", "execution_count": 3, "id": "5f19c3c7", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:06.796003Z", "iopub.status.busy": "2023-07-20T14:28:06.795459Z", "iopub.status.idle": "2023-07-20T14:28:06.805410Z", "shell.execute_reply": "2023-07-20T14:28:06.804554Z" }, "papermill": { "duration": 0.023374, "end_time": "2023-07-20T14:28:06.807410", "exception": false, "start_time": "2023-07-20T14:28:06.784036", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# Ensure deterministic behavior\n", "torch.backends.cudnn.deterministic = True\n", "random.seed(hash('setting random seeds') % 2**32 - 1)\n", "np.random.seed(hash('improves reproducibility') % 2**32 - 1)\n", "torch.manual_seed(hash('by removing stochasticity') % 2**32 - 1)\n", "torch.cuda.manual_seed_all(hash('so runs are repeatable') % 2**32 - 1)" ] }, { "cell_type": "code", "execution_count": 4, "id": "01dc078c", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:06.827990Z", "iopub.status.busy": "2023-07-20T14:28:06.827480Z", "iopub.status.idle": "2023-07-20T14:28:06.898655Z", "shell.execute_reply": "2023-07-20T14:28:06.897736Z" }, "papermill": { "duration": 0.084035, "end_time": "2023-07-20T14:28:06.901165", "exception": false, "start_time": "2023-07-20T14:28:06.817130", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using cuda device\n" ] } ], "source": [ "# Get appropriate device for training\n", "device = (\n", " 'cuda'\n", " if torch.cuda.is_available()\n", " else 'mps'\n", " if torch.backends.mps.is_available()\n", " else 'cpu'\n", ")\n", "print(f'Using {device} device')" ] }, { "cell_type": "code", "execution_count": 5, "id": "379a3fdc", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:06.922221Z", "iopub.status.busy": "2023-07-20T14:28:06.921351Z", "iopub.status.idle": "2023-07-20T14:28:08.961625Z", "shell.execute_reply": "2023-07-20T14:28:08.960641Z" }, "papermill": { "duration": 2.052923, "end_time": "2023-07-20T14:28:08.963787", "exception": false, "start_time": "2023-07-20T14:28:06.910864", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Log in to Weights and Biases for model metric tracking\n", "user_secrets = UserSecretsClient()\n", "wandb_api_key = user_secrets.get_secret('WANDB_API_KEY')\n", "wandb.login(key=wandb_api_key)" ] }, { "cell_type": "code", "execution_count": 6, "id": "9453fe4f", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:08.986015Z", "iopub.status.busy": "2023-07-20T14:28:08.985233Z", "iopub.status.idle": "2023-07-20T14:28:20.697286Z", "shell.execute_reply": "2023-07-20T14:28:20.696399Z" }, "papermill": { "duration": 11.7251, "end_time": "2023-07-20T14:28:20.699412", "exception": false, "start_time": "2023-07-20T14:28:08.974312", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
profileanime_uidscore
0DesolatePsyche340968
1baekbeans3459910
2skrn288917
3edgewalker0029049
4aManOfCulture99418110
\n", "
" ], "text/plain": [ " profile anime_uid score\n", "0 DesolatePsyche 34096 8\n", "1 baekbeans 34599 10\n", "2 skrn 28891 7\n", "3 edgewalker00 2904 9\n", "4 aManOfCulture99 4181 10" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "reviews = pd.read_csv(dataset_path/'reviews.csv', usecols=['profile', 'anime_uid', 'score'])\n", "reviews.head()" ] }, { "cell_type": "code", "execution_count": 7, "id": "c6d3fcd4", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:20.721906Z", "iopub.status.busy": "2023-07-20T14:28:20.721620Z", "iopub.status.idle": "2023-07-20T14:28:20.725602Z", "shell.execute_reply": "2023-07-20T14:28:20.724617Z" }, "papermill": { "duration": 0.017475, "end_time": "2023-07-20T14:28:20.727938", "exception": false, "start_time": "2023-07-20T14:28:20.710463", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "# Use a smaller subset of dataset for quicker iteration\n", "# reviews = reviews.sample(100)" ] }, { "cell_type": "code", "execution_count": 8, "id": "caf09e3b", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:20.749397Z", "iopub.status.busy": "2023-07-20T14:28:20.749148Z", "iopub.status.idle": "2023-07-20T14:28:21.008571Z", "shell.execute_reply": "2023-07-20T14:28:21.007628Z" }, "papermill": { "duration": 0.272956, "end_time": "2023-07-20T14:28:21.011045", "exception": false, "start_time": "2023-07-20T14:28:20.738089", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
uidtitle
028891Haikyuu!! Second Season
123273Shigatsu wa Kimi no Uso
234599Made in Abyss
35114Fullmetal Alchemist: Brotherhood
431758Kizumonogatari III: Reiketsu-hen
\n", "
" ], "text/plain": [ " uid title\n", "0 28891 Haikyuu!! Second Season\n", "1 23273 Shigatsu wa Kimi no Uso\n", "2 34599 Made in Abyss\n", "3 5114 Fullmetal Alchemist: Brotherhood\n", "4 31758 Kizumonogatari III: Reiketsu-hen" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "animes = pd.read_csv(dataset_path/'animes.csv', usecols=['uid', 'title'])\n", "animes.head()" ] }, { "cell_type": "code", "execution_count": 9, "id": "f5d8f61f", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:21.033908Z", "iopub.status.busy": "2023-07-20T14:28:21.033619Z", "iopub.status.idle": "2023-07-20T14:28:21.103374Z", "shell.execute_reply": "2023-07-20T14:28:21.102455Z" }, "papermill": { "duration": 0.08356, "end_time": "2023-07-20T14:28:21.105486", "exception": false, "start_time": "2023-07-20T14:28:21.021926", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
profileanime_uidscoreuidtitle
0DesolatePsyche34096834096Gintama.
1DesolatePsyche34096834096Gintama.
2claudinou34096834096Gintama.
3claudinou34096834096Gintama.
4PeterFromRussia34096834096Gintama.
..................
317474Kuromizue975199751Strike Witches Movie
317475ryanxwonbin975189751Strike Witches Movie
317476AobaSuzukaze9751109751Strike Witches Movie
3174777jaws7975199751Strike Witches Movie
317478arsonal975189751Strike Witches Movie
\n", "

317479 rows Γ— 5 columns

\n", "
" ], "text/plain": [ " profile anime_uid score uid title\n", "0 DesolatePsyche 34096 8 34096 Gintama.\n", "1 DesolatePsyche 34096 8 34096 Gintama.\n", "2 claudinou 34096 8 34096 Gintama.\n", "3 claudinou 34096 8 34096 Gintama.\n", "4 PeterFromRussia 34096 8 34096 Gintama.\n", "... ... ... ... ... ...\n", "317474 Kuromizue 9751 9 9751 Strike Witches Movie\n", "317475 ryanxwonbin 9751 8 9751 Strike Witches Movie\n", "317476 AobaSuzukaze 9751 10 9751 Strike Witches Movie\n", "317477 7jaws7 9751 9 9751 Strike Witches Movie\n", "317478 arsonal 9751 8 9751 Strike Witches Movie\n", "\n", "[317479 rows x 5 columns]" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Add anime titles to dataframe\n", "reviews = pd.merge(reviews, animes, left_on='anime_uid', right_on='uid')\n", "reviews" ] }, { "cell_type": "markdown", "id": "646debf5", "metadata": { "papermill": { "duration": 0.010909, "end_time": "2023-07-20T14:28:21.128178", "exception": false, "start_time": "2023-07-20T14:28:21.117269", "status": "completed" }, "tags": [] }, "source": [ "# Data Exploration" ] }, { "cell_type": "code", "execution_count": 10, "id": "d00c0e69", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:21.152326Z", "iopub.status.busy": "2023-07-20T14:28:21.150647Z", "iopub.status.idle": "2023-07-20T14:28:30.381153Z", "shell.execute_reply": "2023-07-20T14:28:30.380101Z" }, "papermill": { "duration": 9.244604, "end_time": "2023-07-20T14:28:30.383592", "exception": false, "start_time": "2023-07-20T14:28:21.138988", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "cross_tabulation = pd.crosstab(reviews.profile, reviews.title, reviews.score, aggfunc=np.sum)" ] }, { "cell_type": "markdown", "id": "8dc9252e", "metadata": { "execution": { "iopub.execute_input": "2023-07-18T20:29:28.442128Z", "iopub.status.busy": "2023-07-18T20:29:28.441735Z", "iopub.status.idle": "2023-07-18T20:29:28.449166Z", "shell.execute_reply": "2023-07-18T20:29:28.447674Z", "shell.execute_reply.started": "2023-07-18T20:29:28.442097Z" }, "papermill": { "duration": 0.010565, "end_time": "2023-07-20T14:28:30.405702", "exception": false, "start_time": "2023-07-20T14:28:30.395137", "status": "completed" }, "tags": [] }, "source": [ "Here, we preview the cross-tabulated data with the users and animes with the most ratings." ] }, { "cell_type": "code", "execution_count": 11, "id": "e086966a", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:30.429528Z", "iopub.status.busy": "2023-07-20T14:28:30.429231Z", "iopub.status.idle": "2023-07-20T14:28:30.609691Z", "shell.execute_reply": "2023-07-20T14:28:30.608634Z" }, "papermill": { "duration": 0.195586, "end_time": "2023-07-20T14:28:30.613032", "exception": false, "start_time": "2023-07-20T14:28:30.417446", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titleDeath NoteSteins;GateKimi no Na wa.Fullmetal Alchemist: BrotherhoodClannad: After StoryToradora!Mahou Shoujo Madokaβ˜…MagicaMirai NikkiTengen Toppa Gurren LagannAno Hi Mita Hana no Namae wo Bokutachi wa Mada Shiranai.
profile
Stark700NaNNaNNaNNaNNaNNaNNaN36.0NaNNaN
Sidewinder51NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
ktulu007NaN28.0NaN32.0NaN20.036.0NaN24.028.0
LegendAquaNaNNaN40.0NaNNaNNaNNaNNaNNaNNaN
ggultra2764NaNNaN24.024.0NaNNaN32.016.0NaNNaN
literaturenerd24.0NaNNaN28.0NaNNaN20.016.0NaNNaN
BanjoTheBearNaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
BabyGirl0630136.032.036.040.0NaNNaNNaN28.0NaNNaN
PyraXadon28.0NaNNaNNaN40.0NaN36.0NaNNaN36.0
angelsreviewNaNNaNNaNNaNNaN24.032.0NaNNaNNaN
\n", "
" ], "text/plain": [ "title Death Note Steins;Gate Kimi no Na wa. \\\n", "profile \n", "Stark700 NaN NaN NaN \n", "Sidewinder51 NaN NaN NaN \n", "ktulu007 NaN 28.0 NaN \n", "LegendAqua NaN NaN 40.0 \n", "ggultra2764 NaN NaN 24.0 \n", "literaturenerd 24.0 NaN NaN \n", "BanjoTheBear NaN NaN NaN \n", "BabyGirl06301 36.0 32.0 36.0 \n", "PyraXadon 28.0 NaN NaN \n", "angelsreview NaN NaN NaN \n", "\n", "title Fullmetal Alchemist: Brotherhood Clannad: After Story \\\n", "profile \n", "Stark700 NaN NaN \n", "Sidewinder51 NaN NaN \n", "ktulu007 32.0 NaN \n", "LegendAqua NaN NaN \n", "ggultra2764 24.0 NaN \n", "literaturenerd 28.0 NaN \n", "BanjoTheBear NaN NaN \n", "BabyGirl06301 40.0 NaN \n", "PyraXadon NaN 40.0 \n", "angelsreview NaN NaN \n", "\n", "title Toradora! Mahou Shoujo Madokaβ˜…Magica Mirai Nikki \\\n", "profile \n", "Stark700 NaN NaN 36.0 \n", "Sidewinder51 NaN NaN NaN \n", "ktulu007 20.0 36.0 NaN \n", "LegendAqua NaN NaN NaN \n", "ggultra2764 NaN 32.0 16.0 \n", "literaturenerd NaN 20.0 16.0 \n", "BanjoTheBear NaN NaN NaN \n", "BabyGirl06301 NaN NaN 28.0 \n", "PyraXadon NaN 36.0 NaN \n", "angelsreview 24.0 32.0 NaN \n", "\n", "title Tengen Toppa Gurren Lagann \\\n", "profile \n", "Stark700 NaN \n", "Sidewinder51 NaN \n", "ktulu007 24.0 \n", "LegendAqua NaN \n", "ggultra2764 NaN \n", "literaturenerd NaN \n", "BanjoTheBear NaN \n", "BabyGirl06301 NaN \n", "PyraXadon NaN \n", "angelsreview NaN \n", "\n", "title Ano Hi Mita Hana no Namae wo Bokutachi wa Mada Shiranai. \n", "profile \n", "Stark700 NaN \n", "Sidewinder51 NaN \n", "ktulu007 28.0 \n", "LegendAqua NaN \n", "ggultra2764 NaN \n", "literaturenerd NaN \n", "BanjoTheBear NaN \n", "BabyGirl06301 NaN \n", "PyraXadon 36.0 \n", "angelsreview NaN " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "user_groups = reviews.groupby('profile').score.count()\n", "top_users = user_groups.sort_values(ascending=False)[:10].keys()\n", "\n", "anime_groups = reviews.groupby('title').score.count()\n", "top_animes = anime_groups.sort_values(ascending=False)[:10].keys()\n", "\n", "cross_tabulation.loc[top_users, top_animes]" ] }, { "cell_type": "markdown", "id": "0805df78", "metadata": { "papermill": { "duration": 0.011274, "end_time": "2023-07-20T14:28:30.636265", "exception": false, "start_time": "2023-07-20T14:28:30.624991", "status": "completed" }, "tags": [] }, "source": [ "It looks like the review scores show a right-skew." ] }, { "cell_type": "code", "execution_count": 12, "id": "2cce7e27", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:30.660153Z", "iopub.status.busy": "2023-07-20T14:28:30.659859Z", "iopub.status.idle": "2023-07-20T14:28:30.961525Z", "shell.execute_reply": "2023-07-20T14:28:30.960623Z" }, "papermill": { "duration": 0.315931, "end_time": "2023-07-20T14:28:30.963642", "exception": false, "start_time": "2023-07-20T14:28:30.647711", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "(array([ 4870., 5164., 10229., 10909., 16495., 24766., 39624., 57463.,\n", " 70498., 77461.]),\n", " array([ 0. , 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 11. ]),\n", " )" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAGdCAYAAAAbudkLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzsklEQVR4nO3df1BV94H+8YegXJGFU34ErneCCZlliATTZrGLaFrdVcEUZDvZqWlJbnW0aBaV3Ar1R7MzNZ0NxB/R7JTWqNuJaTQlf1jbbFUKabumrKKUhlaMSdqpFYxcsfV6QUsuFM/3j0zON1eszVWzVz55v2bOH/ec557zOWcyuc98OOcYY9u2LQAAAAPdFu0BAAAAfFQoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY42J9gCi6fLlyzpz5owSExMVExMT7eEAAIAPwbZt9ff3y+Px6Lbbrj1n87EuOmfOnFFmZma0hwEAAK5Dd3e37rjjjmtmPtZFJzExUdJ7FyopKSnKowEAAB9GX1+fMjMznd/xa7IjMDQ0ZD/xxBP2XXfdZY8bN87Oysqyn3zySXt4eNjJXL582f7GN75hT5gwwR43bpw9Y8YMu7OzM2w/7777rr18+XI7NTXVHj9+vD1v3jy7u7s7LHP+/Hn70UcftZOSkuykpCT70UcftQOBQFjm1KlTdmlpqT1+/Hg7NTXVXrFihR0KhT70+QSDQVuSHQwGI7kMAAAgiiL5/Y7oZuT169frueeeU319vU6cOKENGzZo48aN+ta3vuVkNmzYoM2bN6u+vl5tbW1yu92aM2eO+vv7nYzP59PevXvV0NCglpYWXbx4UaWlpRoeHnYy5eXl6ujoUGNjoxobG9XR0SGv1+tsHx4eVklJiS5duqSWlhY1NDRoz549qq6ujuSUAACAySJpUCUlJfaiRYvC1j300EP2o48+atv2e7M5brfbfvrpp53t7777rm1Zlv3cc8/Ztm3bFy5csMeOHWs3NDQ4mXfeece+7bbb7MbGRtu2bfuNN96wJdmtra1O5vDhw7Yk+80337Rt27b3799v33bbbfY777zjZL7//e/bLpfrQ8/QMKMDAMDo85HN6DzwwAP66U9/qrfffluS9Otf/1otLS363Oc+J0k6efKk/H6/ioqKnO+4XC7NmDFDhw4dkiS1t7draGgoLOPxeJSXl+dkDh8+LMuyVFBQ4GSmTp0qy7LCMnl5efJ4PE6muLhYoVBI7e3tVx1/KBRSX19f2AIAAMwV0c3Iq1evVjAY1D333KPY2FgNDw/rqaee0pe+9CVJkt/vlyRlZGSEfS8jI0OnTp1yMnFxcUpOTh6Ref/7fr9f6enpI46fnp4elrnyOMnJyYqLi3MyV6qrq9OTTz4ZySkDAIBRLKIZnZdfflm7du3SSy+9pF/96ld64YUXtGnTJr3wwgthuSvfSWPb9t98T82VmavlryfzQWvXrlUwGHSW7u7ua44JAACMbhHN6Hzta1/TmjVr9MUvflGSNHnyZJ06dUp1dXVasGCB3G63pPdmWyZMmOB8r7e315l9cbvdGhwcVCAQCJvV6e3t1bRp05zM2bNnRxz/3LlzYfs5cuRI2PZAIKChoaERMz3vc7lccrlckZwyAAAYxSKa0fnzn/884g2EsbGxunz5siQpKytLbrdbzc3NzvbBwUEdPHjQKTH5+fkaO3ZsWKanp0ednZ1OprCwUMFgUEePHnUyR44cUTAYDMt0dnaqp6fHyTQ1Ncnlcik/Pz+S0wIAAIaKaEZn3rx5euqppzRx4kTde++9ev3117V582YtWrRI0nt/SvL5fKqtrVV2drays7NVW1ur8ePHq7y8XJJkWZYWL16s6upqpaamKiUlRTU1NZo8ebJmz54tSZo0aZLmzp2riooKbdu2TZK0ZMkSlZaWKicnR5JUVFSk3Nxceb1ebdy4UefPn1dNTY0qKip4+R8AAHhPJI9z9fX12Y8//rg9ceJEe9y4cfbdd99tP/HEE2Ev6Xv/hYFut9t2uVz2Zz/7WfvYsWNh+xkYGLCXL19up6Sk2PHx8XZpaand1dUVlvnTn/5kP/LII3ZiYqKdmJhoP/LII1d9YWBJSYkdHx9vp6Sk2MuXL7fffffdD30+PF4OAMDoE8nvd4xt23a0y1a09PX1ybIsBYNBZoEAABglIvn9jugeHQAAgNGEogMAAIxF0QEAAMai6AAAAGNF9Hg5AACIjrvW7Iv2EK7LH54uierxmdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGCsiIrOXXfdpZiYmBHLsmXLJEm2bWvdunXyeDyKj4/XzJkzdfz48bB9hEIhrVixQmlpaUpISFBZWZlOnz4dlgkEAvJ6vbIsS5Zlyev16sKFC2GZrq4uzZs3TwkJCUpLS1NVVZUGBwev4xIAAABTRVR02tra1NPT4yzNzc2SpC984QuSpA0bNmjz5s2qr69XW1ub3G635syZo/7+fmcfPp9Pe/fuVUNDg1paWnTx4kWVlpZqeHjYyZSXl6ujo0ONjY1qbGxUR0eHvF6vs314eFglJSW6dOmSWlpa1NDQoD179qi6uvqGLgYAADBLjG3b9vV+2efz6cc//rF++9vfSpI8Ho98Pp9Wr14t6b3Zm4yMDK1fv15Lly5VMBjU7bffrhdffFEPP/ywJOnMmTPKzMzU/v37VVxcrBMnTig3N1etra0qKCiQJLW2tqqwsFBvvvmmcnJydODAAZWWlqq7u1sej0eS1NDQoIULF6q3t1dJSUkfavx9fX2yLEvBYPBDfwcAMPrdtWZftIfwsfGHp0tu+j4j+f2+7nt0BgcHtWvXLi1atEgxMTE6efKk/H6/ioqKnIzL5dKMGTN06NAhSVJ7e7uGhobCMh6PR3l5eU7m8OHDsizLKTmSNHXqVFmWFZbJy8tzSo4kFRcXKxQKqb29/a+OORQKqa+vL2wBAADmuu6i88Mf/lAXLlzQwoULJUl+v1+SlJGREZbLyMhwtvn9fsXFxSk5OfmamfT09BHHS09PD8tceZzk5GTFxcU5maupq6tz7vuxLEuZmZkRnDEAABhtrrvofPe739WDDz4YNqsiSTExMWGfbdsese5KV2aulr+ezJXWrl2rYDDoLN3d3dccFwAAGN2uq+icOnVKr776qr7yla8469xutySNmFHp7e11Zl/cbrcGBwcVCASumTl79uyIY547dy4sc+VxAoGAhoaGRsz0fJDL5VJSUlLYAgAAzHVdRef5559Xenq6Skr+/w1GWVlZcrvdzpNY0nv38Rw8eFDTpk2TJOXn52vs2LFhmZ6eHnV2djqZwsJCBYNBHT161MkcOXJEwWAwLNPZ2amenh4n09TUJJfLpfz8/Os5JQAAYKAxkX7h8uXLev7557VgwQKNGfP/vx4TEyOfz6fa2lplZ2crOztbtbW1Gj9+vMrLyyVJlmVp8eLFqq6uVmpqqlJSUlRTU6PJkydr9uzZkqRJkyZp7ty5qqio0LZt2yRJS5YsUWlpqXJyciRJRUVFys3Nldfr1caNG3X+/HnV1NSooqKCWRoAAOCIuOi8+uqr6urq0qJFi0ZsW7VqlQYGBlRZWalAIKCCggI1NTUpMTHRyWzZskVjxozR/PnzNTAwoFmzZmnnzp2KjY11Mrt371ZVVZXzdFZZWZnq6+ud7bGxsdq3b58qKys1ffp0xcfHq7y8XJs2bYr0dAAAgMFu6D06ox3v0QGAjyfeo/N/Z9S+RwcAAOBWR9EBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMaKuOi88847evTRR5Wamqrx48frU5/6lNrb253ttm1r3bp18ng8io+P18yZM3X8+PGwfYRCIa1YsUJpaWlKSEhQWVmZTp8+HZYJBALyer2yLEuWZcnr9erChQthma6uLs2bN08JCQlKS0tTVVWVBgcHIz0lAABgqIiKTiAQ0PTp0zV27FgdOHBAb7zxhp555hl94hOfcDIbNmzQ5s2bVV9fr7a2Nrndbs2ZM0f9/f1Oxufzae/evWpoaFBLS4suXryo0tJSDQ8PO5ny8nJ1dHSosbFRjY2N6ujokNfrdbYPDw+rpKREly5dUktLixoaGrRnzx5VV1ffwOUAAAAmibFt2/6w4TVr1uh///d/9Ytf/OKq223blsfjkc/n0+rVqyW9N3uTkZGh9evXa+nSpQoGg7r99tv14osv6uGHH5YknTlzRpmZmdq/f7+Ki4t14sQJ5ebmqrW1VQUFBZKk1tZWFRYW6s0331ROTo4OHDig0tJSdXd3y+PxSJIaGhq0cOFC9fb2Kikp6W+eT19fnyzLUjAY/FB5AIAZ7lqzL9pD+Nj4w9MlN32fkfx+RzSj88orr2jKlCn6whe+oPT0dN1///3asWOHs/3kyZPy+/0qKipy1rlcLs2YMUOHDh2SJLW3t2toaCgs4/F4lJeX52QOHz4sy7KckiNJU6dOlWVZYZm8vDyn5EhScXGxQqFQ2J/SAADAx1dERef3v/+9tm7dquzsbP3kJz/RY489pqqqKn3ve9+TJPn9fklSRkZG2PcyMjKcbX6/X3FxcUpOTr5mJj09fcTx09PTwzJXHic5OVlxcXFO5kqhUEh9fX1hCwAAMNeYSMKXL1/WlClTVFtbK0m6//77dfz4cW3dulVf/vKXnVxMTEzY92zbHrHuSldmrpa/nswH1dXV6cknn7zmOAAAgDkimtGZMGGCcnNzw9ZNmjRJXV1dkiS32y1JI2ZUent7ndkXt9utwcFBBQKBa2bOnj074vjnzp0Ly1x5nEAgoKGhoREzPe9bu3atgsGgs3R3d3+o8wYAAKNTREVn+vTpeuutt8LWvf3227rzzjslSVlZWXK73Wpubna2Dw4O6uDBg5o2bZokKT8/X2PHjg3L9PT0qLOz08kUFhYqGAzq6NGjTubIkSMKBoNhmc7OTvX09DiZpqYmuVwu5efnX3X8LpdLSUlJYQsAADBXRH+6+upXv6pp06aptrZW8+fP19GjR7V9+3Zt375d0nt/SvL5fKqtrVV2drays7NVW1ur8ePHq7y8XJJkWZYWL16s6upqpaamKiUlRTU1NZo8ebJmz54t6b1Zorlz56qiokLbtm2TJC1ZskSlpaXKycmRJBUVFSk3N1der1cbN27U+fPnVVNTo4qKCgoMAACQFGHR+fSnP629e/dq7dq1+uY3v6msrCw9++yzeuSRR5zMqlWrNDAwoMrKSgUCARUUFKipqUmJiYlOZsuWLRozZozmz5+vgYEBzZo1Szt37lRsbKyT2b17t6qqqpyns8rKylRfX+9sj42N1b59+1RZWanp06crPj5e5eXl2rRp03VfDAAAYJaI3qNjGt6jAwAfT7xH5//OqHqPDgAAwGhC0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgRPV4OAMCVeIIJtzJmdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGCuiorNu3TrFxMSELW6329lu27bWrVsnj8ej+Ph4zZw5U8ePHw/bRygU0ooVK5SWlqaEhASVlZXp9OnTYZlAICCv1yvLsmRZlrxery5cuBCW6erq0rx585SQkKC0tDRVVVVpcHAwwtMHAAAmi3hG595771VPT4+zHDt2zNm2YcMGbd68WfX19Wpra5Pb7dacOXPU39/vZHw+n/bu3auGhga1tLTo4sWLKi0t1fDwsJMpLy9XR0eHGhsb1djYqI6ODnm9Xmf78PCwSkpKdOnSJbW0tKihoUF79uxRdXX19V4HAABgoDERf2HMmLBZnPfZtq1nn31WTzzxhB566CFJ0gsvvKCMjAy99NJLWrp0qYLBoL773e/qxRdf1OzZsyVJu3btUmZmpl599VUVFxfrxIkTamxsVGtrqwoKCiRJO3bsUGFhod566y3l5OSoqalJb7zxhrq7u+XxeCRJzzzzjBYuXKinnnpKSUlJ131BAACAOSKe0fntb38rj8ejrKwsffGLX9Tvf/97SdLJkyfl9/tVVFTkZF0ul2bMmKFDhw5Jktrb2zU0NBSW8Xg8ysvLczKHDx+WZVlOyZGkqVOnyrKssExeXp5TciSpuLhYoVBI7e3tf3XsoVBIfX19YQsAADBXREWnoKBA3/ve9/STn/xEO3bskN/v17Rp0/SnP/1Jfr9fkpSRkRH2nYyMDGeb3+9XXFyckpOTr5lJT08fcez09PSwzJXHSU5OVlxcnJO5mrq6Oue+H8uylJmZGcnpAwCAUSaiovPggw/qX//1XzV58mTNnj1b+/btk/Ten6jeFxMTE/Yd27ZHrLvSlZmr5a8nc6W1a9cqGAw6S3d39zXHBQAARrcberw8ISFBkydP1m9/+1vnvp0rZ1R6e3ud2Re3263BwUEFAoFrZs6ePTviWOfOnQvLXHmcQCCgoaGhETM9H+RyuZSUlBS2AAAAc91Q0QmFQjpx4oQmTJigrKwsud1uNTc3O9sHBwd18OBBTZs2TZKUn5+vsWPHhmV6enrU2dnpZAoLCxUMBnX06FEnc+TIEQWDwbBMZ2enenp6nExTU5NcLpfy8/Nv5JQAAIBBInrqqqamRvPmzdPEiRPV29ur//iP/1BfX58WLFigmJgY+Xw+1dbWKjs7W9nZ2aqtrdX48eNVXl4uSbIsS4sXL1Z1dbVSU1OVkpKimpoa509hkjRp0iTNnTtXFRUV2rZtmyRpyZIlKi0tVU5OjiSpqKhIubm58nq92rhxo86fP6+amhpVVFQwSwMAABwRFZ3Tp0/rS1/6kv74xz/q9ttv19SpU9Xa2qo777xTkrRq1SoNDAyosrJSgUBABQUFampqUmJiorOPLVu2aMyYMZo/f74GBgY0a9Ys7dy5U7GxsU5m9+7dqqqqcp7OKisrU319vbM9NjZW+/btU2VlpaZPn674+HiVl5dr06ZNN3QxAACAWWJs27ajPYho6evrk2VZCgaDzAQBwHW6a82+aA8Bt7A/PF1y0/cZye83/9YVAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABjrhopOXV2dYmJi5PP5nHW2bWvdunXyeDyKj4/XzJkzdfz48bDvhUIhrVixQmlpaUpISFBZWZlOnz4dlgkEAvJ6vbIsS5Zlyev16sKFC2GZrq4uzZs3TwkJCUpLS1NVVZUGBwdv5JQAAIBBrrvotLW1afv27brvvvvC1m/YsEGbN29WfX292tra5Ha7NWfOHPX39zsZn8+nvXv3qqGhQS0tLbp48aJKS0s1PDzsZMrLy9XR0aHGxkY1Njaqo6NDXq/X2T48PKySkhJdunRJLS0tamho0J49e1RdXX29pwQAAAxzXUXn4sWLeuSRR7Rjxw4lJyc7623b1rPPPqsnnnhCDz30kPLy8vTCCy/oz3/+s1566SVJUjAY1He/+10988wzmj17tu6//37t2rVLx44d06uvvipJOnHihBobG/Vf//VfKiwsVGFhoXbs2KEf//jHeuuttyRJTU1NeuONN7Rr1y7df//9mj17tp555hnt2LFDfX19N3pdAACAAa6r6CxbtkwlJSWaPXt22PqTJ0/K7/erqKjIWedyuTRjxgwdOnRIktTe3q6hoaGwjMfjUV5enpM5fPiwLMtSQUGBk5k6daosywrL5OXlyePxOJni4mKFQiG1t7dfz2kBAADDjIn0Cw0NDfrVr36ltra2Edv8fr8kKSMjI2x9RkaGTp065WTi4uLCZoLez7z/fb/fr/T09BH7T09PD8tceZzk5GTFxcU5mSuFQiGFQiHnMzM/AACYLaIZne7ubj3++OPatWuXxo0b91dzMTExYZ9t2x6x7kpXZq6Wv57MB9XV1Tk3N1uWpczMzGuOCQAAjG4Rzei0t7ert7dX+fn5zrrh4WG99tprqq+vd+6f8fv9mjBhgpPp7e11Zl/cbrcGBwcVCATCZnV6e3s1bdo0J3P27NkRxz937lzYfo4cORK2PRAIaGhoaMRMz/vWrl2rlStXOp/7+vooOwBuKXet2RftIQBGiWhGZ9asWTp27Jg6OjqcZcqUKXrkkUfU0dGhu+++W263W83Nzc53BgcHdfDgQafE5Ofna+zYsWGZnp4edXZ2OpnCwkIFg0EdPXrUyRw5ckTBYDAs09nZqZ6eHifT1NQkl8sVVsQ+yOVyKSkpKWwBAADmimhGJzExUXl5eWHrEhISlJqa6qz3+Xyqra1Vdna2srOzVVtbq/Hjx6u8vFySZFmWFi9erOrqaqWmpiolJUU1NTWaPHmyc3PzpEmTNHfuXFVUVGjbtm2SpCVLlqi0tFQ5OTmSpKKiIuXm5srr9Wrjxo06f/68ampqVFFRQYEBAACSruNm5L9l1apVGhgYUGVlpQKBgAoKCtTU1KTExEQns2XLFo0ZM0bz58/XwMCAZs2apZ07dyo2NtbJ7N69W1VVVc7TWWVlZaqvr3e2x8bGat++faqsrNT06dMVHx+v8vJybdq06WafEgAAGKVibNu2oz2IaOnr65NlWQoGg8wCAbglcI8OTPOHp0tu+j4j+f3m37oCAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEiKjpbt27Vfffdp6SkJCUlJamwsFAHDhxwttu2rXXr1snj8Sg+Pl4zZ87U8ePHw/YRCoW0YsUKpaWlKSEhQWVlZTp9+nRYJhAIyOv1yrIsWZYlr9erCxcuhGW6uro0b948JSQkKC0tTVVVVRocHIzw9AEAgMkiKjp33HGHnn76af3yl7/UL3/5S/3zP/+z/uVf/sUpMxs2bNDmzZtVX1+vtrY2ud1uzZkzR/39/c4+fD6f9u7dq4aGBrW0tOjixYsqLS3V8PCwkykvL1dHR4caGxvV2Niojo4Oeb1eZ/vw8LBKSkp06dIltbS0qKGhQXv27FF1dfWNXg8AAGCQGNu27RvZQUpKijZu3KhFixbJ4/HI5/Np9erVkt6bvcnIyND69eu1dOlSBYNB3X777XrxxRf18MMPS5LOnDmjzMxM7d+/X8XFxTpx4oRyc3PV2tqqgoICSVJra6sKCwv15ptvKicnRwcOHFBpaam6u7vl8XgkSQ0NDVq4cKF6e3uVlJT0ocbe19cny7IUDAY/9HcA4KN015p90R4CcFP94emSm77PSH6/r/seneHhYTU0NOjSpUsqLCzUyZMn5ff7VVRU5GRcLpdmzJihQ4cOSZLa29s1NDQUlvF4PMrLy3Myhw8flmVZTsmRpKlTp8qyrLBMXl6eU3Ikqbi4WKFQSO3t7X91zKFQSH19fWELAAAwV8RF59ixY/q7v/s7uVwuPfbYY9q7d69yc3Pl9/slSRkZGWH5jIwMZ5vf71dcXJySk5OvmUlPTx9x3PT09LDMlcdJTk5WXFyck7mauro6574fy7KUmZkZ4dkDAIDRJOKik5OTo46ODrW2turf/u3ftGDBAr3xxhvO9piYmLC8bdsj1l3pyszV8teTudLatWsVDAadpbu7+5rjAgAAo1vERScuLk5///d/rylTpqiurk6f/OQn9Z//+Z9yu92SNGJGpbe315l9cbvdGhwcVCAQuGbm7NmzI4577ty5sMyVxwkEAhoaGhox0/NBLpfLeWLs/QUAAJjrht+jY9u2QqGQsrKy5Ha71dzc7GwbHBzUwYMHNW3aNElSfn6+xo4dG5bp6elRZ2enkyksLFQwGNTRo0edzJEjRxQMBsMynZ2d6unpcTJNTU1yuVzKz8+/0VMCAACGGBNJ+Otf/7oefPBBZWZmqr+/Xw0NDfqf//kfNTY2KiYmRj6fT7W1tcrOzlZ2drZqa2s1fvx4lZeXS5Isy9LixYtVXV2t1NRUpaSkqKamRpMnT9bs2bMlSZMmTdLcuXNVUVGhbdu2SZKWLFmi0tJS5eTkSJKKioqUm5srr9erjRs36vz586qpqVFFRQWzNAAAwBFR0Tl79qy8Xq96enpkWZbuu+8+NTY2as6cOZKkVatWaWBgQJWVlQoEAiooKFBTU5MSExOdfWzZskVjxozR/PnzNTAwoFmzZmnnzp2KjY11Mrt371ZVVZXzdFZZWZnq6+ud7bGxsdq3b58qKys1ffp0xcfHq7y8XJs2bbqhiwEAAMxyw+/RGc14jw6AWw3v0YFpRu17dAAAAG51FB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWGOiPQAA+KjctWZftIcAIMqY0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMFVHRqaur06c//WklJiYqPT1dn//85/XWW2+FZWzb1rp16+TxeBQfH6+ZM2fq+PHjYZlQKKQVK1YoLS1NCQkJKisr0+nTp8MygUBAXq9XlmXJsix5vV5duHAhLNPV1aV58+YpISFBaWlpqqqq0uDgYCSnBAAADBZR0Tl48KCWLVum1tZWNTc36y9/+YuKiop06dIlJ7NhwwZt3rxZ9fX1amtrk9vt1pw5c9Tf3+9kfD6f9u7dq4aGBrW0tOjixYsqLS3V8PCwkykvL1dHR4caGxvV2Niojo4Oeb1eZ/vw8LBKSkp06dIltbS0qKGhQXv27FF1dfWNXA8AAGCQGNu27ev98rlz55Senq6DBw/qs5/9rGzblsfjkc/n0+rVqyW9N3uTkZGh9evXa+nSpQoGg7r99tv14osv6uGHH5YknTlzRpmZmdq/f7+Ki4t14sQJ5ebmqrW1VQUFBZKk1tZWFRYW6s0331ROTo4OHDig0tJSdXd3y+PxSJIaGhq0cOFC9fb2Kikp6W+Ov6+vT5ZlKRgMfqg8gNHlrjX7oj0E4GPvD0+X3PR9RvL7fUP36ASDQUlSSkqKJOnkyZPy+/0qKipyMi6XSzNmzNChQ4ckSe3t7RoaGgrLeDwe5eXlOZnDhw/Lsiyn5EjS1KlTZVlWWCYvL88pOZJUXFysUCik9vb2q443FAqpr68vbAEAAOa67qJj27ZWrlypBx54QHl5eZIkv98vScrIyAjLZmRkONv8fr/i4uKUnJx8zUx6evqIY6anp4dlrjxOcnKy4uLinMyV6urqnHt+LMtSZmZmpKcNAABGkesuOsuXL9dvfvMbff/73x+xLSYmJuyzbdsj1l3pyszV8teT+aC1a9cqGAw6S3d39zXHBAAARrfrKjorVqzQK6+8op///Oe64447nPVut1uSRsyo9Pb2OrMvbrdbg4ODCgQC18ycPXt2xHHPnTsXlrnyOIFAQENDQyNmet7ncrmUlJQUtgAAAHNFVHRs29by5cv1gx/8QD/72c+UlZUVtj0rK0tut1vNzc3OusHBQR08eFDTpk2TJOXn52vs2LFhmZ6eHnV2djqZwsJCBYNBHT161MkcOXJEwWAwLNPZ2amenh4n09TUJJfLpfz8/EhOCwAAGGpMJOFly5bppZde0o9+9CMlJiY6MyqWZSk+Pl4xMTHy+Xyqra1Vdna2srOzVVtbq/Hjx6u8vNzJLl68WNXV1UpNTVVKSopqamo0efJkzZ49W5I0adIkzZ07VxUVFdq2bZskacmSJSotLVVOTo4kqaioSLm5ufJ6vdq4caPOnz+vmpoaVVRUMFMDAAAkRVh0tm7dKkmaOXNm2Prnn39eCxculCStWrVKAwMDqqysVCAQUEFBgZqampSYmOjkt2zZojFjxmj+/PkaGBjQrFmztHPnTsXGxjqZ3bt3q6qqynk6q6ysTPX19c722NhY7du3T5WVlZo+fbri4+NVXl6uTZs2RXQBAACAuW7oPTqjHe/RAczGe3SA6BvV79EBAAC4lVF0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjRfRPQAD4+OItwwBGI2Z0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADAWRQcAABiLogMAAIxF0QEAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAscZE+oXXXntNGzduVHt7u3p6erR37159/vOfd7bbtq0nn3xS27dvVyAQUEFBgb797W/r3nvvdTKhUEg1NTX6/ve/r4GBAc2aNUvf+c53dMcddziZQCCgqqoqvfLKK5KksrIyfetb39InPvEJJ9PV1aVly5bpZz/7meLj41VeXq5NmzYpLi7uOi4F8H/nrjX7oj0EAPhYiHhG59KlS/rkJz+p+vr6q27fsGGDNm/erPr6erW1tcntdmvOnDnq7+93Mj6fT3v37lVDQ4NaWlp08eJFlZaWanh42MmUl5ero6NDjY2NamxsVEdHh7xer7N9eHhYJSUlunTpklpaWtTQ0KA9e/aouro60lMCAACGirFt277uL8fEhM3o2LYtj8cjn8+n1atXS3pv9iYjI0Pr16/X0qVLFQwGdfvtt+vFF1/Uww8/LEk6c+aMMjMztX//fhUXF+vEiRPKzc1Va2urCgoKJEmtra0qLCzUm2++qZycHB04cEClpaXq7u6Wx+ORJDU0NGjhwoXq7e1VUlLS3xx/X1+fLMtSMBj8UHngZmFGB8DHxR+eLrnp+4zk9zviP11dy8mTJ+X3+1VUVOSsc7lcmjFjhg4dOqSlS5eqvb1dQ0NDYRmPx6O8vDwdOnRIxcXFOnz4sCzLckqOJE2dOlWWZenQoUPKycnR4cOHlZeX55QcSSouLlYoFFJ7e7v+6Z/+acT4QqGQQqGQ87mvr+9mnj6ihNIAAPhrburNyH6/X5KUkZERtj4jI8PZ5vf7FRcXp+Tk5Gtm0tPTR+w/PT09LHPlcZKTkxUXF+dkrlRXVyfLspwlMzPzOs4SAACMFh/JU1cxMTFhn23bHrHuSldmrpa/nswHrV27VsFg0Fm6u7uvOSYAADC63dSi43a7JWnEjEpvb68z++J2uzU4OKhAIHDNzNmzZ0fs/9y5c2GZK48TCAQ0NDQ0YqbnfS6XS0lJSWELAAAw100tOllZWXK73WpubnbWDQ4O6uDBg5o2bZokKT8/X2PHjg3L9PT0qLOz08kUFhYqGAzq6NGjTubIkSMKBoNhmc7OTvX09DiZpqYmuVwu5efn38zTAgAAo1TENyNfvHhRv/vd75zPJ0+eVEdHh1JSUjRx4kT5fD7V1tYqOztb2dnZqq2t1fjx41VeXi5JsixLixcvVnV1tVJTU5WSkqKamhpNnjxZs2fPliRNmjRJc+fOVUVFhbZt2yZJWrJkiUpLS5WTkyNJKioqUm5urrxerzZu3Kjz58+rpqZGFRUVzNQAAABJ11F0fvnLX4Y90bRy5UpJ0oIFC7Rz506tWrVKAwMDqqysdF4Y2NTUpMTEROc7W7Zs0ZgxYzR//nznhYE7d+5UbGysk9m9e7eqqqqcp7PKysrC3t0TGxurffv2qbKyUtOnTw97YSAAAIB0g+/RGe14j44ZeLwcAG5d0X6PDv/WFQAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYY6I9ANxa7lqzL9pDAADgpqHofIQoDQAARBd/ugIAAMai6AAAAGNRdAAAgLEoOgAAwFgUHQAAYCyKDgAAMBZFBwAAGIuiAwAAjEXRAQAAxqLoAAAAY1F0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGGvVF5zvf+Y6ysrI0btw45efn6xe/+EW0hwQAAG4Ro7rovPzyy/L5fHriiSf0+uuv6zOf+YwefPBBdXV1RXtoAADgFjCqi87mzZu1ePFifeUrX9GkSZP07LPPKjMzU1u3bo320AAAwC1gTLQHcL0GBwfV3t6uNWvWhK0vKirSoUOHrvqdUCikUCjkfA4Gg5Kkvr6+j2SMl0N//kj2CwDAaPFR/Ma+v0/btv9mdtQWnT/+8Y8aHh5WRkZG2PqMjAz5/f6rfqeurk5PPvnkiPWZmZkfyRgBAPi4s5796Pbd398vy7KumRm1Red9MTExYZ9t2x6x7n1r167VypUrnc+XL1/W+fPnlZqa+le/c736+vqUmZmp7u5uJSUl3dR9f5xwHW8OruPNwXW8ObiON8fH+Tratq3+/n55PJ6/mR21RSctLU2xsbEjZm96e3tHzPK8z+VyyeVyha37xCc+8VENUZKUlJT0sfsP8KPAdbw5uI43B9fx5uA63hwf1+v4t2Zy3jdqb0aOi4tTfn6+mpubw9Y3Nzdr2rRpURoVAAC4lYzaGR1JWrlypbxer6ZMmaLCwkJt375dXV1deuyxx6I9NAAAcAsY1UXn4Ycf1p/+9Cd985vfVE9Pj/Ly8rR//37deeed0R6aXC6XvvGNb4z4Uxkiw3W8ObiONwfX8ebgOt4cXMcPJ8b+MM9mAQAAjEKj9h4dAACAv4WiAwAAjEXRAQAAxqLoAAAAY1F0PgLf+c53lJWVpXHjxik/P1+/+MUvoj2kUaWurk6f/vSnlZiYqPT0dH3+85/XW2+9Fe1hjXp1dXWKiYmRz+eL9lBGnXfeeUePPvqoUlNTNX78eH3qU59Se3t7tIc1qvzlL3/Rv//7vysrK0vx8fG6++679c1vflOXL1+O9tBuaa+99prmzZsnj8ejmJgY/fCHPwzbbtu21q1bJ4/Ho/j4eM2cOVPHjx+PzmBvURSdm+zll1+Wz+fTE088oddff12f+cxn9OCDD6qrqyvaQxs1Dh48qGXLlqm1tVXNzc36y1/+oqKiIl26dCnaQxu12tratH37dt13333RHsqoEwgENH36dI0dO1YHDhzQG2+8oWeeeeYjf6u6adavX6/nnntO9fX1OnHihDZs2KCNGzfqW9/6VrSHdku7dOmSPvnJT6q+vv6q2zds2KDNmzervr5ebW1tcrvdmjNnjvr7+/+PR3oLs3FT/eM//qP92GOPha2755577DVr1kRpRKNfb2+vLck+ePBgtIcyKvX399vZ2dl2c3OzPWPGDPvxxx+P9pBGldWrV9sPPPBAtIcx6pWUlNiLFi0KW/fQQw/Zjz76aJRGNPpIsvfu3et8vnz5su12u+2nn37aWffuu+/almXZzz33XBRGeGtiRucmGhwcVHt7u4qKisLWFxUV6dChQ1Ea1egXDAYlSSkpKVEeyei0bNkylZSUaPbs2dEeyqj0yiuvaMqUKfrCF76g9PR03X///dqxY0e0hzXqPPDAA/rpT3+qt99+W5L061//Wi0tLfrc5z4X5ZGNXidPnpTf7w/7zXG5XJoxYwa/OR8wqt+MfKv54x//qOHh4RH/qGhGRsaIf3wUH45t21q5cqUeeOAB5eXlRXs4o05DQ4N+9atfqa2tLdpDGbV+//vfa+vWrVq5cqW+/vWv6+jRo6qqqpLL5dKXv/zlaA9v1Fi9erWCwaDuuecexcbGanh4WE899ZS+9KUvRXtoo9b7vytX+805depUNIZ0S6LofARiYmLCPtu2PWIdPpzly5frN7/5jVpaWqI9lFGnu7tbjz/+uJqamjRu3LhoD2fUunz5sqZMmaLa2lpJ0v3336/jx49r69atFJ0IvPzyy9q1a5deeukl3Xvvvero6JDP55PH49GCBQuiPbxRjd+ca6Po3ERpaWmKjY0dMXvT29s7onHjb1uxYoVeeeUVvfbaa7rjjjuiPZxRp729Xb29vcrPz3fWDQ8P67XXXlN9fb1CoZBiY2OjOMLRYcKECcrNzQ1bN2nSJO3ZsydKIxqdvva1r2nNmjX64he/KEmaPHmyTp06pbq6OorOdXK73ZLem9mZMGGCs57fnHDco3MTxcXFKT8/X83NzWHrm5ubNW3atCiNavSxbVvLly/XD37wA/3sZz9TVlZWtIc0Ks2aNUvHjh1TR0eHs0yZMkWPPPKIOjo6KDkf0vTp00e83uDtt9++Jf7x4NHkz3/+s267LfwnJzY2lsfLb0BWVpbcbnfYb87g4KAOHjzIb84HMKNzk61cuVJer1dTpkxRYWGhtm/frq6uLj322GPRHtqosWzZMr300kv60Y9+pMTERGeGzLIsxcfHR3l0o0diYuKI+5oSEhKUmprK/U4R+OpXv6pp06aptrZW8+fP19GjR7V9+3Zt37492kMbVebNm6ennnpKEydO1L333qvXX39dmzdv1qJFi6I9tFvaxYsX9bvf/c75fPLkSXV0dCglJUUTJ06Uz+dTbW2tsrOzlZ2drdraWo0fP17l5eVRHPUtJroPfZnp29/+tn3nnXfacXFx9j/8wz/wWHSEJF11ef7556M9tFGPx8uvz3//93/beXl5tsvlsu+55x57+/bt0R7SqNPX12c//vjj9sSJE+1x48bZd999t/3EE0/YoVAo2kO7pf385z+/6v8PFyxYYNv2e4+Yf+Mb37Ddbrftcrnsz372s/axY8eiO+hbTIxt23aUOhYAAMBHint0AACAsSg6AADAWBQdAABgLIoOAAAwFkUHAAAYi6IDAACMRdEBAADGougAAABjUXQAAICxKDoAAMBYFB0AAGAsig4AADDW/wNgrJI30GKH7AAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.hist(reviews.score)" ] }, { "cell_type": "markdown", "id": "1e11dae1", "metadata": { "papermill": { "duration": 0.011356, "end_time": "2023-07-20T14:28:30.987234", "exception": false, "start_time": "2023-07-20T14:28:30.975878", "status": "completed" }, "tags": [] }, "source": [ "# Create DataLoaders" ] }, { "cell_type": "markdown", "id": "317f12b9", "metadata": { "papermill": { "duration": 0.011235, "end_time": "2023-07-20T14:28:31.010233", "exception": false, "start_time": "2023-07-20T14:28:30.998998", "status": "completed" }, "tags": [] }, "source": [ "We must create indexes for each user and anime to correspond to." ] }, { "cell_type": "code", "execution_count": 13, "id": "2cd2c452", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.035287Z", "iopub.status.busy": "2023-07-20T14:28:31.034709Z", "iopub.status.idle": "2023-07-20T14:28:31.119861Z", "shell.execute_reply": "2023-07-20T14:28:31.118932Z" }, "papermill": { "duration": 0.100153, "end_time": "2023-07-20T14:28:31.122158", "exception": false, "start_time": "2023-07-20T14:28:31.022005", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "unique_users = reviews.profile.unique()\n", "user_to_index = {}\n", "for index, user in enumerate(unique_users):\n", " user_to_index[user] = index\n", " \n", "unique_animes = reviews.title.unique()\n", "anime_to_index = {}\n", "index_to_anime = {}\n", "for index, anime in enumerate(unique_animes):\n", " anime_to_index[anime] = index\n", " index_to_anime[index] = anime" ] }, { "cell_type": "code", "execution_count": 14, "id": "baaff81e", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.147263Z", "iopub.status.busy": "2023-07-20T14:28:31.146975Z", "iopub.status.idle": "2023-07-20T14:28:31.156162Z", "shell.execute_reply": "2023-07-20T14:28:31.155345Z" }, "papermill": { "duration": 0.023843, "end_time": "2023-07-20T14:28:31.158136", "exception": false, "start_time": "2023-07-20T14:28:31.134293", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "class ReviewDataset(Dataset):\n", " '''\n", " A class for a Pytorch dataset that stores users, animes, and scores.\n", " '''\n", " \n", " def __init__(self, dataframe, user_to_index, anime_to_index):\n", " # Convert users to integers\n", " user_indexes = dataframe.profile.map(user_to_index)\n", " \n", " # Convert animes to integers\n", " anime_indexes = dataframe.title.map(anime_to_index)\n", " \n", " self.X = pd.DataFrame({'user_index': user_indexes, 'anime_index': anime_indexes})\n", " self.y = dataframe.score.astype(np.intc)\n", " \n", " def __len__(self):\n", " return len(self.X)\n", " \n", " def __getitem__(self, index):\n", " X = torch.tensor(self.X.iloc[index], dtype=torch.int32).to(device)\n", " y = torch.tensor([self.y.iloc[index]], dtype=torch.float32).to(device)\n", " return X, y" ] }, { "cell_type": "code", "execution_count": 15, "id": "a6eb2143", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.183086Z", "iopub.status.busy": "2023-07-20T14:28:31.182563Z", "iopub.status.idle": "2023-07-20T14:28:31.265246Z", "shell.execute_reply": "2023-07-20T14:28:31.264245Z" }, "papermill": { "duration": 0.097842, "end_time": "2023-07-20T14:28:31.267461", "exception": false, "start_time": "2023-07-20T14:28:31.169619", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
profileanime_uidscoreuidtitle
196752samuel_sfx14741914741Chuunibyou demo Koi ga Shitai!
210305Popaglockin1205120Fruits Basket
213579Zyzoxing31859931859Hai to Gensou no Grimgar
204607azuslu7jpg10357910357Jinrui wa Suitai Shimashita
29003JyoStar40010400Seihou Bukyou Outlaw Star
..................
119879ratchet573508185081Bakemonogatari
259178Lord_Odous10793610793Guilty Crown
131932LacePendragon13601913601Psycho-Pass
146867Tozzy32932Neon Genesis Evangelion: The End of Evangelion
121958BaronBrixius389931038993Karakai Jouzu no Takagi-san 2
\n", "

253983 rows Γ— 5 columns

\n", "
" ], "text/plain": [ " profile anime_uid score uid \\\n", "196752 samuel_sfx 14741 9 14741 \n", "210305 Popaglockin 120 5 120 \n", "213579 Zyzoxing 31859 9 31859 \n", "204607 azuslu7jpg 10357 9 10357 \n", "29003 JyoStar 400 10 400 \n", "... ... ... ... ... \n", "119879 ratchet573 5081 8 5081 \n", "259178 Lord_Odous 10793 6 10793 \n", "131932 LacePendragon 13601 9 13601 \n", "146867 Tozzy 32 9 32 \n", "121958 BaronBrixius 38993 10 38993 \n", "\n", " title \n", "196752 Chuunibyou demo Koi ga Shitai! \n", "210305 Fruits Basket \n", "213579 Hai to Gensou no Grimgar \n", "204607 Jinrui wa Suitai Shimashita \n", "29003 Seihou Bukyou Outlaw Star \n", "... ... \n", "119879 Bakemonogatari \n", "259178 Guilty Crown \n", "131932 Psycho-Pass \n", "146867 Neon Genesis Evangelion: The End of Evangelion \n", "121958 Karakai Jouzu no Takagi-san 2 \n", "\n", "[253983 rows x 5 columns]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_reviews, test_reviews = train_test_split(reviews, test_size=0.2, random_state=42)\n", "validation_reviews, test_reviews = train_test_split(test_reviews, test_size=0.5, random_state=42)\n", "train_reviews" ] }, { "cell_type": "markdown", "id": "281f132a", "metadata": { "papermill": { "duration": 0.012187, "end_time": "2023-07-20T14:28:31.292358", "exception": false, "start_time": "2023-07-20T14:28:31.280171", "status": "completed" }, "tags": [] }, "source": [ "# Create Model" ] }, { "cell_type": "code", "execution_count": 16, "id": "e94c1403", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.317800Z", "iopub.status.busy": "2023-07-20T14:28:31.317470Z", "iopub.status.idle": "2023-07-20T14:28:31.327482Z", "shell.execute_reply": "2023-07-20T14:28:31.326612Z" }, "papermill": { "duration": 0.025283, "end_time": "2023-07-20T14:28:31.329478", "exception": false, "start_time": "2023-07-20T14:28:31.304195", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "class CollaborativeFilteringNeuralNetwork(nn.Module):\n", " '''\n", " Creates a neural network with embedding layers.\n", " \n", " Arguments:\n", " num_users:\n", " Number of unique users\n", " \n", " num_items:\n", " Number of unique items\n", " \n", " num_factors:\n", " Number of latent factors for each user and item\n", " \n", " hiddens:\n", " A list of integers defining the number of units in each hidden layer.\n", " \n", " embedding_dropout:\n", " Dropout rate to apply after embeddings layer\n", " \n", " dropouts:\n", " List of dropout rates to apply after each hidden layer\n", " '''\n", " \n", " def __init__(self, num_users, num_items, num_factors, hiddens, embedding_dropout, dropouts):\n", " super().__init__()\n", " \n", " def generate_layers(num_in):\n", " '''\n", " Generator that creates layers\n", " '''\n", " \n", " for num_out, dropout in zip(hiddens, dropouts):\n", " yield nn.Linear(num_in, num_out)\n", " yield nn.ReLU()\n", " if dropout > 0.:\n", " yield nn.Dropout(dropout)\n", " num_in = num_out\n", " \n", " # Output layer\n", " yield nn.Linear(num_in, 1)\n", " yield nn.Sigmoid()\n", " \n", " self.user_embeddings = nn.Embedding(num_users, num_factors)\n", " self.item_embeddings = nn.Embedding(num_items, num_factors)\n", " self.embedding_dropout = nn.Dropout(embedding_dropout)\n", " self.linear_relu_stack = nn.Sequential(*list(generate_layers(num_factors * 2)))\n", " \n", " def forward(self, x):\n", " user_embeddings = self.user_embeddings(x[:, 0])\n", " item_embeddings = self.item_embeddings(x[:, 1])\n", " nn_input = torch.cat((user_embeddings, item_embeddings), dim=1)\n", " nn_input = self.embedding_dropout(nn_input)\n", " nn_output = self.linear_relu_stack(nn_input)\n", " return nn_output" ] }, { "cell_type": "markdown", "id": "195a2871", "metadata": { "papermill": { "duration": 0.011791, "end_time": "2023-07-20T14:28:31.353325", "exception": false, "start_time": "2023-07-20T14:28:31.341534", "status": "completed" }, "tags": [] }, "source": [ "# Train Model" ] }, { "cell_type": "code", "execution_count": 17, "id": "909e1b36", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.378627Z", "iopub.status.busy": "2023-07-20T14:28:31.378334Z", "iopub.status.idle": "2023-07-20T14:28:31.389128Z", "shell.execute_reply": "2023-07-20T14:28:31.388234Z" }, "papermill": { "duration": 0.025972, "end_time": "2023-07-20T14:28:31.391045", "exception": false, "start_time": "2023-07-20T14:28:31.365073", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def model_pipeline(init_arguments):\n", " with wandb.init(**init_arguments):\n", " config = wandb.config\n", " \n", " # Make datasets\n", " train_dataset = ReviewDataset(train_reviews, user_to_index, anime_to_index)\n", " validation_dataset = ReviewDataset(validation_reviews, user_to_index, anime_to_index)\n", " test_dataset = ReviewDataset(test_reviews, user_to_index, anime_to_index)\n", "\n", " # Make dataloaders\n", " train_dataloader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True)\n", " validation_dataloader = DataLoader(validation_dataset, batch_size=config.batch_size, shuffle=True)\n", " test_dataloader = DataLoader(test_dataset, batch_size = config.batch_size, shuffle=True)\n", " \n", " # Make model\n", " num_users = len(unique_users)\n", " num_items = len(unique_animes)\n", " model = CollaborativeFilteringNeuralNetwork(\n", " num_users, \n", " num_items, \n", " config.latent_factors, \n", " config.hidden_layers,\n", " config.embedding_dropout,\n", " config.dropouts\n", " ).to(device)\n", " \n", " loss_function = nn.MSELoss()\n", " optimizer = torch.optim.SGD(model.parameters(), lr=config.learning_rate)\n", "\n", " print('πŸ’ͺ Training! ≧◑≦\\n')\n", " train(train_dataloader, validation_dataloader, model, config.epochs, loss_function, optimizer, config.patience)\n", " \n", " print('πŸ§ͺ Testing! ≧◑≦\\n')\n", " test(test_dataloader, model, loss_function, False)\n", " \n", " print('πŸ“¦ Exporting! ≧◑≦\\n')\n", " \n", " # Export model\n", " sample_users = torch.randint(len(unique_users), (config.batch_size, 1))\n", " sample_animes = torch.randint(len(unique_animes), (config.batch_size, 1))\n", " sample_input = torch.cat((sample_users, sample_animes), 1)\n", " sample_input = sample_input.to(device)\n", " export_model(model, sample_input, 'model.onnx')\n", " \n", " # Export anime indexes\n", " export_anime_indexes('anime_indexes.csv')\n", " \n", " # Export anime embeddings\n", " anime_embeddings = model.item_embeddings.weight\n", " export_anime_embeddings(anime_embeddings, 'anime_embeddings.csv')\n", "\n", " return model, anime_embeddings" ] }, { "cell_type": "code", "execution_count": 18, "id": "8cab5f27", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.417120Z", "iopub.status.busy": "2023-07-20T14:28:31.416179Z", "iopub.status.idle": "2023-07-20T14:28:31.425747Z", "shell.execute_reply": "2023-07-20T14:28:31.424918Z" }, "papermill": { "duration": 0.024826, "end_time": "2023-07-20T14:28:31.427847", "exception": false, "start_time": "2023-07-20T14:28:31.403021", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def train(dataloader, validation_dataloader, model, epochs, loss_function, optimizer, patience):\n", " '''\n", " Training loop.\n", " '''\n", " \n", " # Tell wandb to watch what the model gets up to: gradients, weights, and more!\n", " wandb.watch(model, loss_function, log='all', log_freq=10)\n", " \n", " examples_seen = 0\n", " best_validation_loss = np.inf\n", " best_weights = None\n", " # Put model in training mode. Important for batch normalization and dropout\n", " model.train()\n", " \n", " for epoch in tqdm(range(epochs)):\n", " print(f'Epoch {epoch+1}\\n-------------------------------')\n", " \n", " for batch, (X, y) in enumerate(dataloader):\n", " loss = train_batch(X, y, model, loss_function, optimizer)\n", " examples_seen += len(X)\n", "\n", " # Report metrics every couple of batches\n", " if batch % 200 == 0:\n", " train_log(loss, examples_seen, epoch)\n", " \n", " # Validate model after each epoch\n", " validation_loss = test(validation_dataloader, model, loss_function, True)\n", " \n", " # Early stopping\n", " if validation_loss < best_validation_loss:\n", " best_validation_loss = validation_loss\n", " best_weights = copy.deepcopy(model.state_dict())\n", " no_improvements_streak = 0\n", " else:\n", " no_improvements_streak = 1\n", " \n", " if no_improvements_streak >= patience:\n", " print(f'Early stopping after {epoch} epochs')\n", " break;\n", " \n", " model.load_state_dict(best_weights)" ] }, { "cell_type": "code", "execution_count": 19, "id": "46b1853c", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.452945Z", "iopub.status.busy": "2023-07-20T14:28:31.452677Z", "iopub.status.idle": "2023-07-20T14:28:31.458316Z", "shell.execute_reply": "2023-07-20T14:28:31.457368Z" }, "papermill": { "duration": 0.020376, "end_time": "2023-07-20T14:28:31.460286", "exception": false, "start_time": "2023-07-20T14:28:31.439910", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def train_batch(X, y, model, loss_function, optimizer):\n", " '''\n", " Trains a single batch.\n", " '''\n", " \n", " # Move tensors to device\n", " X, y = X.to(device), y.to(device)\n", "\n", " # Compute prediction error\n", " predictions = model(X) * 10\n", " loss = loss_function(predictions, y)\n", "\n", " # Backpropogation\n", " loss.backward()\n", " optimizer.step()\n", " optimizer.zero_grad()\n", " \n", " return loss.item()" ] }, { "cell_type": "code", "execution_count": 20, "id": "680c21b6", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.485910Z", "iopub.status.busy": "2023-07-20T14:28:31.485110Z", "iopub.status.idle": "2023-07-20T14:28:31.490372Z", "shell.execute_reply": "2023-07-20T14:28:31.489549Z" }, "papermill": { "duration": 0.020358, "end_time": "2023-07-20T14:28:31.492581", "exception": false, "start_time": "2023-07-20T14:28:31.472223", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def train_log(loss, examples_seen, epoch):\n", " '''\n", " Print progress and save metrics to Weights and Biases\n", " '''\n", " \n", " wandb.log({'epoch': epoch, 'loss': loss}, step=examples_seen)\n", " print(f'Training loss after {examples_seen:>5d} examples: {loss:>7f}')" ] }, { "cell_type": "code", "execution_count": 21, "id": "50c4f129", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.517933Z", "iopub.status.busy": "2023-07-20T14:28:31.517193Z", "iopub.status.idle": "2023-07-20T14:28:31.524564Z", "shell.execute_reply": "2023-07-20T14:28:31.523746Z" }, "papermill": { "duration": 0.022071, "end_time": "2023-07-20T14:28:31.526511", "exception": false, "start_time": "2023-07-20T14:28:31.504440", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def test(dataloader, model, loss_function, isValidation):\n", " '''\n", " Does validation/testing.\n", " '''\n", "\n", " num_batches = len(dataloader)\n", " # Set the model to evaluation mode - important for batch normalization and dropout layers\n", " model.eval()\n", " loss = 0\n", " \n", " with torch.no_grad():\n", " for X, y in dataloader:\n", " # Move tensors to device\n", " X, y = X.to(device), y.to(device)\n", " \n", " # Compute prediction error\n", " predictions = model(X) * 10\n", " \n", " # Compute loss and accuracy\n", " loss += loss_function(predictions, y).item()\n", " \n", " \n", " loss /= num_batches\n", " \n", " if isValidation:\n", " wandb.log({'validation_loss': loss})\n", " print(f'Validation Error: \\n Validation loss: {loss:>8f} \\n')\n", " else:\n", " wandb.log({'test_loss': loss})\n", " print(f'Test Error: \\n Test loss: {loss:>8f} \\n')\n", " \n", " return loss" ] }, { "cell_type": "code", "execution_count": 22, "id": "31623e52", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.552118Z", "iopub.status.busy": "2023-07-20T14:28:31.551237Z", "iopub.status.idle": "2023-07-20T14:28:31.556694Z", "shell.execute_reply": "2023-07-20T14:28:31.555859Z" }, "papermill": { "duration": 0.020071, "end_time": "2023-07-20T14:28:31.558688", "exception": false, "start_time": "2023-07-20T14:28:31.538617", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def export_model(model, sample_input, file_name):\n", " torch.onnx.export(model, sample_input, file_name)\n", " wandb.save(str(output_path/file_name))\n", " print('Model exported!')" ] }, { "cell_type": "code", "execution_count": 23, "id": "838374ed", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.584069Z", "iopub.status.busy": "2023-07-20T14:28:31.583332Z", "iopub.status.idle": "2023-07-20T14:28:31.589761Z", "shell.execute_reply": "2023-07-20T14:28:31.588910Z" }, "papermill": { "duration": 0.021682, "end_time": "2023-07-20T14:28:31.592081", "exception": false, "start_time": "2023-07-20T14:28:31.570399", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def export_anime_indexes(file_name):\n", " with open(file_name, mode='w', newline='') as file:\n", " writer = csv.writer(file)\n", "\n", " # Write header\n", " writer.writerow(['Index', 'Anime'])\n", "\n", " # Write key-value pairs\n", " for index, anime in index_to_anime.items():\n", " writer.writerow([index, anime])\n", "\n", " wandb.save(str(output_path/file_name))\n", " print('Anime indexes CSV exported!')" ] }, { "cell_type": "code", "execution_count": 24, "id": "70adc6a3", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.619126Z", "iopub.status.busy": "2023-07-20T14:28:31.618261Z", "iopub.status.idle": "2023-07-20T14:28:31.624758Z", "shell.execute_reply": "2023-07-20T14:28:31.623783Z" }, "papermill": { "duration": 0.022205, "end_time": "2023-07-20T14:28:31.626906", "exception": false, "start_time": "2023-07-20T14:28:31.604701", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "def export_anime_embeddings(anime_embeddings, file_name):\n", " with open(file_name, mode='w', newline='') as file:\n", " writer = csv.writer(file)\n", "\n", " for row in anime_embeddings:\n", " writer.writerow(row.tolist())\n", "\n", " wandb.save(str(output_path/file_name))\n", " print('Anime embeddings CSV exported!')" ] }, { "cell_type": "code", "execution_count": 25, "id": "c91b1867", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.653487Z", "iopub.status.busy": "2023-07-20T14:28:31.652729Z", "iopub.status.idle": "2023-07-20T14:28:31.658277Z", "shell.execute_reply": "2023-07-20T14:28:31.657433Z" }, "papermill": { "duration": 0.021024, "end_time": "2023-07-20T14:28:31.660308", "exception": false, "start_time": "2023-07-20T14:28:31.639284", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "config = {\n", " 'architecture': 'Neural Collaborative Filtering',\n", " 'epochs': 100,\n", " 'batch_size': 2000,\n", " 'hidden_layers': [500, 500, 500],\n", " 'learning_rate': 1e-3,\n", " 'latent_factors': 150,\n", " 'embedding_dropout': 0.05,\n", " 'dropouts': [0.5, 0.5, 0.25],\n", " 'patience': 10\n", "}" ] }, { "cell_type": "code", "execution_count": 26, "id": "3f1ea3db", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.685953Z", "iopub.status.busy": "2023-07-20T14:28:31.685190Z", "iopub.status.idle": "2023-07-20T14:28:31.689968Z", "shell.execute_reply": "2023-07-20T14:28:31.689117Z" }, "papermill": { "duration": 0.019747, "end_time": "2023-07-20T14:28:31.692156", "exception": false, "start_time": "2023-07-20T14:28:31.672409", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "init_arguments = {\n", " 'project': 'anime-collaborative-filtering-system',\n", " 'config': config,\n", " 'name': 'serious',\n", " 'notes': \"I'm using the whole dataset and serious parameters this time!\"\n", "}" ] }, { "cell_type": "code", "execution_count": 27, "id": "9b75fef5", "metadata": { "execution": { "iopub.execute_input": "2023-07-20T14:28:31.717683Z", "iopub.status.busy": "2023-07-20T14:28:31.716930Z", "iopub.status.idle": "2023-07-20T15:52:24.225700Z", "shell.execute_reply": "2023-07-20T15:52:24.224713Z" }, "papermill": { "duration": 5032.524102, "end_time": "2023-07-20T15:52:24.228198", "exception": false, "start_time": "2023-07-20T14:28:31.704096", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33meddiezhuang\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.15.5\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/kaggle/working/wandb/run-20230720_142831-jeqqjth0\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mserious\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/eddiezhuang/anime-collaborative-filtering-system\u001b[0m\n", "\u001b[34m\u001b[1mwandb\u001b[0m: πŸš€ View run at \u001b[34m\u001b[4mhttps://wandb.ai/eddiezhuang/anime-collaborative-filtering-system/runs/jeqqjth0\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "πŸ’ͺ Training! ≧◑≦\n", "\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "b2526a62bb494e6d86266bb070ee637e", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/100 [00:00