{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "fe69586c", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:38:36.554750Z", "iopub.status.busy": "2022-11-28T07:38:36.554189Z", "iopub.status.idle": "2022-11-28T07:39:02.037804Z", "shell.execute_reply": "2022-11-28T07:39:02.037127Z", "shell.execute_reply.started": "2022-11-28T07:38:36.554696Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: fair-esm[esmfold] in /usr/local/lib/python3.9/dist-packages (2.0.0)\n", "Requirement already satisfied: omegaconf in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (2.2.3)\n", "Requirement already satisfied: ml-collections in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (0.1.1)\n", "Requirement already satisfied: pytorch-lightning in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (1.8.3.post1)\n", "Requirement already satisfied: deepspeed==0.5.9 in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (0.5.9)\n", "Requirement already satisfied: scipy in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (1.8.1)\n", "Requirement already satisfied: biopython in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (1.80)\n", "Requirement already satisfied: einops in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (0.6.0)\n", "Requirement already satisfied: dm-tree in /usr/local/lib/python3.9/dist-packages (from fair-esm[esmfold]) (0.1.7)\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (1.23.1)\n", "Requirement already satisfied: ninja in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (1.11.1)\n", "Requirement already satisfied: hjson in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (3.1.0)\n", "Requirement already satisfied: triton==1.0.0 in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (1.0.0)\n", "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (9.0.0)\n", "Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (1.12.0+cu116)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (5.9.1)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (21.3)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from deepspeed==0.5.9->fair-esm[esmfold]) (4.64.0)\n", "Requirement already satisfied: PyYAML in /usr/local/lib/python3.9/dist-packages (from ml-collections->fair-esm[esmfold]) (5.4.1)\n", "Requirement already satisfied: contextlib2 in /usr/local/lib/python3.9/dist-packages (from ml-collections->fair-esm[esmfold]) (21.6.0)\n", "Requirement already satisfied: absl-py in /usr/local/lib/python3.9/dist-packages (from ml-collections->fair-esm[esmfold]) (1.1.0)\n", "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from ml-collections->fair-esm[esmfold]) (1.14.0)\n", "Requirement already satisfied: antlr4-python3-runtime==4.9.* in /usr/local/lib/python3.9/dist-packages (from omegaconf->fair-esm[esmfold]) (4.9.3)\n", "Requirement already satisfied: tensorboardX>=2.2 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning->fair-esm[esmfold]) (2.5.1)\n", "Requirement already satisfied: fsspec[http]>2021.06.0 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning->fair-esm[esmfold]) (2022.5.0)\n", "Requirement already satisfied: torchmetrics>=0.7.0 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning->fair-esm[esmfold]) (0.10.3)\n", "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning->fair-esm[esmfold]) (4.3.0)\n", "Requirement already satisfied: lightning-utilities==0.3.* in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning->fair-esm[esmfold]) (0.3.0)\n", "Requirement already satisfied: fire in /usr/local/lib/python3.9/dist-packages (from lightning-utilities==0.3.*->pytorch-lightning->fair-esm[esmfold]) (0.4.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (3.8.1)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (2.28.1)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.9/dist-packages (from packaging->deepspeed==0.5.9->fair-esm[esmfold]) (3.0.9)\n", "Requirement already satisfied: protobuf<=3.20.1,>=3.8.0 in /usr/local/lib/python3.9/dist-packages (from tensorboardX>=2.2->pytorch-lightning->fair-esm[esmfold]) (3.19.4)\n", "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (2.1.0)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (1.7.2)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (6.0.2)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (18.2.0)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (4.0.2)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (1.3.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (1.2.0)\n", "Requirement already satisfied: termcolor in /usr/local/lib/python3.9/dist-packages (from fire->lightning-utilities==0.3.*->pytorch-lightning->fair-esm[esmfold]) (1.1.0)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (2.8)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (1.26.10)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning->fair-esm[esmfold]) (2019.11.28)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0mCollecting git+https://github.com/huggingface/transformers\n", " Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-97h8x1l5\n", " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-97h8x1l5\n", " Resolved https://github.com/huggingface/transformers to commit 61d3928bfb3029bceb5be3e68ca3d4bf8456758f\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (5.4.1)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (1.23.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (2022.7.9)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (4.64.0)\n", "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (0.12.1)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (21.3)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (0.11.0)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (3.7.1)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers==4.25.0.dev0) (2.28.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers==4.25.0.dev0) (4.3.0)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.9/dist-packages (from packaging>=20.0->transformers==4.25.0.dev0) (3.0.9)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers==4.25.0.dev0) (2019.11.28)\n", "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.25.0.dev0) (2.1.0)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers==4.25.0.dev0) (2.8)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.25.0.dev0) (1.26.10)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0mRequirement already satisfied: scikit-learn in /usr/local/lib/python3.9/dist-packages (1.1.1)\n", "Requirement already satisfied: joblib>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from scikit-learn) (1.1.0)\n", "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from scikit-learn) (3.1.0)\n", "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.9/dist-packages (from scikit-learn) (1.8.1)\n", "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.9/dist-packages (from scikit-learn) (1.23.1)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0mRequirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.3.2)\n", "Requirement already satisfied: evaluate in /usr/local/lib/python3.9/dist-packages (0.3.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.1)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.4.3)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.23.1)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (21.3)\n", "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.5.1)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.0.0)\n", "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.28.1)\n", "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.13)\n", "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.18.0)\n", "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (8.0.0)\n", "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.64.0)\n", "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.11.0)\n", "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2022.5.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (5.4.1)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.7.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.3.0)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.9/dist-packages (from packaging->datasets) (3.0.9)\n", "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2.1.0)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.10)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2019.11.28)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets) (2.8)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (18.2.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.2.0)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.7.2)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.1)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.14.0)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0mRequirement already satisfied: pynvml in /usr/local/lib/python3.9/dist-packages (11.4.1)\n", "Requirement already satisfied: numba in /usr/local/lib/python3.9/dist-packages (0.56.4)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.9/dist-packages (from numba) (63.1.0)\n", "Requirement already satisfied: numpy<1.24,>=1.18 in /usr/local/lib/python3.9/dist-packages (from numba) (1.23.1)\n", "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.9/dist-packages (from numba) (0.39.1)\n", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", "\u001b[0m" ] } ], "source": [ "# !pip3 install git+https://github.com/facebookresearch/esm.git\n", "# !pip3 install matplotlib pandas seaborn fair-esm \n", "# !curl -O https://dl.fbaipublicfiles.com/fair-esm/examples/P62593_reprs.tar.gz\n", "# !tar -xzf P62593_reprs.tar.gz\n", "# !curl -O https://dl.fbaipublicfiles.com/fair-esm/examples/P62593.fasta\n", "# !pwd\n", "# !ls\n", "\n", "!pip3 install fair-esm[esmfold]\n", "!pip3 install git+https://github.com/huggingface/transformers\n", "!pip3 install scikit-learn\n", "!pip3 install datasets evaluate\n", "!pip3 install pynvml numba" ] }, { "cell_type": "code", "execution_count": 2, "id": "84cb2ae8", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:04.432813Z", "iopub.status.busy": "2022-11-28T07:39:04.432163Z", "iopub.status.idle": "2022-11-28T07:39:07.141388Z", "shell.execute_reply": "2022-11-28T07:39:07.140791Z", "shell.execute_reply.started": "2022-11-28T07:39:04.432786Z" } }, "outputs": [], "source": [ "import random\n", "from collections import Counter\n", "from tqdm import tqdm\n", "\n", "import torch\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "from sklearn.manifold import TSNE\n", "import esm" ] }, { "cell_type": "code", "execution_count": 3, "id": "32153b10", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T09:38:14.236331Z", "iopub.status.busy": "2022-11-27T09:38:14.235062Z", "iopub.status.idle": "2022-11-27T09:38:14.239177Z", "shell.execute_reply": "2022-11-27T09:38:14.238551Z", "shell.execute_reply.started": "2022-11-27T09:38:14.236295Z" } }, "outputs": [], "source": [ "# # Load ESM-2 model\n", "# model, alphabet = esm.pretrained.esm2_t36_3B_UR50D()\n", "# batch_converter = alphabet.get_batch_converter()\n", "# model.eval() # disables dropout for deterministic results" ] }, { "cell_type": "code", "execution_count": 4, "id": "f640e760", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T09:38:15.628104Z", "iopub.status.busy": "2022-11-27T09:38:15.627002Z", "iopub.status.idle": "2022-11-27T09:38:15.631545Z", "shell.execute_reply": "2022-11-27T09:38:15.631030Z", "shell.execute_reply.started": "2022-11-27T09:38:15.628072Z" } }, "outputs": [], "source": [ "# # Prepare data (first 2 sequences from ESMStructuralSplitDataset superfamily / 4)\n", "# data = [\n", "# (\"protein1\", \"MKTVRQERLKSIVRILERSKEPVSGAQLAEELSVSRQVIVQDIAYLRSLGYNIVATPRGYVLAGG\"),\n", "# (\"protein2\", \"KALTARQQEVFDLIRDHISQTGMPPTRAEIAQRLGFRSPNAAEEHLKALARKGVIEIVSGASRGIRLLQEE\"),\n", "# (\"protein2 with mask\",\"KALTARQQEVFDLIRDISQTGMPPTRAEIAQRLGFRSPNAAEEHLKALARKGVIEIVSGASRGIRLLQEE\"),\n", "# (\"protein3\", \"K A I S Q\"),\n", "# ]\n", "# batch_labels, batch_strs, batch_tokens = batch_converter(data)\n", "\n", "# # Extract per-residue representations (on CPU)\n", "# with torch.no_grad():\n", "# results = model(batch_tokens, repr_layers=[33], return_contacts=True)\n", "# token_representations = results[\"representations\"][33]\n", "\n", "# # Generate per-sequence representations via averaging\n", "# # NOTE: token 0 is always a beginning-of-sequence token, so the first residue is token 1.\n", "# sequence_representations = []\n", "# for i, (_, seq) in enumerate(data):\n", "# sequence_representations.append(token_representations[i, 1 : len(seq) + 1].mean(0))\n", "\n", "# # Look at the unsupervised self-attention map contact predictions\n", "# import matplotlib.pyplot as plt\n", "# for (_, seq), attention_contacts in zip(data, results[\"contacts\"]):\n", "# plt.matshow(attention_contacts[: len(seq), : len(seq)])\n", "# plt.title(seq)\n", "# plt.show()" ] }, { "cell_type": "code", "execution_count": 3, "id": "e9fe8b8f-b376-4aa0-81f1-2bc4c14e6524", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:07.617311Z", "iopub.status.busy": "2022-11-28T07:39:07.616988Z", "iopub.status.idle": "2022-11-28T07:39:07.624470Z", "shell.execute_reply": "2022-11-28T07:39:07.624109Z", "shell.execute_reply.started": "2022-11-28T07:39:07.617292Z" } }, "outputs": [ { "data": { "text/plain": [ "(True, 1, 0, , 'NVIDIA RTX A4000')" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import torch\n", "\n", "torch.cuda.is_available(), torch.cuda.device_count(), torch.cuda.current_device(), torch.cuda.device(0), torch.cuda.get_device_name(0)" ] }, { "cell_type": "code", "execution_count": 4, "id": "f114a016-37ca-4e17-996f-c97aa83826fd", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:09.417829Z", "iopub.status.busy": "2022-11-28T07:39:09.417468Z", "iopub.status.idle": "2022-11-28T07:39:09.687179Z", "shell.execute_reply": "2022-11-28T07:39:09.686721Z", "shell.execute_reply.started": "2022-11-28T07:39:09.417810Z" } }, "outputs": [], "source": [ "from pynvml import *\n", "\n", "\n", "def print_gpu_utilization():\n", " nvmlInit()\n", " handle = nvmlDeviceGetHandleByIndex(0)\n", " info = nvmlDeviceGetMemoryInfo(handle)\n", " print(f\"GPU memory occupied: {info.used//1024**2} MB.\")\n", "\n", "\n", "def print_summary(result):\n", " print(f\"Time: {result.metrics['train_runtime']:.2f}\")\n", " print(f\"Samples/second: {result.metrics['train_samples_per_second']:.2f}\")\n", " print_gpu_utilization()\n", "\n", "from numba import cuda \n", "device = cuda.get_current_device()\n", "device.reset()\n" ] }, { "cell_type": "code", "execution_count": 5, "id": "b6ccef63-5673-4436-a839-b88f977b77e1", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:11.552379Z", "iopub.status.busy": "2022-11-28T07:39:11.552003Z", "iopub.status.idle": "2022-11-28T07:39:11.555008Z", "shell.execute_reply": "2022-11-28T07:39:11.554582Z", "shell.execute_reply.started": "2022-11-28T07:39:11.552359Z" } }, "outputs": [], "source": [ "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "2b1b11fa-e941-4416-b5e2-9f4a44dd8b4f", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:13.390194Z", "iopub.status.busy": "2022-11-28T07:39:13.389804Z", "iopub.status.idle": "2022-11-28T07:39:13.404642Z", "shell.execute_reply": "2022-11-28T07:39:13.404193Z", "shell.execute_reply.started": "2022-11-28T07:39:13.390174Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GPU memory occupied: 261 MB.\n" ] } ], "source": [ "print_gpu_utilization()" ] }, { "cell_type": "code", "execution_count": 7, "id": "4cd893df", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:15.134274Z", "iopub.status.busy": "2022-11-28T07:39:15.133901Z", "iopub.status.idle": "2022-11-28T07:39:15.220554Z", "shell.execute_reply": "2022-11-28T07:39:15.220082Z", "shell.execute_reply.started": "2022-11-28T07:39:15.134254Z" } }, "outputs": [], "source": [ "import csv\n", "csvfilepath = \"./train.csv\"\n", "\n", "X_train = []\n", "y_train = []\n", "\n", "with open(csvfilepath) as csvfile:\n", " csvreader = csv.reader(csvfile)\n", " for row in csvreader:\n", " #print(row)\n", " if '1' not in row and '0' not in row:\n", " X_train.append(row[0])\n", " elif '1' in row:\n", " y_train.append(1)\n", " elif '0' in row:\n", " y_train.append(0)\n", " else:\n", " pass" ] }, { "cell_type": "code", "execution_count": 8, "id": "9e0d1919", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:17.358545Z", "iopub.status.busy": "2022-11-28T07:39:17.358188Z", "iopub.status.idle": "2022-11-28T07:39:17.361885Z", "shell.execute_reply": "2022-11-28T07:39:17.361523Z", "shell.execute_reply.started": "2022-11-28T07:39:17.358527Z" } }, "outputs": [ { "data": { "text/plain": [ "('SNAMKFSEFRYERPDIAQLQASFQEALDSFRRAGSAALQHEAMKRINELRRRYSTMANLCHIRHTIDTNDEFYKKEQDFFDETEPVVKGLVNDYYRALVSSPFRAELEQVWGKQLFALAETQLKTYAPVIVEDLQKENKLASEYTKLIASAKIMFEGEERTLAQLQPFVESPDRAMRQRASEARFSFFKDYEKELDELYDELVHVRTAIARKLGFQNFVELGYARLGRTDYNADMVAGYRRQVKTHIVPLAAKLRERQRQRIQVEKLYYYDEPFMFPTGNPTPKGDADWIVQNGRQMYEELSPETGEFFRYMVEHELMDLVAKKGKAGGGYCTYIDDYKAPFIFSNFTGTSGDIDVLTHEAGHAFQVYESRHYDIPEYNWPTLEACEIHSMSMEFFTWPWMELFFGEDADKYRFAHLSDALLFLPYGVAVDEFQHAVYENPDMTPAERKSVWRNIEKAYLPTRDYADHDYLERGGFWQRQGHIYTDPFYYIDYTLAQVCAFQFWKRAQEDRASAWRDYVALCRLGGSRPFTELVKSANLQSPFADGAVASVVGHIERWLDSVDDKAL',\n", " 567)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X_train[0], len(X_train[0])" ] }, { "cell_type": "code", "execution_count": 9, "id": "bc8a5466", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:18.977334Z", "iopub.status.busy": "2022-11-28T07:39:18.976964Z", "iopub.status.idle": "2022-11-28T07:39:18.980444Z", "shell.execute_reply": "2022-11-28T07:39:18.980058Z", "shell.execute_reply.started": "2022-11-28T07:39:18.977314Z" } }, "outputs": [ { "data": { "text/plain": [ "1" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_train[0]" ] }, { "cell_type": "code", "execution_count": 10, "id": "72b3b826", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:20.833262Z", "iopub.status.busy": "2022-11-28T07:39:20.832901Z", "iopub.status.idle": "2022-11-28T07:39:20.836547Z", "shell.execute_reply": "2022-11-28T07:39:20.836159Z", "shell.execute_reply.started": "2022-11-28T07:39:20.833242Z" } }, "outputs": [ { "data": { "text/plain": [ "(26821, 26821)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(X_train), len(y_train)" ] }, { "cell_type": "code", "execution_count": 11, "id": "395750c9", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:22.175551Z", "iopub.status.busy": "2022-11-28T07:39:22.175193Z", "iopub.status.idle": "2022-11-28T07:39:23.886101Z", "shell.execute_reply": "2022-11-28T07:39:23.885613Z", "shell.execute_reply.started": "2022-11-28T07:39:22.175533Z" } }, "outputs": [], "source": [ "import torch \n", "import torch.nn as nn\n", "from transformers import EsmTokenizer, EsmForSequenceClassification" ] }, { "cell_type": "code", "execution_count": 58, "id": "68a5641e-4650-4a75-8d09-556f8593f2b6", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T09:55:06.624563Z", "iopub.status.busy": "2022-11-27T09:55:06.623937Z", "iopub.status.idle": "2022-11-27T09:55:06.630599Z", "shell.execute_reply": "2022-11-27T09:55:06.629954Z", "shell.execute_reply.started": "2022-11-27T09:55:06.624536Z" } }, "outputs": [], "source": [ "# class CustomESMCrystalModel(nn.Module):\n", "# def __init__(self,num_labels): \n", "# super(CustomESMCrystalModel,self).__init__() \n", "# self.num_labels = num_labels \n", "\n", "# #Load Model with given checkpoint and extract its body\n", "# self.model = EsmForSequenceClassification.from_pretrained(\"facebook/esm2_t6_8M_UR50D\", num_labels=768, output_attentions=True, output_hidden_states=True)\n", "# self.dropout = nn.Dropout(0.1) \n", "# self.classifier = nn.Linear(768,num_labels) # load and initialize weights\n", " \n", "# def forward(self, input_ids=None, attention_mask=None,labels=None):\n", "# #Extract outputs from the body\n", "# outputs = self.model(input_ids=input_ids, attention_mask=attention_mask)\n", "\n", "# #Add custom layers\n", "# # outputs has the following shape: (batch_size, sequence_length, 768)\n", "\n", "# #sequence_output = self.dropout(outputs[0]) #outputs[0]=last hidden state\n", "\n", "# logits = self.classifier(outputs[0,0,:].view(-1,768)) # calculate losses\n", "\n", "# loss = None\n", "# if labels is not None:\n", "# loss_fct = nn.CrossEntropyLoss()\n", "# loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n", " \n", "# return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states,attentions=outputs.attentions)" ] }, { "cell_type": "code", "execution_count": 12, "id": "5d5a9706", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:25.009785Z", "iopub.status.busy": "2022-11-28T07:39:25.009230Z", "iopub.status.idle": "2022-11-28T07:39:26.940999Z", "shell.execute_reply": "2022-11-28T07:39:26.940518Z", "shell.execute_reply.started": "2022-11-28T07:39:25.009762Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of the model checkpoint at facebook/esm2_t12_35M_UR50D were not used when initializing EsmForSequenceClassification: ['lm_head.dense.weight', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n", "- This IS expected if you are initializing EsmForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing EsmForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", "Some weights of EsmForSequenceClassification were not initialized from the model checkpoint at facebook/esm2_t12_35M_UR50D and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] } ], "source": [ "tokenizer = EsmTokenizer.from_pretrained(\"facebook/esm2_t12_35M_UR50D\")\n", "model = EsmForSequenceClassification.from_pretrained(\"facebook/esm2_t12_35M_UR50D\", num_labels=2).to(device)" ] }, { "cell_type": "code", "execution_count": 13, "id": "580e9b2c", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:39:28.878595Z", "iopub.status.busy": "2022-11-28T07:39:28.878212Z", "iopub.status.idle": "2022-11-28T07:40:04.449409Z", "shell.execute_reply": "2022-11-28T07:40:04.448893Z", "shell.execute_reply.started": "2022-11-28T07:39:28.878574Z" } }, "outputs": [], "source": [ "from datasets import Dataset \n", "from torch.utils.data import DataLoader\n", "\n", "ds = Dataset.from_dict({ \n", " \"input_ids\": tokenizer(X_train, padding=True, # Pad to max_length\n", " truncation=True,\n", " return_tensors='pt').to(device)[\"input_ids\"],\n", " \"attention_mask\": tokenizer(X_train, padding=True, # Pad to max_length\n", " truncation=True,\n", " return_tensors='pt').to(device)[\"attention_mask\"],\n", " \"label\": y_train}, split=\"train\").with_format(\"torch\")\n", "#ds_split = ds.train_test_split(test_size=0.3)" ] }, { "cell_type": "code", "execution_count": 14, "id": "22e98ebd", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:40:04.451014Z", "iopub.status.busy": "2022-11-28T07:40:04.450564Z", "iopub.status.idle": "2022-11-28T07:40:04.457117Z", "shell.execute_reply": "2022-11-28T07:40:04.456573Z", "shell.execute_reply.started": "2022-11-28T07:40:04.450991Z" } }, "outputs": [ { "data": { "text/plain": [ "torch.Size([1024])" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ds[1]['attention_mask'].shape" ] }, { "cell_type": "code", "execution_count": 15, "id": "7a53ae84-3068-4627-bdb6-0e3e406b472f", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:40:04.457924Z", "iopub.status.busy": "2022-11-28T07:40:04.457699Z", "iopub.status.idle": "2022-11-28T07:40:04.461591Z", "shell.execute_reply": "2022-11-28T07:40:04.460844Z", "shell.execute_reply.started": "2022-11-28T07:40:04.457902Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GPU memory occupied: 1852 MB.\n" ] } ], "source": [ "print_gpu_utilization()\n" ] }, { "cell_type": "code", "execution_count": 22, "id": "c0aed3ce", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T06:45:14.940646Z", "iopub.status.busy": "2022-11-28T06:45:14.939833Z", "iopub.status.idle": "2022-11-28T06:45:14.944087Z", "shell.execute_reply": "2022-11-28T06:45:14.943371Z", "shell.execute_reply.started": "2022-11-28T06:45:14.940589Z" } }, "outputs": [], "source": [ "#tokenizer(ds[1][\"data\"], return_tensors=\"pt\")#[\"input_ids\"].shape" ] }, { "cell_type": "code", "execution_count": null, "id": "0e55364c", "metadata": { "execution": { "iopub.status.busy": "2022-11-28T06:45:06.461862Z", "iopub.status.idle": "2022-11-28T06:45:06.462161Z", "shell.execute_reply": "2022-11-28T06:45:06.462032Z", "shell.execute_reply.started": "2022-11-28T06:45:06.462019Z" } }, "outputs": [], "source": [ "#ds[1]['data']" ] }, { "cell_type": "code", "execution_count": 31, "id": "5c7cb931", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T09:44:14.666664Z", "iopub.status.busy": "2022-11-27T09:44:14.665797Z", "iopub.status.idle": "2022-11-27T09:44:14.669816Z", "shell.execute_reply": "2022-11-27T09:44:14.669079Z", "shell.execute_reply.started": "2022-11-27T09:44:14.666633Z" } }, "outputs": [], "source": [ "#encoded_dataset = ds.map(lambda ex: tokenizer(ex['data'], padding='max_length', # Pad to max_length\n", "# truncation=True,\n", "# return_tensors='pt'), batched=True, batch_size=8)" ] }, { "cell_type": "code", "execution_count": 16, "id": "8bddd041", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:40:04.463171Z", "iopub.status.busy": "2022-11-28T07:40:04.462936Z", "iopub.status.idle": "2022-11-28T07:40:04.479843Z", "shell.execute_reply": "2022-11-28T07:40:04.479296Z", "shell.execute_reply.started": "2022-11-28T07:40:04.463149Z" } }, "outputs": [], "source": [ "ds_split = ds.shuffle(seed=41).train_test_split(test_size=0.05)" ] }, { "cell_type": "code", "execution_count": 17, "id": "b125cc01", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:40:04.480907Z", "iopub.status.busy": "2022-11-28T07:40:04.480671Z", "iopub.status.idle": "2022-11-28T07:40:04.485491Z", "shell.execute_reply": "2022-11-28T07:40:04.484719Z", "shell.execute_reply.started": "2022-11-28T07:40:04.480883Z" } }, "outputs": [ { "data": { "text/plain": [ "DatasetDict({\n", " train: Dataset({\n", " features: ['input_ids', 'attention_mask', 'label'],\n", " num_rows: 25479\n", " })\n", " test: Dataset({\n", " features: ['input_ids', 'attention_mask', 'label'],\n", " num_rows: 1342\n", " })\n", "})" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ds_split" ] }, { "cell_type": "code", "execution_count": 18, "id": "c52db6be", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:40:14.241230Z", "iopub.status.busy": "2022-11-28T07:40:14.240946Z", "iopub.status.idle": "2022-11-28T07:40:14.336503Z", "shell.execute_reply": "2022-11-28T07:40:14.335801Z", "shell.execute_reply.started": "2022-11-28T07:40:14.241211Z" } }, "outputs": [], "source": [ "from transformers import TrainingArguments, Trainer\n", "\n", "training_args = TrainingArguments(output_dir=\"test_trainer\", evaluation_strategy=\"steps\", \n", " per_device_train_batch_size=8, per_device_eval_batch_size=8,\n", " logging_first_step=True, logging_steps = 1, num_train_epochs=7,\n", " resume_from_checkpoint=True, save_steps=2000, save_total_limit=10,\n", " gradient_accumulation_steps=4,\n", " load_best_model_at_end=True, metric_for_best_model=\"accuracy\",\n", " save_strategy=\"steps\", eval_steps=500)\n", "\n", "import numpy as np\n", "import evaluate\n", "from sklearn.metrics import accuracy_score\n", "\n", "#metric = evaluate.load(\"accuracy\")\n", "\n", "def compute_metrics(eval_pred):\n", " logits, labels = eval_pred\n", " predictions = np.argmax(logits, axis=-1)\n", " return metrics.compute(references=labels, predictions=predictions)\n", "\n", " # {\n", " # \"accuracy\": float(\n", " # accuracy_score(references=labels, predictions=predictions)\n", " # )\n", " # }\n", "\n", "\n", "#inputs = tokenizer(\"MKTVRQERLKSIVRILERSKEPVSGAQLAEELSVSRQVIVQDIAYLRSLGYNIVATPRGYVLAGG\", return_tensors=\"pt\")\n", "\n", "\n", "trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=ds_split[\"train\"],\n", " eval_dataset=ds_split[\"test\"],\n", " compute_metrics=compute_metrics,\n", ")\n", "\n", "# with torch.no_grad():\n", "# logits = model(**inputs).logits\n", "\n", "# predicted_class_id = logits.argmax().item()\n", "# model.config.id2label[predicted_class_id]\n" ] }, { "cell_type": "code", "execution_count": 19, "id": "ef1bf91f", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:40:16.772244Z", "iopub.status.busy": "2022-11-28T07:40:16.771937Z", "iopub.status.idle": "2022-11-28T07:40:17.071452Z", "shell.execute_reply": "2022-11-28T07:40:17.070859Z", "shell.execute_reply.started": "2022-11-28T07:40:16.772223Z" } }, "outputs": [ { "data": { "text/plain": [ "torch.Size([25479, 1024])" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ds_split[\"train\"]['input_ids'].size()" ] }, { "cell_type": "code", "execution_count": null, "id": "6b3e6223", "metadata": { "execution": { "iopub.execute_input": "2022-11-28T07:40:18.615107Z", "iopub.status.busy": "2022-11-28T07:40:18.614860Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n", "***** Running training *****\n", " Num examples = 25479\n", " Num Epochs = 7\n", " Instantaneous batch size per device = 8\n", " Total train batch size (w. parallel, distributed & accumulation) = 32\n", " Gradient Accumulation steps = 4\n", " Total optimization steps = 5572\n", " Number of trainable parameters = 33993602\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [ 418/5572 20:19 < 4:11:50, 0.34 it/s, Epoch 0.52/7]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining LossValidation Loss

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "trainer.train()" ] }, { "cell_type": "code", "execution_count": 28, "id": "373a1e1b-7f00-488a-8831-cde3d6b42f2d", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:02:05.754083Z", "iopub.status.busy": "2022-11-27T15:02:05.753496Z", "iopub.status.idle": "2022-11-27T15:02:05.762500Z", "shell.execute_reply": "2022-11-27T15:02:05.761084Z", "shell.execute_reply.started": "2022-11-27T15:02:05.754024Z" } }, "outputs": [ { "data": { "text/plain": [ "'|===========================================================================|\\n| PyTorch CUDA memory summary, device ID 0 |\\n|---------------------------------------------------------------------------|\\n| CUDA OOMs: 0 | cudaMalloc retries: 0 |\\n|===========================================================================|\\n| Metric | Cur Usage | Peak Usage | Tot Alloc | Tot Freed |\\n|---------------------------------------------------------------------------|\\n| Allocated memory | 775 MB | 2487 MB | 708093 GB | 708092 GB |\\n| from large pool | 512 MB | 2224 MB | 694548 GB | 694547 GB |\\n| from small pool | 262 MB | 264 MB | 13545 GB | 13545 GB |\\n|---------------------------------------------------------------------------|\\n| Active memory | 775 MB | 2487 MB | 708093 GB | 708092 GB |\\n| from large pool | 512 MB | 2224 MB | 694548 GB | 694547 GB |\\n| from small pool | 262 MB | 264 MB | 13545 GB | 13545 GB |\\n|---------------------------------------------------------------------------|\\n| GPU reserved memory | 2626 MB | 2626 MB | 2626 MB | 0 B |\\n| from large pool | 2330 MB | 2330 MB | 2330 MB | 0 B |\\n| from small pool | 296 MB | 296 MB | 296 MB | 0 B |\\n|---------------------------------------------------------------------------|\\n| Non-releasable memory | 531305 KB | 775491 KB | 440753 GB | 440752 GB |\\n| from large pool | 499440 KB | 764400 KB | 425795 GB | 425795 GB |\\n| from small pool | 31865 KB | 37127 KB | 14957 GB | 14957 GB |\\n|---------------------------------------------------------------------------|\\n| Allocations | 1234 | 1463 | 124221 K | 124220 K |\\n| from large pool | 145 | 306 | 77747 K | 77747 K |\\n| from small pool | 1089 | 1298 | 46473 K | 46472 K |\\n|---------------------------------------------------------------------------|\\n| Active allocs | 1234 | 1463 | 124221 K | 124220 K |\\n| from large pool | 145 | 306 | 77747 K | 77747 K |\\n| from small pool | 1089 | 1298 | 46473 K | 46472 K |\\n|---------------------------------------------------------------------------|\\n| GPU reserved segments | 176 | 176 | 176 | 0 |\\n| from large pool | 28 | 28 | 28 | 0 |\\n| from small pool | 148 | 148 | 148 | 0 |\\n|---------------------------------------------------------------------------|\\n| Non-releasable allocs | 167 | 170 | 76938 K | 76938 K |\\n| from large pool | 19 | 23 | 43922 K | 43922 K |\\n| from small pool | 148 | 151 | 33015 K | 33015 K |\\n|---------------------------------------------------------------------------|\\n| Oversize allocations | 0 | 0 | 0 | 0 |\\n|---------------------------------------------------------------------------|\\n| Oversize GPU segments | 0 | 0 | 0 | 0 |\\n|===========================================================================|\\n'" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.cuda.memory_summary(device=None, abbreviated=False)" ] }, { "cell_type": "code", "execution_count": 29, "id": "9e75f2f4", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:02:05.764346Z", "iopub.status.busy": "2022-11-27T15:02:05.764007Z", "iopub.status.idle": "2022-11-27T15:04:22.857548Z", "shell.execute_reply": "2022-11-27T15:04:22.856378Z", "shell.execute_reply.started": "2022-11-27T15:02:05.764316Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "***** Running Evaluation *****\n", " Num examples = 1342\n", " Batch size = 1\n" ] }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "{'eval_loss': 0.4808196425437927,\n", " 'eval_accuracy': 0.8956780923994039,\n", " 'eval_runtime': 137.0744,\n", " 'eval_samples_per_second': 9.79,\n", " 'eval_steps_per_second': 9.79,\n", " 'epoch': 3.0}" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "trainer.evaluate()" ] }, { "cell_type": "code", "execution_count": 31, "id": "f0786767", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:07:09.833934Z", "iopub.status.busy": "2022-11-27T15:07:09.833518Z", "iopub.status.idle": "2022-11-27T15:07:09.937912Z", "shell.execute_reply": "2022-11-27T15:07:09.936925Z", "shell.execute_reply.started": "2022-11-27T15:07:09.833904Z" } }, "outputs": [], "source": [ "import csv\n", "testcsvfilepath = \"./test.csv\"\n", "\n", "X_test = []\n", "y_test = []\n", "\n", "with open(testcsvfilepath) as testcsvfile:\n", " csvreader = csv.reader(testcsvfile)\n", " for row in csvreader:\n", " #print(row)\n", " if '1' not in row and '0' not in row:\n", " X_test.append(row[0])\n", " elif '1' in row:\n", " y_test.append(1)\n", " elif '0' in row:\n", " y_test.append(0)\n", " else:\n", " pass" ] }, { "cell_type": "code", "execution_count": 32, "id": "ee634d8e", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:07:12.469579Z", "iopub.status.busy": "2022-11-27T15:07:12.469241Z", "iopub.status.idle": "2022-11-27T15:07:12.477058Z", "shell.execute_reply": "2022-11-27T15:07:12.476026Z", "shell.execute_reply.started": "2022-11-27T15:07:12.469567Z" } }, "outputs": [ { "data": { "text/plain": [ "(1898, 1898)" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(X_test), len(y_test)" ] }, { "cell_type": "code", "execution_count": 33, "id": "555698f9", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:07:14.551233Z", "iopub.status.busy": "2022-11-27T15:07:14.550832Z", "iopub.status.idle": "2022-11-27T15:07:18.577711Z", "shell.execute_reply": "2022-11-27T15:07:18.576832Z", "shell.execute_reply.started": "2022-11-27T15:07:14.551199Z" } }, "outputs": [], "source": [ "test_ds = Dataset.from_dict({ \n", " \"input_ids\": tokenizer(X_test, padding=True, # Pad to max_length\n", " truncation=True,\n", " return_tensors='pt').to(device)[\"input_ids\"],\n", " \"attention_mask\": tokenizer(X_test, padding=True, # Pad to max_length\n", " truncation=True,\n", " return_tensors='pt').to(device)[\"attention_mask\"],\n", " \"label\": y_test}, split=\"test\").with_format(\"torch\")" ] }, { "cell_type": "code", "execution_count": null, "id": "0c2c9e02", "metadata": {}, "outputs": [], "source": [ "# encoded_test_dataset = test_ds.map(lambda ex: tokenizer(ex['data'], padding='max_length', # Pad to max_length\n", "# truncation=True,\n", "# return_tensors='pt'), batched=True, batch_size=64)" ] }, { "cell_type": "code", "execution_count": 34, "id": "9538d8bb", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:07:20.785144Z", "iopub.status.busy": "2022-11-27T15:07:20.784783Z", "iopub.status.idle": "2022-11-27T15:07:20.791623Z", "shell.execute_reply": "2022-11-27T15:07:20.790512Z", "shell.execute_reply.started": "2022-11-27T15:07:20.785116Z" } }, "outputs": [ { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask', 'label'],\n", " num_rows: 1898\n", "})" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_ds" ] }, { "cell_type": "code", "execution_count": 72, "id": "f7024f8b", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:37:27.861769Z", "iopub.status.busy": "2022-11-27T15:37:27.861403Z", "iopub.status.idle": "2022-11-27T15:41:03.767689Z", "shell.execute_reply": "2022-11-27T15:41:03.766377Z", "shell.execute_reply.started": "2022-11-27T15:37:27.861742Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "***** Running Prediction *****\n", " Num examples = 1898\n", " Batch size = 1\n" ] }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "(1898, 2) (1898,)\n" ] } ], "source": [ "predictions = trainer.predict(test_ds)\n", "print(predictions.predictions.shape, predictions.label_ids.shape)" ] }, { "cell_type": "code", "execution_count": 79, "id": "495d6fbe-8bad-4ea2-ae60-f63905e09407", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:49:54.107115Z", "iopub.status.busy": "2022-11-27T15:49:54.106749Z", "iopub.status.idle": "2022-11-27T15:49:54.113077Z", "shell.execute_reply": "2022-11-27T15:49:54.111850Z", "shell.execute_reply.started": "2022-11-27T15:49:54.107114Z" } }, "outputs": [], "source": [ "def softmax(x, axis=None):\n", " x = x - x.max(axis=axis, keepdims=True)\n", " y = np.exp(x)\n", " return y / y.sum(axis=axis, keepdims=True)" ] }, { "cell_type": "code", "execution_count": 82, "id": "ee103bfe-9a0b-4352-bdf6-57b4e754855c", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:50:56.470217Z", "iopub.status.busy": "2022-11-27T15:50:56.469850Z", "iopub.status.idle": "2022-11-27T15:50:56.476932Z", "shell.execute_reply": "2022-11-27T15:50:56.475914Z", "shell.execute_reply.started": "2022-11-27T15:50:56.470202Z" } }, "outputs": [ { "data": { "text/plain": [ "array([[-2.2164035, 2.563882 ],\n", " [ 2.5647802, -2.9282844],\n", " [-2.463273 , 2.861759 ],\n", " ...,\n", " [ 3.3143802, -3.8342342],\n", " [ 2.5462682, -2.9016144],\n", " [ 3.7110417, -4.2077055]], dtype=float32)" ] }, "execution_count": 82, "metadata": {}, "output_type": "execute_result" } ], "source": [ "predictions.predictions" ] }, { "cell_type": "code", "execution_count": 88, "id": "ea449d71-3c81-4f06-84fb-547eafe2828c", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:56:09.719783Z", "iopub.status.busy": "2022-11-27T15:56:09.719382Z", "iopub.status.idle": "2022-11-27T15:56:09.726919Z", "shell.execute_reply": "2022-11-27T15:56:09.725763Z", "shell.execute_reply.started": "2022-11-27T15:56:09.719734Z" } }, "outputs": [], "source": [ "#m = nn.Softmax(dim=-1)\n", "#predictions_proba = m(predictions.predictions)\n", "predictions_probs = softmax(predictions.predictions, axis=-1)\n", "predictions_proba = [i[1] for i in softmax(predictions.predictions, axis=-1)]" ] }, { "cell_type": "code", "execution_count": 38, "id": "ced85a0b", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:10:50.915144Z", "iopub.status.busy": "2022-11-27T15:10:50.914776Z", "iopub.status.idle": "2022-11-27T15:10:50.920606Z", "shell.execute_reply": "2022-11-27T15:10:50.919250Z", "shell.execute_reply.started": "2022-11-27T15:10:50.915118Z" } }, "outputs": [], "source": [ "import numpy as np\n", "\n", "preds = np.argmax(predictions.predictions, axis=-1)" ] }, { "cell_type": "code", "execution_count": 39, "id": "1ae20949", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:10:53.952553Z", "iopub.status.busy": "2022-11-27T15:10:53.952011Z", "iopub.status.idle": "2022-11-27T15:10:53.979089Z", "shell.execute_reply": "2022-11-27T15:10:53.977835Z", "shell.execute_reply.started": "2022-11-27T15:10:53.952513Z" } }, "outputs": [ { "data": { "text/plain": [ "{'accuracy': 0.8018967334035827}" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metric.compute(predictions=preds, references=predictions.label_ids)" ] }, { "cell_type": "code", "execution_count": 40, "id": "7303d6a5-2f7d-4f4e-8b45-2247cfcc5f6c", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:11:18.218982Z", "iopub.status.busy": "2022-11-27T15:11:18.218606Z", "iopub.status.idle": "2022-11-27T15:11:21.410998Z", "shell.execute_reply": "2022-11-27T15:11:21.409804Z", "shell.execute_reply.started": "2022-11-27T15:11:18.218955Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Saving model checkpoint to test_trainer\n", "Configuration saved in test_trainer/config.json\n", "Model weights saved in test_trainer/pytorch_model.bin\n" ] } ], "source": [ "trainer.save_model()" ] }, { "cell_type": "code", "execution_count": null, "id": "f1a167c7", "metadata": {}, "outputs": [], "source": [ "#output = \n", "\n", "#trainer.predict(tokenizer(ds[1][\"data\"], return_tensors=\"pt\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "3ed1d4ef", "metadata": {}, "outputs": [], "source": [ "#model = EsmForSequenceClassification.from_pretrained(\"test_trainer/checkpoint-500/\")" ] }, { "cell_type": "code", "execution_count": null, "id": "0e4419b2", "metadata": {}, "outputs": [], "source": [ "model.config" ] }, { "cell_type": "code", "execution_count": 42, "id": "8652a916", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:16:20.635706Z", "iopub.status.busy": "2022-11-27T15:16:20.635239Z", "iopub.status.idle": "2022-11-27T15:16:20.678984Z", "shell.execute_reply": "2022-11-27T15:16:20.678044Z", "shell.execute_reply.started": "2022-11-27T15:16:20.635706Z" } }, "outputs": [], "source": [ "model_inputs = tokenizer(\"SNAMKFSEFRYERPDIAQLQASFQEALDSFRRAGSAALQHEAMKRINELRRRYSTMANLCHIRHTIDTNDEFYKKEQDFFDETEPVVKGLVNDYYRALVSSPFRAELEQVWGKQLFALAETQLKTYAPVIVEDLQKENKLASEYTKLIASAKIMFEGEERTLAQLQPFVESPDRAMRQRASEARFSFFKDYEKELDELYDELVHVRTAIARKLGFQNFVELGYARLGRTDYNADMVAGYRRQVKTHIVPLAAKLRERQRQRIQVEKLYYYDEPFMFPTGNPTPKGDADWIVQNGRQMYEELSPETGEFFRYMVEHELMDLVAKKGKAGGGYCTYIDDYKAPFIFSNFTGTSGDIDVLTHEAGHAFQVYESRHYDIPEYNWPTLEACEIHSMSMEFFTWPWMELFFGEDADKYRFAHLSDALLFLPYGVAVDEFQHAVYENPDMTPAERKSVWRNIEKAYLPTRDYADHDYLERGGFWQRQGHIYTDPFYYIDYTLAQVCAFQFWKRAQEDRASAWRDYVALCRLGGSRPFTELVKSANLQSPFADGAVASVVGHIERWLDSVDDKAL\", \n", " padding=True, # Pad to max_length\n", " truncation=True,\n", " return_tensors='pt').to(device)\n", "logits = model(**model_inputs)\n", "probabilities = torch.nn.functional.softmax(logits.logits, dim=-1)\n", "predictions = torch.argmax(probabilities, axis=1)" ] }, { "cell_type": "code", "execution_count": 69, "id": "bbd9e911", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:36:14.846129Z", "iopub.status.busy": "2022-11-27T15:36:14.845795Z", "iopub.status.idle": "2022-11-27T15:36:14.867999Z", "shell.execute_reply": "2022-11-27T15:36:14.867136Z", "shell.execute_reply.started": "2022-11-27T15:36:14.846104Z" } }, "outputs": [ { "data": { "text/plain": [ "0.9985633492469788" ] }, "execution_count": 69, "metadata": {}, "output_type": "execute_result" } ], "source": [ "probabilities[0][1].item()" ] }, { "cell_type": "code", "execution_count": 48, "id": "6cb35fd8", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:20:52.540296Z", "iopub.status.busy": "2022-11-27T15:20:52.539713Z", "iopub.status.idle": "2022-11-27T15:20:52.550556Z", "shell.execute_reply": "2022-11-27T15:20:52.548820Z", "shell.execute_reply.started": "2022-11-27T15:20:52.540255Z" } }, "outputs": [ { "data": { "text/plain": [ "Dataset({\n", " features: ['input_ids', 'attention_mask', 'label'],\n", " num_rows: 1898\n", "})" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_ds" ] }, { "cell_type": "code", "execution_count": 56, "id": "00deece0", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:27:47.857891Z", "iopub.status.busy": "2022-11-27T15:27:47.857484Z", "iopub.status.idle": "2022-11-27T15:27:47.866640Z", "shell.execute_reply": "2022-11-27T15:27:47.864887Z", "shell.execute_reply.started": "2022-11-27T15:27:47.857891Z" } }, "outputs": [ { "data": { "text/plain": [ "array([1, 0, 1, ..., 0, 0, 0])" ] }, "execution_count": 56, "metadata": {}, "output_type": "execute_result" } ], "source": [ "preds" ] }, { "cell_type": "code", "execution_count": 62, "id": "c4332f0c-e4e6-4889-9202-0ec1e1c49a83", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:31:59.440940Z", "iopub.status.busy": "2022-11-27T15:31:59.440530Z", "iopub.status.idle": "2022-11-27T15:31:59.448187Z", "shell.execute_reply": "2022-11-27T15:31:59.446613Z", "shell.execute_reply.started": "2022-11-27T15:31:59.440924Z" } }, "outputs": [], "source": [ "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score\n", "target_names = ['non-crystallizable', 'crystallizable']" ] }, { "cell_type": "code", "execution_count": 58, "id": "5fbc1db0-e711-440a-bb33-d3f96350314d", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:29:34.320967Z", "iopub.status.busy": "2022-11-27T15:29:34.319797Z", "iopub.status.idle": "2022-11-27T15:29:34.333164Z", "shell.execute_reply": "2022-11-27T15:29:34.332262Z", "shell.execute_reply.started": "2022-11-27T15:29:34.320962Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " precision recall f1-score support\n", "\n", "non-crystallizable 0.73 0.98 0.84 1000\n", " crystallizable 0.96 0.60 0.74 898\n", "\n", " accuracy 0.80 1898\n", " macro avg 0.85 0.79 0.79 1898\n", " weighted avg 0.84 0.80 0.79 1898\n", "\n" ] } ], "source": [ "print(classification_report(y_test, preds, target_names=target_names))" ] }, { "cell_type": "code", "execution_count": 61, "id": "3b54acfb-1499-4b44-a612-7359c0a96feb", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:30:50.999233Z", "iopub.status.busy": "2022-11-27T15:30:50.998837Z", "iopub.status.idle": "2022-11-27T15:30:51.006218Z", "shell.execute_reply": "2022-11-27T15:30:51.005351Z", "shell.execute_reply.started": "2022-11-27T15:30:50.999205Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[[979 21]\n", " [355 543]]\n" ] } ], "source": [ "print(confusion_matrix(y_test, preds))" ] }, { "cell_type": "code", "execution_count": 87, "id": "8f71e061-316a-48d9-baa7-bc93d7984233", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:55:14.984002Z", "iopub.status.busy": "2022-11-27T15:55:14.982662Z", "iopub.status.idle": "2022-11-27T15:55:14.993448Z", "shell.execute_reply": "2022-11-27T15:55:14.992203Z", "shell.execute_reply.started": "2022-11-27T15:55:14.983957Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.9569543429844098\n" ] } ], "source": [ "print(roc_auc_score(y_test, predictions_proba))" ] }, { "cell_type": "code", "execution_count": 94, "id": "6397ee1e-eb5a-45a5-b149-8b34a602fc32", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T15:59:12.247036Z", "iopub.status.busy": "2022-11-27T15:59:12.246661Z", "iopub.status.idle": "2022-11-27T15:59:12.259350Z", "shell.execute_reply": "2022-11-27T15:59:12.258082Z", "shell.execute_reply.started": "2022-11-27T15:59:12.247010Z" } }, "outputs": [], "source": [ "from sklearn.metrics import roc_curve, auc\n", "n_classes = 2\n", "# Compute ROC curve and ROC area for each class\n", "fpr = dict()\n", "tpr = dict()\n", "roc_auc = dict()\n", "for i in range(n_classes):\n", " fpr[i], tpr[i], _ = roc_curve(y_test, predictions_probs[:, i])\n", " roc_auc[i] = auc(fpr[i], tpr[i])\n", "\n", "# Compute micro-average ROC curve and ROC area\n", "fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test, predictions_proba)\n", "roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])" ] }, { "cell_type": "code", "execution_count": 97, "id": "72e26aaf-2809-4aa9-b2a9-09447d418442", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:00:19.756047Z", "iopub.status.busy": "2022-11-27T16:00:19.755636Z", "iopub.status.idle": "2022-11-27T16:00:19.930057Z", "shell.execute_reply": "2022-11-27T16:00:19.929202Z", "shell.execute_reply.started": "2022-11-27T16:00:19.756018Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABDUklEQVR4nO3dd3gU5fbA8e9JDx1CEem9SJWIKEpTiqDYRVQUL1dFRC/lIhYsP0CvBVGRJjZUrqLgRVEEFBsogiC9SxNC7y0kpJzfHzNJlpCykN1syvk8zz6Z+s7Zye6cnXdm3ldUFWOMMSYzQYEOwBhjTN5micIYY0yWLFEYY4zJkiUKY4wxWbJEYYwxJkuWKIwxxmTJEkUBISJrRaRdoOMINBGZKCLP5PI2J4vIyNzcpr+IyN0i8t0FrltgP4MioiJSO9BxBIrYcxS+JyLbgQpAEnASmAP0V9WTgYyroBGR3sA/VfWqAMcxGYhR1WEBjuN5oLaq3pML25pMHnjPuUVEFKijqpsDHUsg2BmF/9ygqsWAZkBz4MnAhnP+RCSkMG47kGyfmzxJVe3l4xewHbjWY/wVYJbHeCtgIXAUWAm085hXBvgA2A0cAb70mHc9sMJdbyHQJP02gYuB00AZj3nNgYNAqDv+D2C9W/5coJrHsgo8AvwFbMvk/XUH1rpx/Aw0SBfHk8A6t/wPgIjzeA9DgVVAPBACPAFsAU64Zd7sLtsAiCPtrO2oO30yMNIdbgfEAIOB/cAe4H6P7UUBXwPHgSXASODXLP6vV3n833YCvT22OQ6Y5ca5GKjlsd6b7vLHgT+Bqz3mPQ9MB6a48/8JtAR+d7ezBxgLhHmscwnwPXAY2Ac8BXQBzgAJ7v5Y6S5bEnjPLWeX+x6D3Xm9gd+A14FD7rzeKfsAEHfefje21UAj4EF3O2fcbX2d/nMPBLtxpfzv/gSqZLJfM/w+AFfifG6ruONNcT5T9d3xDD8bGby3o8BWt7ze7v9iP3Cfx/KTgYnufj0B/MK534va7nA4MArY4e7/iUBkoI87fj2mBTqAgvhK94Wp7H7B3nTHK7lfyq44Z3Qd3fFy7vxZwGdAaSAUaOtOb+5+uC93v4T3udsJz2CbPwIPeMTzKjDRHb4R2IxzoA0BhgELPZZV98tSJqMPP1AXOOXGHQo87pYX5hHHGqCKW8ZvpB24vXkPK9x1I91pt+MkvyCgh7vtiu683qQ7sHNuokgEhruxdgVigdLu/KnuqwjQEOcAkmGiAKrhHEB6umVFAc08tnkI5wAfAvwXmOqx7j3u8iE4SWsvbvLESRQJwE3ue4wEWuAcPEOA6jhJfYC7fHGcg/5gIMIdv9yjrCnp4p4BvA0UBcoDfwAPeey/ROBRd1uRnJ0oOuMc4EvhJI0GHvs+dT9n8rkfgvO5r+eu2xSIymC/Zvd9eAHn8xzpltffY93sPhuJwP04n7WROAf2cTgH+k7u/7OYx/s5AbRx57+Jx2eBsxPF68BMnM93cZwfG/8J9HHHr8e0QAdQEF/uF+ak+8FT4AeglDtvKPBxuuXn4hw0KwLJuAeydMtMAEakm7aRtETi+SX9J/CjOyw4B8A27vhsoI9HGUE4B89q7rgCHbJ4b88An6dbfxdpvwK3A3095ncFtpzHe/hHNvt2BXCjO9yb7BPFaSDEY/5+nINwMM4Bup7HvEzPKHDOkmZkMm8y8G6697whi/dwBGjqDj8PzM/mPQ9I2TZOolqeyXLP45EocK6TxeOR8N31f/LYfzvSlZG6T4EOwCZ3fwVltp/Tfe5TPoMbU/5P2by3TL8P7nAoTrJajXOtT87js/GXx7zGOJ/tCh7TDnF2svdM7sVwzlZTzmYUqI3zfTrF2WeMV5DJ2XdBedk1Cv+5SVWL4xys6gNl3enVgNtF5GjKC6dKoyLOL+nDqnokg/KqAYPTrVcF5xdVel8AV4hIRZxfSMnAAo9y3vQo4zDOh7+Sx/o7s3hfFwN/p4yoarK7fGbr/+0Rozfv4axti8i9IrLCY/lGpO1LbxxS1USP8Vicg0A5nF/RntvL6n1XwanmyMzeDLYBgIj8W0TWi8gx9z2U5Oz3kP491xWRb0Rkr4gcB170WD67ODxVwznQ7vHYf2/jnFlkuG1PqvojTrXXOGC/iEwSkRJebtvbOLP6PqCqCTgH8UbAa+oemcGrz8Y+j+HTbnnppxXzGE/dF+rceHKYc79f5XDOQP/02O4cd3qBZYnCz1T1F5wP+ih30k6cX1ClPF5FVfUld14ZESmVQVE7gRfSrVdEVT/NYJtHgO9wTsfvwvmlpB7lPJSunEhVXehZRBZvaTfOlxsAERGcg8Iuj2WqeAxXddfx9j14HgiqAe8A/XGqLUrhVGuJF3Fm5wBO1UTlTOJObydQ63w3IiJX41TP3YFzplgKOEbae4Bz38cEYAPOXTYlcOr6U5bfCdTMZHPpy9mJc0ZR1mN/l1DVS7JY5+wCVceoagucqrm6OFVK2a6H9/srq+8DIlIJeA7nWtdrIhLuTs/us3EhUv//IlIMp2ppd7plDuIkmEs84i2pzo0rBZYlitzxBtBRRJriXLS8QUQ6i0iwiESISDsRqayqe3CqhsaLSGkRCRWRNm4Z7wB9ReRycRQVkW4iUjyTbX4C3Avc5g6nmAg8KSKXAIhISRG5/Tzey+dANxG5RkRCcerK43EuRqZ4REQqi0gZ4Gmcay4X8h6K4hyQDrix3o/zqzHFPqCyiISdR/wAqGoS8D/geREpIiL1cfZXZv4LXCsid4hIiIhEiUgzLzZVHCchHQBCRORZILtf5cVxLh6fdON62GPeN0BFERkgIuEiUlxELnfn7QOqi0iQ+x734PxgeE1ESohIkIjUEpG2XsSNiFzm/q9Ccapb4nDOTlO2lVnCAngXGCEiddz/dRMRicpguUy/D+6PkMk4F+P74FybGeGul91n40J0FZGr3M/TCGCRqp51xuWeQb8DvC4i5d1tVxKRzjncdp5miSIXqOoB4CPgWfeDdyPOr8QDOL+ohpD2v+iFU3e+Aac+fYBbxlLgAZyqgCM4F5B7Z7HZmUAdYK+qrvSIZQbwMjDVrdZYA1x3Hu9lI87F2bdwfl3dgHMr8BmPxT7BOUBtxal+GHkh70FV1wGv4dwBtA+nnvk3j0V+xLn7aq+IHPT2PXjoj1MNtBf4GPgUJ+llFMsOnGsPg3GqJFbgXKDNzlycqolNONVwcWRdxQXwb5wzwRM4B6WURIuqnsC54HuDG/dfQHt39jT37yERWeYO3wuEkXYX2nTcah0vlHC3f8SN/RDOjRHgHLwbutUvX2aw7micHxXf4SS993AuSJ8lm+/DYzjVZM+4Z8T3A/eLyNVefDYuxCc4Zy+HcW4oyOx5lKE4n91F7ndoHs5F+wLLHrgzPiXOw4b/VNV5gY7lfInIy8BFqnpfoGMxuUsK2QOE58vOKEyhJSL13SoREZGWONUbMwIdlzF5jT2JaQqz4jjVTRfjVF+8BnwV0IiMyYOs6skYY0yWrOrJGGNMlvJd1VPZsmW1evXqgQ7DGGPylT///POgql7Qg4H5LlFUr16dpUuXBjoMY4zJV0Tk7+yXyphVPRljjMmSJQpjjDFZskRhjDEmS5YojDHGZMkShTHGmCxZojDGGJMlvyUKEXlfRPaLyJpM5ouIjBGRzSKySkQu9VcsxhhjLpw/zygm43T4npnrcJrBroPTWfsEP8ZijDGFU0IsZzbOylERfksUqjofp133zNwIfKSORUApcbruNMYY4yNDBn9L1xs+z1EZgbxGUYmzO3CJ4ex+l1OJyIMislRElh44cCBXgjPGmIKgUYNSLNhaNUdl5IuL2ao6SVWjVTW6XLkC3Ye5McbkyLp1B5gyZVXq+L09a7Fx6NgclRnItp52cXZn9pXdacYYY85TbGwCI0fO59VXFxIcLLRqVZnatcsgIlQvczRHZQcyUcwE+ovIVOBy4JjbGbwxxpjzMHv2XzzyyLds23YUgD59WhAVdU4X5RfMb4lCRD4F2gFlRSQGp9PyUABVnQh8i9NZ/WYgFqfjdGOMMV7ates4AwbMZfr0dQA0aVKBiRO7ccUVVbJZ8/z4LVGoas9s5ivwiL+2b4wxBd0jj3zLV19tpEiRUIYPb8e//tWKkBDfX3rOd/1RGGNMYZaYmJyaDF5++VpCQ4N57bVOVK1a0m/bzBd3PRljTGF37Fgcjz76Ld26fYJTIQP16pVl2rTb/ZokwM4ojDEmT1NVpk1bx4ABc9iz5yTBwcKKFXtp3jz3nk+2RGGMMXnUli2H6d9/NnPmbAbgiisqM3Hi9TRpUiFX47BEYYwxedCoUQt55pmfiItLpFSpCF5++Vr++c9LCQqSXI/FEoUxxuRBsbEJxMUl0qtXE0aN6kT58kUDFoslCmOMyQMOHDjFxo2HuOoqp12moUNb065dddq0qRbgyOyuJ2OMCajkZOXdd5dRr95YbrnlMw4fPg1AeHhInkgSYGcUxhgTMGvW7Kdv32/47TenIe2OHWsSG5tAmTK+a37DFyxRGGNMLjt16gzDh//C6NGLSExMpkKForzxRhd69LgEkdy/WJ0dSxTGGJPLbrttGnPmbEYE+vWL5oUXrqFUqYhAh5UpSxTGGJPLhg5tzb59J5kwoRuXX1450OFkyxKFMcb4UWJiMm+9tZjt24/y5pvXAdCuXXWWLn0wIM9EXAhLFMYY4yd//LGLhx76hhUr9gLw4IMtuOSS8gD5JkmA3R5rjDE+d/RoHP36zaJVq3dZsWIv1aqV5Ouve6YmifzGziiMMcaHpk5dw4ABc9i37xQhIUEMHnwFzzzThqJFwwId2gWzRGGMMT703Xdb2LfvFK1bV2HChG40bpy7Dfj5gyUKY4zJgfj4RHbtOkHNmqUBeOWVjlx9dVXuu69ZvroOkRW7RmGMMRfoxx+30aTJRLp1+4QzZ5IAKFu2CPff3zwPJQnNcQmWKIwx5jzt23eSXr1mcM01H7Fp0yEAYmKOBziqTGydleMirOrJGGO8lJysvPPOnzzxxA8cPRpHREQIw4ZdzZAhrQkLCw50eBlLiM1xEZYojDHGSzff/BkzZ24EoHPnWowb15VatcoEOCr/s6onY4zx0i231Oeii4rx2We3MXv23fkjSWz+MsdF2BmFMcZkYubMjcTEHKdfv8sAuPfeptxySwOKFw8PcGTnIdGqnowxxud27DjGY4/N5quvNhIeHkyXLrWpWbM0IpK/kgRAUGiOi7BEYYwxroSEJMaMWcxzz/3MqVMJFC8exsiRHahWrWSgQwsoSxTGGAMsWhTDQw99w6pV+wC4/faGvP56ZypVKhHgyALPEoUxxgDPPPMTq1bto0aNUowd25WuXesEOqQ8wxKFMaZQUlVOnDhDiRLONYexY6/jo49W8vTTbShSJOf1+gWJ3R5rjCl0Nm48yLXXfswtt3yGqtPERb16ZXnhhWssSWTAziiMMYVGXFwi//nPAl566TfOnEkiKiqS7duPUqNG6UCHlqdZojDGFArff7+Ffv2+ZfPmwwD84x/NeOWVjkRFFQlwZHmfX6ueRKSLiGwUkc0i8kQG86uKyE8islxEVolIV3/GY4wpfFSVf/zjKzp1msLmzYdp2LAc8+f35r33brQk4SW/nVGISDAwDugIxABLRGSmqq7zWGwY8LmqThCRhsC3QHV/xWSMKXxEhOrVSxEZGcKzz7Zl0KAr8m4DfnmUP6ueWgKbVXUrgIhMBW4EPBOFAik3KZcEdvsxHmNMIbFixV727DnBddc5t7gOHdqaXr2a2LWIC+TPqqdKwE6P8Rh3mqfngXtEJAbnbOLRjAoSkQdFZKmILD1w4IA/YjXGFAAnTsQzaNBcWrSYxH33fcnhw6cBCA8PsSSRA4G+PbYnMFlVKwNdgY9F5JyYVHWSqkaranS5cuVyPUhjTN6mqsyYsZ6GDcfz+uuLALjrrsaEhgb6EFcw+LPqaRdQxWO8sjvNUx+gC4Cq/i4iEUBZYL8f4zLGFCB//32U/v1n8803mwCIjr6Yt9++nksvrRjgyAoOf6bbJUAdEakhImHAncDMdMvsAK4BEJEGQARgdUvGGK+oKrfe+jnffLOJEiXCGTv2OhYt6mNJwsf8dkahqoki0h+YCwQD76vqWhEZDixV1ZnAYOAdERmIc2G7t6Y8JmmMMZlITlaCggQRYdSoTkycuJTXX+9MxYrFAx1agST57bgcHR2tS5cuDXQYxpgAOHQolieemAfAO+90D3A0+cRrAoD8mz9VNfpCirArPcaYPE9V+fDDFdSvP453313ORx+tIibmeKDDyvtO7fVJMdaEhzEmT1u//gAPPzyLX375G4B27aozYUI3Kle2fiKylXjaJ8VYojDG5EmqyrPP/sTLL/9GQkIyZcsW4bXXOtGrVxNEJNDh5S8lqgPbL3h1SxTGmDxJRNi16wQJCck88MClvPTStZQpExnosAolSxTGmDxj9+4THDwYS5MmFQB45ZWO9OnTnNatqwY4ssLNLmYbYwIuKSmZsWP/oEGDcdx553TOnEkCoGzZIpYk8gA7ozDGBNSyZXt46KFvWLrUaRO0TZtqHD8eT9my1gR4XmGJwhgTEMePx/PMMz8yduwSkpOVypVLMGZMF266qb5drM5jvE4UIlJEVWP9GYwxpnBQVdq0+YCVK/cRHCwMGtSK559vR/Hi4YEOzWQg22sUInKliKwDNrjjTUVkvN8jM8YUWCLCwIGtaNmyEkuXPshrr3W2JJGHeXNG8TrQGbdBP1VdKSJt/BqVMaZAOXMmidGjfyc4WBgypDUA997blHvuaUJwsN1Tk9d5VfWkqjvT1Rkm+SccY0xBs2DB3/TtO4t16w4QHh7Mvfc2pUKFYogIwcF2LSI/8CZR7BSRKwEVkVDgX8B6/4ZljMnvDh6M5fHHv+eDD1YAUKdOGcaP70aFCsUCG5g5b94kir7AmzjdmO4CvgP6+TMoY0z+papMnryCIUO+59Ch04SFBfPkk1fxxBNXERFhN1rmR9781+qp6t2eE0SkNfCbf0IyxuR3U6as5tCh03ToUIPx47tSr17ZQIdkcsCbRPEWcKkX04wxhVRsbALHjsVRsWJxRITx47uyZMlu7r67sT0TUQBkmihE5ArgSqCciAzymFUCp8c6Y4xh9uy/eOSRb6lZszTff98LEaFevbJ2FlGAZHVGEQYUc5fx7F/wOHCbP4MyxuR9u3YdZ8CAuUyfvg6A4sXDOXTotDW9UQBlmihU9RfgFxGZrKp/52JMxpg8LCkpmXHjljBs2I+cOHGGokVDGT68PY89djkhIfZMREHkzTWKWBF5FbgEiEiZqKod/BaVMSZPSk5W2radzG+/7QTgppvq8+abXahatWSAIzMZOu6b3/jepP//4jTfUQP4P5xukpb4ZOvGmHwlKEjo1KkWVaqU4Kuv7mTGjB6WJPKyXe7Nqce356gYbxJFlKq+BySo6i+q+g/AziaMKQRUlc8+W8MXX6xLnTZ0aGvWrXuE7t3rBTAy45Ugt9KoyYM5KsabqqcE9+8eEekG7AbK5Girxpg8b8uWw/Tr9y3ffbeFcuWK0KFDDUqXjiQ8PIRwa78vfwkvlaPVvUkUI0WkJDAY5/mJEsCAHG3VGJNnxccn8uqrC3nhhQXExSVSunQEL7zQgZIlI7Jf2RRI2SYKVf3GHTwGtIfUJ7ONMQXMzz9v5+GHZ7Fhw0EAevVqwqhRnShfvmiAIzOBlNUDd8HAHThtPM1R1TUicj3wFBAJNM+dEI0xuSEpKZl+/ZwkUa9eFBMmdKN9+xqBDsvkAVmdUbwHVAH+AMaIyG4gGnhCVb/MhdiMMX6WnKzExSVSpEgowcFBTJjQjfnz/+bxx1sTHm4N+OV7qj4pJqtPQjTQRFWTRSQC2AvUUtVDPtmyMSagVq/eR9++s6hfP4r33rsRgLZtq9O2bfXABmZ8Z9nrzt8cJoysEsUZVU12tqFxIrLVkoQx+d+pU2cYPvwXRo9eRGJiMtu2HeHIkdOULh0Z6NCMrxWrDLH7oXiVHBWTVaKoLyKr3GEBarnjAqiqNsnRlo0xue7rrzfSv/9sduw4hgj06xfNCy9cQ6lSdkdTgaMK+5c5w5WuzFFRWSWKBjkq2RiTZyQmJtOjx3T+9z+nc8pmzS7i7bevp2XLSgGOzPiNZ/MdxSrnqKisGgW0hgCNKSBCQoIoWTKcYsXCGDGiPf37t7QG/Aq6re6TDSERULRCjory6ydFRLqIyEYR2SwiT2SyzB0isk5E1orIJ/6Mx5jCZPHiGBYvjkkdf/XVjqxf/wgDBrSyJFEYnDnp/I3IeUMafrv/zX0OYxzQEYgBlojITFVd57FMHeBJoLWqHhGR8v6Kx5jC4ujROJ58ch5vv/0n9euXZcWKvoSFBRMVZf1EFEoNeuW4CK8ShYhEAlVVdeN5lN0S2KyqW90ypgI3Aus8lnkAGKeqRwBUdf95lG+M8aCqfPrpGgYNmsu+facICQmie/d6JCUlY51SmpzINlGIyA3AKJwe72qISDNguKp2z2bVSsBOj/EY4PJ0y9R1t/Ebzif5eVWd413oxpgUf/11iH79vmXevK0AtG5dhYkTr6dRIztJNznnzRnF8zhnBz8DqOoKEfHVc/0hQB2gHVAZmC8ijVX1qOdCIvIg8CBA1apVfbRpYwqGhIQkOnT4iJiY45QpE8krr1zL/fc3JyhIAh2aKSC8amZcVY+JnPWh8+Yxv104TYCkqOxO8xQDLFbVBGCbiGzCSRxndYykqpOASQDR0dG+eSbdmHxOVRERQkODeeGFDvz003ZeeeVaypWzBvyMb3lz68NaEbkLCBaROiLyFrDQi/WWAHVEpIaIhAF3AjPTLfMlztkEIlIWpypqq5exG1Mo7dt3kl69ZjBy5PzUaffe25QPPrjRkoTxC28SxaM4/WXHA5/gNDc+ILuVVDUR6A/MBdYDn6vqWhEZLiIp1zfmAodEZB3wEzDEmgkxJmPJycrbby+lfv1xTJmyitGjF3HiRHygwzKFgDdVT/VV9Wng6fMtXFW/Bb5NN+1Zj2EFBrkvY0wmVq7cS9++s1i0yHkuokuX2owb15Xixa2rOZOJg6uyX8ZL3iSK10TkImA68JmqrvHZ1o0xWUpISOLJJ3/gjTcWkZSkVKxYjDff7MJttzUk3XVDY9IkxsGGT53hpJyfdWZb9aSq7XF6tjsAvC0iq0VkWI63bIzJVkhIEMuX7yU5WXn00ZasX/8It99+iSUJk7XlY9OGowfnuDjR82inXEQaA48DPVQ1LMdbvwDR0dG6dOnSQGzamFyxY8cxkpKSqVGjNOA8I3HsWDzR0RcHODKTb0y4CGL3OcODnWO8iPypqtEXUly2ZxQi0kBEnheR1UDKHU85a4rQGHOOhIQkRo1aSIMG43jgga9J+RFXp06UJQlzflKSxM2zfFKcN9co3gc+Azqr6m6fbNUYc5bff99J376zWLXK+YKXKRNJbGwCRYsG5MTd5GfHPRrEKFHNJ0VmmyhU9QqfbMkYc44jR07zxBPzmDTJ6WCmRo1SjBvXleuuqxPgyEy+te7DtOGohj4pMtNEISKfq+odbpWT54UM6+HOGB+Ij0+kWbO32bHjGKGhQQwZciVPP92GIkVCAx2ayc/2LHL+VroafHTTQ1ZnFP9y/17vky0ZY84SHh5Cnz7N+eGHbUyY0I2GDcsFOiRTEMS4T+xf1NJnRWZ6MVtV97iD/VT1b88X0M9nERhTSMTFJfLccz/xySerU6c99dTV/PzzfZYkjG/EH4MzJ5zhurf6rFhvmvDomMG063wWgTGFwPffb6Fx4wkMHz6fgQPncvp0AuA8J2HPRBifObUvbfiiy3xWbFbXKB7GOXOoKSKez4IXB37zWQTGFGB7955k0KC5fPqp06DBJZeUY+LE64mMtOsQxo9K14Eg33VgmlVJnwCzgf8Anv1dn1DVwz6LwJgCKCkpmbff/pOnnvqBY8fiiYwM4bnn2jJw4BWEhVlvcyZ/ySpRqKpuF5FH0s8QkTKWLIzJXFKS8tZbf3DsWDxdu9Zh7NjrUp+0Nia/ye6M4nrgT5zbYz0rUhWo6ce4jMl3TpyIJylJKVUqgrCwYN555wb27TvJLbc0sOsQJnecjPFLsZkmClW93v3rq25PjSmQVJUZMzbw2GOz6dy5Fu+9dyMAV11l3faaXJZya+zxv31arDdtPbUWkaLu8D0iMlpE7BtgDLB9+1G6d5/Krbd+zq5dJ1iz5gBxcYmBDssUVivGO38b9PJpsd7cHjsBiBWRpsBgYAvwsU+jMCafSUhI4uWXf6Vhw3F8880mSpQIZ+zY61i48B9ERPjubhNjvLZ3KZw+4AyXa+rTor35RCeqqorIjcBYVX1PRPr4NApj8pHY2ARatXqX1av3A3DnnY0YPboTFSsWD3BkplDbPidtuKFvzyi8SRQnRORJoBdwtYgEAXYTuCm0ihQJJTr6YmJjExg/vhudOtUKdEjGpGkxGCJK+bRIbxJFD+Au4B+qute9PvGqT6MwJg9TVT76aCW1apVJvUD9+uudCQsLtgfnTN4TEuHzIr3pCnUv8F+gpIhcD8Sp6kc+j8SYPGj9+gO0b/8hvXt/xYMPfs2ZM0kAlCwZYUnCFBre3PV0B/AHcDtwB7BYRG7zd2DGBNLp0wkMG/YjTZtO5Jdf/qZcuSI8+eRVhIZ6c/+HMQWLN1VPTwOXqep+ABEpB8wDpvszMGMCZc6czTzyyLds3XoEgAceuJSXXrqWMmUiAxyZMYHhTaIISkkSrkN4d1utMfnOyZNn6NVrBgcPxtKoUXkmTuxG69b22JAp3LxJFHNEZC7wqTveA/jWfyEZk7uSkpJJTlZCQ4MpViyMN9/sQkzMcQYObEVoqDXgZ4w3fWYPEZFbgKvcSZNUdYZ/wzImd/z5524eeugbbryxHs880xaAu+5qHOCojLkAq9/zW9FZ9UdRBxgF1AJWA/9W1V1+i8SYXHT8eDzPPPMjY8cuITlZOX48nieeuMrOIEz+FVbM+RtazOdFZ3Wt4X3gG+BWnBZk3/L51o3JZarKtGlrqV9/LGPG/IEIDBrUimXLHrIkYfI5t4Ximl19XnJWVU/FVfUdd3ijiCzz+daNyUUnTsTTo8d0Zs/eDMDll1di4sTradbsogBHZkzellWiiBCR5qT1QxHpOa6qljhMvlKsWBjx8UmULBnOSy9dy4MPtiAoyPqJMCY7WSWKPcBoj/G9HuMKdPBXUMb4yvz5f1OxYjHq1IlCRHj//e5ERIRQoYLv63GNKaiy6riofW4GYowvHTwYy+OPf88HH6zgmmtq8P33vRARqlUrFejQjMl3rOF8U6AkJyuTJ69gyJDvOXz4NGFhwVx9dVWSkpSQEKtmMuZC+PUJaxHpIiIbRWSziDyRxXK3ioiKSLQ/4zEF29q1+2nXbjJ9+szk8OHTXHNNDVavfpjnnmtHSIg1JmDMhfLbGYWIBAPjgI5ADLBERGaq6rp0yxUH/gUs9lcspuA7diyOVq3e4+TJM5QvX5TRoztx112NEbGzCGNyKttEIc437W6gpqoOd/ujuEhV/8hm1ZbAZlXd6pYzFbgRWJduuRHAy8CQ8w3eGFVFRChZMoKhQ1uza9dxXnzxGkqXtgb8TCFzcLXfivbmfHw8cAXQ0x0/gXOmkJ1KwE6P8Rh3WioRuRSooqqzsipIRB4UkaUisvTAgQNebNoUdLt2Hee22z5nypRVqdOefvpqJky43pKEKXz2LkkbDivh8+K9SRSXq+ojQByAqh4BwnK6YbdL1dHA4OyWVdVJqhqtqtHlypXL6aZNPpaYmMybby6ifv1xfPHFep577meSkpIBrJrJFF5bZqYNl6zu8+K9uUaR4F5vUEjtjyLZi/V2AVU8xiu701IUBxoBP7tf8IuAmSLSXVWXelG+KWSWLNlF376zWLZsDwA33VSfMWO6EBxsF6pNIbdqkvO34b1+Kd6bRDEGmAGUF5EXgNuAYV6stwSoIyI1cBLEnTh9bwOgqseAsinjIvIzTsODliTMWU6dOsPQofMYP34JqlC1akneeus6unevF+jQjAm85CSIdbsMqtDCL5vwppnx/4rIn8A1OM133KSq671YL1FE+gNzgWDgfVVdKyLDgaWqOjPrEoxxhIQEMW/eVoKChEGDruC559pStGiOaz+NKRgWeDx50LCXXzYhqpr1As5dTudQ1R1+iSgb0dHRunSpnXQUdFu2HKZUqQiioooATrVTREQIjRtXCHBkxuQhf82Ambc4wxFl4JFDmS4qIn+q6gU9q+ZN1dMsnOsTAkQANYCNwCUXskFjshIfn8irry7khRcWcPfdjXn33e4AXHZZpWzWNKYQ2r8ibfjOX/22GW+qns7q7su9pbWf3yIyhdbPP2/n4YdnsWHDQcC5wykpKdkuVhuTmZQ7/a54DqIa+G0z5/1ktqouE5HL/RGMKZz27z/FkCHf89FHKwGoVy+KCRO60b59jQBHZkwel5yQK5vx5snsQR6jQcClwG6/RWQKlYMHY2nQYByHD58mPDyYp5++mscfb014uLVXaUymVOGjJnBwTa5szptvY3GP4UScaxZf+CccU9iULVuEG2+sR0zMccaP70bt2mUCHZIxed/RLWcniWqd/Lq5LBOF+6BdcVX9t1+jMIXGqVNnGD78F7p1q0ubNtUAGD++G+HhwfZktTHeSopLGx6YCEH+7e8900QhIiHusxCt/RqBKTS+/noj/fvPZseOY8ya9RerVj1MUJAQEWHVTMZckKhL/J4kIOszij9wrkesEJGZwDTgVMpMVf2fn2MzBcTOncf417/mMGPGBgCaN7+It9++3vqrNuZCHUrfCLd/efNTLgI4hNNHdsrzFApYojBZSkxMZsyYxTz77E+cOpVAsWJhjBzZnkceaWkdCRlzoZKT4JsezvCxrbmyyawSRXn3jqc1pCWIFFk/zm0McPx4PP/5z6+cOpXArbc24I03ulC5su+bQDamUNn3Z9pw99y5ryirRBEMFOPsBJHCEoXJ0NGjcURGhhAeHkKZMpG8/fb1hIcH061b3UCHZkz+p8nwifsYW3A41LguVzabVaLYo6rDcyUKk++pKp9+uoaBA+fSv/9lPPNMWwBuucV/T4saU+h43hLb8onMl/OxrBKFXWk0Xtm06RD9+s3ihx+2ATB//o7ULkqNMT708aVpw1c+n2ubzSpRXJNrUZh8KS4ukZdf/pUXX/yVM2eSKFMmkldf7Ujv3s0sSRjjawfXgiY5ww3uztVNZ5ooVPVwbgZi8pe9e0/Sps0H/PWX8zHp3bsZr77akbJliwQ4MmMKqP95XI/oOClXN21POpkLUqFCUapUKUlISBATJnSjbdvqgQ7JmIJrzx9wYqczfNnjEJq7P8gsURivJCcr77zzJ+3b16Bu3ShEhE8+uYXSpSMJC/P/k6HGFGqbv0wbbvVMrm/ennoy2Vq5ci+tW79P376z6NdvFim9IlaoUMyShDG5wn0ioeWTEFYs17duZxQmUydPnuH553/mjTcWkZSkXHxxcfr2vaCeFI0xFyohFv54yRkOK571sn5iicJk6MsvN/Doo7OJiTlOUJDw6KMtGTmyAyVKhAc6NGMKl+keTYiHRAQkBEsU5hy7dh3nzjunEx+fRIsWFZk48Xqioy8OdFjGFE5HNqYNNw1ML9SWKAwACQlJhIQEISJUqlSCF17oQFhYMP36XWZ9VhsTKHFH4LTThzz/2AQhgTmjtyOAYeHCnbRoMYkpU1alThs8+EoeffRySxLGBNK2OWnDJWsGLAw7ChRihw+f5qGHvqZ16/dZvXo/48cvTb2jyRiTB/w8wPkbWTZXOijKjFU9FUKqypQpqxg8+DsOHIglNDSIxx9vzdNPX21NbxiTl8Tud/5e+q+AhmGJopDZt+8kPXt+wU8/bQegbdtqTJjQjQYNygU2MGPM2TyrnRr9I3BxYImi0ClVKoI9e05StmwRRo3qyL33NrWzCGPyoj/+kzZcLLB3HVqiKAS+/34Ll15akaioIoSHhzBt2u1UrFiMqChrwM+YPOnQOoiZ7wxf8XxAQwG7mF2g7dlzgp49v6BTpykMHTovdXqjRuUtSRiTl62YkDbc5IHAxeGyM4oCKCkpmbff/pMnn/yB48fjiYwMoV69KOtMyJj8IOkMxPzsDNe+KeDVTmCJosBZtmwPfft+w5IluwHo1q0OY8d2pXr1UoENzBjjnQ+bpD2NXbltYGNxWaIoQLZvP0rLlu+QlKRUqlScMWOu4+ab69tZhDH5xfr/ejTZIVDvjoCGk8KviUJEugBvAsHAu6r6Urr5g4B/AonAAeAfqvq3P2MqyKpXL8X99zejePFw/u//2lG8uDXgZ0y+sXws/Pho2nj/oxBeImDhePLbxWwRCQbGAdcBDYGeItIw3WLLgWhVbQJMB17xVzwF0fbtR7nhhk/55ZftqdMmTbqB0aM7W5IwJr/xTBL3rswzSQL8e0bREtisqlsBRGQqcCOwLmUBVf3JY/lFwD1+jKfASEhIYvTo3/m///uF06cTOXgwlt9/7wNg1UzG5Ed//S9tuPc6iGoQuFgy4M9EUQnY6TEeA1yexfJ9gNkZzRCRB4EHAapWreqr+PKlX3/dQd++37B27QEA7ryzEaNHd8pmLWNMnvZ32u3reS1JQB65mC0i9wDRQIaX+FV1EjAJIDo6ulC2WnfkyGmGDPme995bDkCtWqUZP74bnTrVCnBkxpgcOboFVrrPTbQeEdhYMuHPRLELqOIxXtmddhYRuRZ4GmirqvF+jCdfS05WvvpqI6GhQTzxxFU8+eRVREaGBjosY0xOLXk1bbhkjcDFkQV/JoolQB0RqYGTIO4E7vJcQESaA28DXVR1vx9jyZc2bDhIjRqlCA8PISqqCP/97y1UrVqS+vXLBjo0Y4wvJMbDqred4SrtoW7euB02Pb/d9aSqiUB/YC6wHvhcVdeKyHAR6e4u9ipQDJgmIitEZKa/4slPYmMTePrpH2jSZAKvvPJb6vROnWpZkjCmIPn7u7ThaydCcN6sJfDrNQpV/Rb4Nt20Zz2Gr/Xn9vOjOXM206/fLLZtOwrAwYOxgQ3IGOMfhzfBl+5v5mIXQ5m6gY0nC3niYraB3btPMGDAHKZNc+4ebty4PBMnXs+VV1bJZk1jTL4zvfPZZxMtnwxcLF6wRJEHbNp0iOjoSZw4cYYiRUJ5/vm2DBjQitDQwHV9aIzxMVXYuwRm9YRjW9OmRw+B5v0DF5cXLFHkAXXqlOGyyypRtGgob711HdWqlQp0SMYYX5t1F2yceva0f52GkIjAxHMerD+KADh+PJ4BA+awadMhwHmaeubMO5k5s6clCWMKouM7zk4SdW+Dhw/kiyQBdkaRq1SV6dPX8a9/zWHPnpNs2HCQOXOcVkuKFg0LcHTGGL95v07a8MMHoEj+unvREkUu2br1CP37f8vs2ZsBaNWqMi+/bDd9GVPgnT7kdEYEzvWIfJYkwBKF3505k8SoUQsZMWI+cXGJlCoVwUsvXcMDD7QgKMga8DOmwPttWNrw1f8JXBw5YInCz3buPMbw4b8QH5/E3Xc35rXXOlGhQrFAh2WMyQ0bPoOVE53hiq0gKH/eyWiJwg+OHDlNqVIRiAi1apXhzTe7ULt2Ga65pmagQzPG5KYf+qUNd3oncHHkkN315EPJycr77y+ndu23mDJlVer0hx6KtiRhTGGydwm8EQ5xh53xHgugbKPAxpQDlih8ZO3a/bRrN5k+fWZy+PDp1IvWxphC5tg2+G/LtAvYAJVaBy4eH7CqpxyKjU1gxIhfGDXqdxITkylfviivv96Znj3z768HY8x5+nse7PgBts2BAyvSpl8zDpo8BPm850lLFDmwadMhOneewvbtRxGBvn1b8OKL11C6dGSgQzPG5JZ9y2F6x3On1+sBzfqdOz0fskSRA9WqlSQiIoSmTSswceL1tGpVOdAhmTwmISGBmJgY4uLiAh2K8ZczCXCV24tzWDEgGEIjITgc1q/P9XAiIiKoXLkyoaG+a7LcEsV5SExMZuLEpfTs2YioqCKEh4cwZ87dVKpUgpAQu9xjzhUTE0Px4sWpXr06ks+rH0wGNBn2LQPKQlhxKFMvsOGocujQIWJiYqhRw3e95dnRzUt//LGLli3f4dFHZzN0aFpH6NWqlbIkYTIVFxdHVFSUJYmCJjnJuVi9b1natKIXBS4el4gQFRXl8zNYO6PIxrFjcTz99I+MH78EVahatSQ33hjYXw0mf7EkUcCcOQGHN549LaI0hJcMTDzp+OPzZokiE6rKZ5+tZeDAuezde5KQkCAGDWrFs8+2tQb8jCmsNBmO/JU2HhQCocWgVK3AxZQLrM4kEytX7qNnzy/Yu/ckV15ZhWXLHuTllztakjD5TnBwMM2aNaNRo0bccMMNHD16NHXe2rVr6dChA/Xq1aNOnTqMGDECVU2dP3v2bKKjo2nYsCHNmzdn8ODBAXgHWVu+fDl9+vTx/4ZUnaomTXbGi10M5ZtB6dpZrhYfH0+PHj2oXbs2l19+Odu3b89wuTlz5lCvXj1q167NSy+9dM78xx57jGLF0pr/GTt2LO+///6Fvpvzo6r56tWiRQv1l8TEpLPGBw6co++886cmJSX7bZumYFu3bl2gQ9CiRYumDt977706cuRIVVWNjY3VmjVr6ty5c1VV9dSpU9qlSxcdO3asqqquXr1aa9asqevXr1dV1cTERB0/frxPY0tISMhxGbfddpuuWLHC/9uMP666Z4nz2rdcNcm7csaNG6cPPfSQqqp++umnescdd5yzTGJiotasWVO3bNmi8fHx2qRJE127dm3q/CVLlug999xz1v/y1KlT2qxZswy3mdHnDliqF3jcDfiB/3xf/koUP/64VevXH6u//LLdL+WbwumsL+wo/PPKhufBZcKECfrwww+rquq7776rvXr1OmvZzZs3a+XKlVVVtVevXvree+9lW/6JEye0d+/e2qhRI23cuLFOnz79nO1OmzZN77vvPlVVve+++/Shhx7Sli1b6sCBA7VatWp65MiR1GVr166te/fu1f379+stt9yi0dHRGh0drb/++us52z5+/LjWrVs3dXzx4sXaqlUrbdasmV5xxRW6YcMGVVX94IMP9IYbbtD27dtrmzZt9OTJk3r//ffrZZddps2aNdMvv/xSVVW3bdumV111lTZv3lybN2+uv/32m1NwQpzq3qVpiSLZ+x+PnTp10oULFzrFJCRoVFSUJqdbf+HChdqpU6fU8RdffFFffPFFVXWSSLt27XT37t1n7VNV1ZtuukkXL158zjZ9nSgK/TWK/ftPMWTI93z00UoARo/+nTZtqgU4KmN8LykpiR9++CG1mmbt2rW0aNHirGVq1arFyZMnOX78OGvWrPGqqmnEiBGULFmS1atXA3DkyJFs14mJiWHhwoUEBweTlJTEjBkzuP/++1m8eDHVqlWjQoUK3HXXXQwcOJCrrrqKHTt20LlzZ9aney5h6dKlNGqU1gpC/fr1WbBgASEhIcybN4+nnnqKL774AoBly5axatUqypQpw1NPPUWHDh14//33OXr0KC1btuTaa6+lfPnyfP/990RERPDXpo30vOsulv46D45uSdtoZFmubtOGEydOnPO+Ro0axbXXnt3PzK5du6hSpQoAISEhlCxZkkOHDlG2bNkMlwGoXLkyixcvBpwqpu7du1OxYsVzthcdHc2CBQto2bJltvs8JwptokhOVt57bxlDh87jyJE4wsODGTasDUOGXBno0ExBNVizX8YPTp8+TbNmzdi1axcNGjSgY8cMniLOgXnz5jF1alo3n6VLl852ndtvv53gYKfJ7R49ejB8+HDuv/9+pk6dSo8ePVLLXbduXeo6x48f5+TJk2fV0+/Zs4dy5cqljh87doz77ruPv/76CxEhISEhdV7Hjh0pU6YMAN999x0zZ85k1KhRgHMb846/t3NxkTj6Dx3JitXrCA5SNm3ZcXaSKF4Zil7EggULzmcXXbDdu3czbdo0fv755wznly9fng0bNvg9jkKZKLZtO8I998xg4cKdAHTqVItx47pSu3aZAEdmjO9FRkayYsUKYmNj6dy5M+PGjeOxxx6jYcOGzJ8//6xlt27dSrFixShRogSXXHIJf/75J02bNr2g7Xreppn+vv6iRYumDl9xxRVs3ryZAwcO8OWXXzJsmNPRT3JyMosWLSIiIvN+pSMjI88q+5lnnqF9+/bMmDGD7du3065duwy3qap8Me1z6tWrC6cPOs9ExO7n+VGTqFCmKCvnfUxycjIR1a+C4DBnfuk6qbfAXn311V6fUVSqVImdO3dSuXJlEhMTOXbsGFFRURkukyImJoZKlSqxfPlyNm/eTO3azgXz2NhYateuzebNm1P3a2Sk/5sMKpR3PZUoEc6mTYe46KJiTJ16K3Pm3G1JwhR4RYoUYcyYMbz22mskJiZy99138+uvvzJvnvMA6enTp3nsscd4/PHHARgyZAgvvvgimzZtApwD98SJE88pt2PHjowbNy51PKXqqUKFCqxfv57k5GRmzJiRaVwiws0338ygQYNo0KBB6kG0U6dOvPXWW6nLrVix4px1GzRokHrQBOeMolKlSgBMnjzZmagKCbGQGOe07Hr8bzq3bsJbrz6P7lsOJ2JYvtg5Qzh2/CQVK5QjqHQtPp61hKSkJCjXBC6KPus5iQULFrBixYpzXumTBED37t358MMPAZg+fTodOnQ451mHyy67jL/++ott27Zx5swZpk6dSvfu3enWrRt79+5l+/btbN++nSJFipz1fjdt2nRW1Zu/FJpEMXfuZuLjEwGIiirCzJl3smHDI/To0cgeiDKFRvPmzWnSpAmffvopkZGRfPXVV4wcOZJ69erRuHFjLrvsMvr37w9AkyZNeOONN+jZsycNGjSgUaNGbN269Zwyhw0bxpEjR2jUqBFNmzblp59+AuCll17i+uuv58orr8ywft1Tjx49mDJlSmq1E8CYMWNYunQpTZo0oWHDhhkmqfr163Ps2LHUX/ePP/44Tz75JM2bNycx0fm+s38ZxO6DM8ed/qtjD/DMwD4kJCTSpENPLml7B8+8OgmKXkS/AUP58H8/0LRVBzZsjTnrLORC9enTh0OHDlG7dm1Gjx6deuvr7t276dq1K+Bcuxg7diydO3emQYMG3HHHHVxyySXZlv3bb7/5vCoxI6IamHrTCxUdHa1Lly71evmdO4/x2GNz+PLLDYwY0Z5hw9r4MTpjzrZ+/XoaNGgQ6DAKtNdff53ixYvzz3/+0zl7SDwNp/amdRrkKaK08xxEeEnnQbnQIrkfsI8sX76c0aNH8/HHH58zL6PPnYj8qarRF7KtAnuNIjExmTFjFvPssz9x6lQCxYqFUaaMNf9tTEHz8MMPM+2zqXByt/PKTIUW+b5fCE8HDx5kxIgRubKtApkoFi2KoW/fb1i5ch8At97agDff7EKlSiUCHJkxxidUnYvQcYeISIyjV+dG5yaJ8BJQ5CKn6W8peLXsuVHllKLAJYrFi2O48sr3UIXq1Usxdux1dOtWN9BhmUJMVe062IVQhYSTTlVR4mmI3e/08QBOw3wZCQ6DqIZOG0yFlD8uJxS4vdmyZSU6d65N8+YXMWxYG4oU8V3nHcacr4iICA4dOmRNjWckKQFIPnta7EHQROcOpYySgWc/1CmKlIewEs4ZRAE8czgfqk5/FFndUnwh8n2i+OuvQwwcOJfRoztTt67zZZw16y6CguxLaQKvcuXKxMTEcODAgUCHEjgpCSH2gHN2IEGgSedXRkgEJCU61UgpZxVBwRAUCkdOAaeAPT4OPH9K6eHOl/JtooiPT+Sll37lP//5lfj4JCIiQpg+/Q4ASxImzwgNDfVpT2N+k5QAyRn8WvfWqX2wfgoku7ekbp3l3JbqjeJV04Y1yTk7aPYIJMVD3dugRNXM1zW5wq+JQkS6AG8CwcC7qvpSuvnhwEdAC+AQ0ENVt2dX7g8/bKVfv2/ZtOkQAPff34xXXsm9CzvGnEWT4cTO7JdL7+AaiD/m+3g8JcTC399BZDnY+aNT7x+Srlri2DbnGQN/K9cEStaEdq+7/TgUdW5XNXme356jEJFgYBPQEYgBlgA9VXWdxzL9gCaq2ldE7gRuVtUeGRboiipdXQ8fvR+ABnWKMfGVxrS5IiqrVQIvKQF2LUg7ZS5Mks7AobXOASuv2PmT8+s5tFj2y3rj5C7flJMXhOTguYLEWKhzq5MQwNm/l9znPLNQiC8u5xV59TmKlsBmVd0KICJTgRuBdR7L3Ag87w5PB8aKiGgW2evI0TNEhCTwbMdfGNz2d8K2JMGWzJY2Jgu+/jUfWfb8DrSa7CSZ+nf6No70EmKd/pzLNnYuFFe6+tyLvkGhzt1CdsHdZMCfZxS3AV1U9Z/ueC/gclXt77HMGneZGHd8i7vMwXRlPQg86I42Atb4Jej8pyxwMNulCgfbF2lsX6SxfZGmnqoWv5AV88X5oKpOAiYBiMjSCz19KmhsX6SxfZHG9kUa2xdpRMT7to/S8edNx7uAKh7jld1pGS4jIiFASZyL2sYYY/IIfyaKJUAdEakhImHAncDMdMvMBO5zh28Dfszq+oQxxpjc57eqJ1VNFJH+wFyc22PfV9W1IjIcp+/WmcB7wMcishk4jJNMsjPJXzHnQ7Yv0ti+SGP7Io3tizQXvC/yXTPjxhhjclfhbhjFGGNMtixRGGOMyVKeTRQi0kVENorIZhF5IoP54SLymTt/sYhUD0CYucKLfTFIRNaJyCoR+UFEqgUiztyQ3b7wWO5WEVERKbC3RnqzL0TkDvezsVZEPsntGHOLF9+RqiLyk4gsd78nXQMRp7+JyPsist99Ri2j+SIiY9z9tEpELvWqYFXNcy+ci99bgJpAGLASaJhumX7ARHf4TuCzQMcdwH3RHijiDj9cmPeFu1xxYD6wCIgOdNwB/FzUAZYDpd3x8oGOO4D7YhLwsDvcENge6Lj9tC/aAJcCazKZ3xWYDQjQCljsTbl59YwitfkPVT0DpDT/4elG4EN3eDpwjRTMBv+z3Req+pOqpjSmtAjnmZWCyJvPBcAI4GUgLjeDy2Xe7IsHgHGqegRAVffncoy5xZt9oUBKF5clgSz6TM2/VHU+zh2kmbkR+Egdi4BSIlIxu3LzaqKoBHg2xxnjTstwGVVNBI4Bebx1wAvizb7w1AfnF0NBlO2+cE+lq6jqrNwMLAC8+VzUBeqKyG8isshtzbkg8mZfPA/cIyIxwLfAo7kTWp5zvscTIJ804WG8IyL3ANFA20DHEggiEgSMBnoHOJS8IgSn+qkdzlnmfBFprKpHAxlUgPQEJqvqayJyBc7zW41UNTm7FU3ePaOw5j/SeLMvEJFrgaeB7qoan0ux5bbs9kVxnEYjfxaR7Th1sDML6AVtbz4XMcBMVU1Q1W04zf7XyaX4cpM3+6IP8DmAqv4OROA0GFjYeHU8SS+vJgpr/iNNtvtCRJoDb+MkiYJaDw3Z7AtVPaaqZVW1uqpWx7le011VL7gxtDzMm+/IlzhnE4hIWZyqqK25GGNu8WZf7ACuARCRBjiJojD2TzsTuNe9+6kVcExVs+1DNk9WPan/mv/Id7zcF68CxYBp7vX8HaraPWBB+4mX+6JQ8HJfzAU6icg6IAkYoqoF7qzby30xGHhHRAbiXNjuXRB/WIrIpzg/Dsq612OeA0IBVHUizvWZrsBmIBa436tyC+C+MsYY40N5terJGGNMHmGJwhhjTJYsURhjjMmSJQpjjDFZskRhjDEmS5YoTJ4kIkkissLjVT2LZU/6YHuTRWSbu61l7tO751vGuyLS0B1+Kt28hTmN0S0nZb+sEZGvRaRUNss3K6gtpZrcY7fHmjxJRE6qajFfL5tFGZOBb1R1uoh0AkapapMclJfjmLIrV0Q+BDap6gtZLN8bpwXd/r6OxRQedkZh8gURKeb2tbFMRFaLyDmtxopIRRGZ7/GL+2p3eicR+d1dd5qIZHcAnw/Udtcd5Ja1RkQGuNOKisgsEVnpTu/hTv9ZRKJF5CUg0o3jv+68k+7fqSLSzSPmySJym4gEi8irIrLE7SfgIS92y++4DbqJSEv3PS4XkYUiUs99Snk40MONpYcb+/si8oe7bEat7xpztkC3n24ve2X0wnmSeIX7moHTikAJd15ZnCdLU86IT7p/BwNPu8PBOG0/lcU58Bd1pw8Fns1ge5OB29zh24HFQAtgNVAU58n3tUBz4FbgHY91S7p/f8bt/yIlJo9lUmK8GfjQHQ7DackzEngQGOZODweWAjUyiPOkx/ubBnRxx0sAIe7wtcAX7nBvYKzH+i8C97jDpXDafyoa6P+3vfL2K0824WEMcFpVm6WMiEgo8KKItAGScX5JVwD2eqyzBHjfXfZLVV0hIm1xOqr5zW3eJAznl3hGXhWRYThtAPXBaRtohqqecmP4H3A1MAd4TURexqmuWnAe72s28KaIhANdgPmqetqt7moiIre5y5XEacBvW7r1I0Vkhfv+1wPfeyz/oYjUwWmiIjST7XcCuovIv93xCKCqW5YxGbJEYfKLu4FyQAtVTRCnddgIzwVUdb6bSLoBk0VkNHAE+F5Ve3qxjSGqOj1lRESuyWghVd0kTr8XXYGRIvKDqg735k2oapyI/Ax0BnrgdLIDTo9jj6rq3GyKOK2qzUSkCE7bRo8AY3A6a/pJVW92L/z/nMn6Atyqqhu9idcYsGsUJv8oCex3k0R74Jx+wcXpK3yfqr4DvIvTJeQioLWIpFxzKCoidb3c5gLgJhEpIiJFcaqNFojIxUCsqk7BaZAxo36HE9wzm4x8htMYW8rZCTgH/YdT1hGRuu42M6ROj4aPAYMlrZn9lOaie3ssegKnCi7FXOBRcU+vxGl52JgsWaIw+cV/gWgRWQ3cC2zIYJl2wEoRWY7za/1NVT2Ac+D8VERW4VQ71fdmg6q6DOfaxR841yzeVdXlQGPgD7cK6DlgZAarTwJWpVzMTuc7nM6l5qnTdSc4iW0dsExE1uA0G5/lGb8byyqcTnleAf7jvnfP9X4CGqZczMY58wh1Y1vrjhuTJbs91hhjTJbsjMIYY0yWLFEYY4zJkiUKY4wxWbJEYYwxJkuWKIwxxmTJEoUxxpgsWaIwxhiTpf8Hwzbicwt0KF0AAAAASUVORK5CYII=\n", "text/plain": [ "

" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "plt.figure()\n", "lw = 2\n", "plt.plot(\n", " fpr[0],\n", " tpr[0],\n", " color=\"darkorange\",\n", " lw=lw,\n", " label=\"ROC curve (area = %0.2f)\" % roc_auc[0],\n", ")\n", "plt.plot([0, 1], [0, 1], color=\"navy\", lw=lw, linestyle=\"--\")\n", "plt.xlim([0.0, 1.0])\n", "plt.ylim([0.0, 1.05])\n", "plt.xlabel(\"False Positive Rate\")\n", "plt.ylabel(\"True Positive Rate\")\n", "plt.title(\"Receiver operating characteristic example\")\n", "plt.legend(loc=\"lower right\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 98, "id": "660345f5-2dd9-41ce-96b2-c55d94922844", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:00:34.785255Z", "iopub.status.busy": "2022-11-27T16:00:34.784806Z", "iopub.status.idle": "2022-11-27T16:00:34.926373Z", "shell.execute_reply": "2022-11-27T16:00:34.925453Z", "shell.execute_reply.started": "2022-11-27T16:00:34.785231Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/FklEQVR4nO3dd3gU5fbA8e9JIQm9I9IFpErRCCoKCFIEBK8NG4piQYoFfhQFlIvgRUWUDlb0ehUVRVGaqChYUFroRUSE0HsLCSnn98dMwhKSzRKy2ZTzeZ59stPPTmbn7Pu+M++IqmKMMcakJyjQARhjjMnZLFEYY4zxyhKFMcYYryxRGGOM8coShTHGGK8sURhjjPHKEkUeISLrRaRloOMINBGZKiLDsnmb00VkZHZu019E5D4R+TaTy+bZY1BEVERqBDqOQBG7jyLrich2oByQCJwE5gN9VPVkIOPKa0SkO/CIql4f4DimA9GqOjTAcQwHaqjq/dmwrenkgM+cXUREgZqqujXQsQSClSj85xZVLQw0AhoDzwY2nAsnIiH5cduBZPvc5Eiqaq8sfgHbgZs8hl8B5ngMXwP8ChwFVgMtPaaVBN4DdgNHgC89pnUCotzlfgUapN4mcClwGijpMa0xcBAIdYcfBja6618AVPGYV4HewJ/A3+l8vs7AejeOH4E6qeJ4Ftjgrv89IPwCPsMgYA0QB4QAg4G/gBPuOv/lzlsHiOVsqe2oO346MNJ93xKIBvoD+4E9wEMe2ysFfA0cB5YBI4Gfvfxfr/f4v+0EuntscxIwx43zd6C6x3Lj3PmPAyuAGzymDQdmAh+60x8BmgC/udvZA0wECngsUw9YCBwG9gHPAe2BM0C8uz9Wu/MWA95x17PL/YzB7rTuwC/A68Ahd1r35H0AiDttvxvbWqA+8Ji7nTPutr5OfdwDwW5cyf+7FUCldPZrmt8H4Dqc47aSO9wQ55iq7Q6neWyk8dmOAtvc9XV3/xf7gQc95p8OTHX36wngJ87/XtRw34cBY4Ad7v6fCkQE+rzj13NaoAPIi69UX5iK7hdsnDtcwf1SdsAp0bVxh8u40+cAnwAlgFCghTu+sXtwN3W/hA+62wlLY5s/AI96xPMqMNV93wXYinOiDQGGAr96zKvul6VkWgc/cDlwyo07FBjorq+ARxzrgEruOn7h7Inbl88Q5S4b4Y67Eyf5BQFd3W2Xd6d1J9WJnfMTRQIwwo21AxADlHCnz3BfBYG6OCeQNBMFUAXnBHKPu65SQCOPbR7COcGHAP8DZngse787fwhO0tqLmzxxEkU8cKv7GSOAq3BOniFAVZyk/rQ7fxGck35/INwdbuqxrg9TxT0LmAYUAsoCfwCPe+y/BKCvu60Izk0U7XBO8MVxkkYdj32fsp/TOe4H4Bz3tdxlGwKl0tivGX0fRuEczxHu+vp4LJvRsZEAPIRzrI3EObFPwjnRt3X/n4U9Ps8JoLk7fRwexwLnJorXgdk4x3cRnB8b/wn0ecev57RAB5AXX+4X5qR74CnwPVDcnTYI+G+q+RfgnDTLA0m4J7JU80wBXkw1bjNnE4nnl/QR4Af3veCcAJu7w/OAHh7rCMI5eVZxhxVo5eWzDQM+TbX8Ls7+CtwO9PSY3gH46wI+w8MZ7NsooIv7vjsZJ4rTQIjH9P04J+FgnBN0LY9p6ZYocEpJs9KZNh14O9Vn3uTlMxwBGrrvhwOLM/jMTydvGydRrUpnvuF4JAqcdrI4PBK+u/wij/23I9U6UvYp0ArY4u6voPT2c6rjPvkY3Jz8f8rgs6X7fXDfh+Ikq7U4bX1yAcfGnx7TrsA5tst5jDvEucneM7kXximtJpdmFKiB8306xbklxmtJp/SdV17WRuE/t6pqEZyTVW2gtDu+CnCniBxNfuFUaZTH+SV9WFWPpLG+KkD/VMtVwvlFldrnwLUiUh7nF1ISsMRjPeM81nEY5+Cv4LH8Ti+f61Lgn+QBVU1y509v+X88YvTlM5yzbRF5QESiPOavz9l96YtDqprgMRyDcxIog/Mr2nN73j53JZxqjvTsTWMbAIjI/4nIRhE55n6GYpz7GVJ/5stF5BsR2Ssix4GXPObPKA5PVXBOtHs89t80nJJFmtv2pKo/4FR7TQL2i8ibIlLUx237Gqe37wOqGo9zEq8PvKbumRl8Ojb2ebw/7a4v9bjCHsMp+0KdC08Oc/73qwxOCXSFx3bnu+PzLEsUfqaqP+Ec6GPcUTtxfkEV93gVUtXR7rSSIlI8jVXtBEalWq6gqn6cxjaPAN/iFMfvxfmlpB7reTzVeiJU9VfPVXj5SLtxvtwAiIjgnBR2ecxTyeN9ZXcZXz+D54mgCvAW0Aen2qI4TrWW+BBnRg7gVE1UTCfu1HYC1S90IyJyA0713F04JcXiwDHOfgY4/3NMATbhXGVTFKeuP3n+ncBl6Wwu9Xp24pQoSnvs76KqWs/LMueuUHW8ql6FUzV3OU6VUobL4fv+8vZ9QEQqAC/gtHW9JiJh7viMjo3MSPn/i0hhnKql3anmOYiTYOp5xFtMnQtX8ixLFNnjDaCNiDTEabS8RUTaiUiwiISLSEsRqaiqe3CqhiaLSAkRCRWR5u463gJ6ikhTcRQSkY4iUiSdbX4EPADc4b5PNhV4VkTqAYhIMRG58wI+y6dARxFpLSKhOHXlcTiNkcl6i0hFESkJDMFpc8nMZyiEc0I64Mb6EM6vxmT7gIoiUuAC4gdAVROBL4DhIlJQRGrj7K/0/A+4SUTuEpEQESklIo182FQRnIR0AAgRkeeBjH6VF8FpPD7pxvWEx7RvgPIi8rSIhIlIERFp6k7bB1QVkSD3M+7B+cHwmogUFZEgEakuIi18iBsRudr9X4XiVLfE4pROk7eVXsICeBt4UURquv/rBiJSKo350v0+uD9CpuM0xvfAaZt50V0uo2MjMzqIyPXu8fQisFRVzylxuSXot4DXRaSsu+0KItLuIredo1miyAaqegD4AHjePfC64PxKPIDzi2oAZ/8X3XDqzjfh1Kc/7a5jOfAoTlXAEZwG5O5eNjsbqAnsVdXVHrHMAl4GZrjVGuuAmy/gs2zGaZydgPPr6hacS4HPeMz2Ec4JahtO9cPIzHwGVd0AvIZzBdA+nHrmXzxm+QHn6qu9InLQ18/goQ9ONdBe4L/AxzhJL61YduC0PfTHqZKIwmmgzcgCnKqJLTjVcLF4r+IC+D+ckuAJnJNScqJFVU/gNPje4sb9J3CjO/kz9+8hEVnpvn8AKMDZq9Bm4lbr+KCou/0jbuyHcC6MAOfkXdetfvkyjWXH4vyo+BYn6b2D0yB9jgy+D0/iVJMNc0vEDwEPicgNPhwbmfERTunlMM4FBendjzII59hd6n6HvsNptM+z7IY7k6XEudnwEVX9LtCxXCgReRm4RFUfDHQsJntJPruB8EJZicLkWyJS260SERFpglO9MSvQcRmT09idmCY/K4JT3XQpTvXFa8BXAY3ImBzIqp6MMcZ4ZVVPxhhjvMp1VU+lS5fWqlWrBjoMY4zJVVasWHFQVTN1Y2CuSxRVq1Zl+fLlgQ7DGGNyFRH5J+O50mZVT8YYY7yyRGGMMcYrSxTGGGO8skRhjDHGK0sUxhhjvLJEYYwxxiu/JQoReVdE9ovIunSmi4iMF5GtIrJGRK70VyzGGGMyz58liuk4D3xPz8043WDXxHlY+xQ/xmKMMfnWmTOJF7W83264U9XFIlLVyyxdgA/cfuaXikhxESnvPmzFmNzpi47w99xAR2FMigFft2HVbl8fQZK2QN6ZXYFzH+AS7Y47L1GIyGM4pQ4qV66cLcGZALCTrDFZrv4l+xn/c9OMZ/QiV3ThoapvAm8CREZGWne3gWYndO+qdYDb5gQ6CpNPbdhwgJUr93D//Q0AeECVFqOPUa3ayEyvM5CJYhfnPsy+ojvO5ASBSgZ2kjUmU2Ji4hk5cjGvvvorwcHCNddUpEaNkogIVasWv6h1BzJRzAb6iMgMoClwzNonstHFJgI7oRuTY8yb9ye9e8/l77+PAtCjx1WUKnXeI8ozzW+JQkQ+BloCpUUkGueh5aEAqjoVmIvzsPqtQAzOg9NNVrJkYEyetmvXcZ5+egEzZ24AoEGDckyd2pFrr62UwZIXxp9XPd2TwXQFevtr+/nShSYGSwTG5Gq9e8/lq682U7BgKCNGtOSpp64hJCTr73rIFY3ZJgPeEoQlA2PylISEpJRk8PLLNxEaGsxrr7WlcuViftumJYrcJqNSgyUGY/KkY8diGTr0B7ZsOcz8+fchItSqVZrPPrvT79u2RJGbWMnBmHxHVfnssw08/fR89uw5SXCwEBW1l8aNL+4mugthiSInSy8xWFIwJl/466/D9Okzj/nztwJw7bUVmTq1Ew0alMvWOCxR5BS+NkRbkjAmXxgz5leGDVtEbGwCxYuH8/LLN/HII1cSFCTZHoslikCxxGCM8SImJp7Y2AS6dWvAmDFtKVu2UMBisUSRnawh2hiTjgMHTrF58yGuv97pz27QoGa0bFmV5s2rBDgySxTZw9oajDHpSEpS3n13FQMHLiQkJIhNm/pQsmQEYWEhOSJJgCWK7OGZJCw5GGNc69btp2fPb/jlF6cj7TZtLiMmJp6SJbOu+42sYIkiO/W3jm+NMXDq1BlGjPiJsWOXkpCQRLlyhXjjjfZ07VoPkexvrM6IJQp/+6JjoCMwxuQwd9zxGfPnb0UEevWKZNSo1hQvHh7osNJlicKfPNsmqnUIbCzGmBxj0KBm7Nt3kilTOtK0acVAh5MhSxT+kLrx2toljMm3EhKSmDDhd7ZvP8q4cTcD0LJlVZYvfywg90RkhiWKrGZJwhjj+uOPXTz++DdERe0F4LHHrqJevbIAuSZJgCWKrJW6qskShDH50tGjsTz33PdMnbocVahSpRgTJ3ZISRK5jSWKrGRJwph8b8aMdTz99Hz27TtFSEgQ/ftfy7BhzSlUqECgQ8s0SxQXI70b6SxJGJNvffvtX+zbd4pmzSoxZUpHrrgiezvw8wdLFBcjvbutjTH5RlxcArt2neCyy0oA8Morbbjhhso8+GCjXNUO4Y0lisxIXZKwG+mMyZd++OFvnnhiDkFBwurVPSlQIJjSpQvy0EONAx1alsr6h6vmB6mvajLG5Cv79p2kW7dZtG79AVu2HAIgOvp4gKPyHytRXCjPO62tJGFMvpKUpLz11goGD/6eo0djCQ8PYejQGxgwoBkFCgQHOjy/sURxoexOa2PyrX/96xNmz94MQLt21Zk0qQPVq5cMcFT+Z1VPF8KzNGFXNhmT79x2W20uuaQwn3xyB/Pm3ZcvkgRYieLCWGnCmHxl9uzNREcfp1evqwF44IGG3HZbHYoUCQtwZNnLEoWvrDRhTL6xY8cxnnxyHl99tZmwsGDat6/BZZeVQETyXZIASxS+s9KEMXlefHwi48f/zgsv/MipU/EUKVKAkSNbUaVKsUCHFlCWKLxJ685rK00YkyctXRrN449/w5o1+wC48866vP56OypUKBrgyALPEoU3qZOElSaMybOGDVvEmjX7qFatOBMndqBDh5qBDinHsESRFrvz2pg8T1U5ceIMRYs6bQ4TJ97MBx+sZsiQ5hQsGBrg6HIWuzw2tbSeJ2GMyVM2bz7ITTf9l9tu+wRV54dgrVqlGTWqtSWJNFiJIjXrKtyYPCs2NoH//GcJo0f/wpkziZQqFcH27UepVq1EoEPL0SxRpMeShDF5ysKFf9Gr11y2bj0MwMMPN+KVV9pQqlTBAEeW8/m16klE2ovIZhHZKiKD05heWUQWicgqEVkjIoGt5/G8V8IYkyeoKg8//BVt237I1q2HqVu3DIsXd+edd7pYkvCR30oUIhIMTALaANHAMhGZraobPGYbCnyqqlNEpC4wF6jqr5gyZPdKGJPniAhVqxYnIiKE559vQb9+1+bpDvz8wZ9VT02Araq6DUBEZgBdAM9EoUDyRcrFgN1+jMc7u/PamDwjKmove/ac4OabnUtcBw1qRrduDawtIpP8WfVUAdjpMRztjvM0HLhfRKJxShN901qRiDwmIstFZPmBAwf8EauVJozJA06ciKNfvwVcddWbPPjglxw+fBqAsLAQSxIXIdCXx94DTFfVikAH4L8icl5MqvqmqkaqamSZMmWyPgorTRiTq6kqs2ZtpG7dybz++lIA7r33CkJDA32Kyxv8WfW0C6jkMVzRHeepB9AeQFV/E5FwoDSw349xncvzvgkrTRiT6/zzz1H69JnHN99sASAy8lKmTevElVeWD3BkeYc/0+0yoKaIVBORAsDdwOxU8+wAWgOISB0gHPBT3VIaUicJK00Yk6uoKrff/inffLOFokXDmDjxZpYu7WFJIov5rUShqgki0gdYAAQD76rqehEZASxX1dlAf+AtEXkGp2G7uybfJulvliSMybWSkpSgIEFEGDOmLVOnLuf119tRvnyRQIeWJ0l2nZezSmRkpC5fvvziV/SaOH8tSRiTaxw6FMPgwd8B8NZbnQMcTe4iIitUNTIzy1pLjyUJY3I8VeX996OoXXsSb7+9ig8+WEN09PFAh5Vv5M8uPOwObGNyjY0bD/DEE3P46ad/AGjZsipTpnSkYkV7TkR2yZ+Jwq5yMibHU1Wef34RL7/8C/HxSZQuXZDXXmtLt24NEJFAh5ev5M9EkcyqnYzJsUSEXbtOEB+fxKOPXsno0TdRsmREoMPKl/JforBqJ2NyrN27T3DwYAwNGpQD4JVX2tCjR2OaNasc4Mjyt/zXmG3VTsbkOImJSUyc+Ad16kzi7rtncuZMIgClSxe0JJED5L8SRTKrdjImR1i5cg+PP/4Ny5c7fYI2b16F48fjKF3augDPKfJvojDGBNTx43EMG/YDEycuIylJqVixKOPHt+fWW2tbY3UO43OiEJGCqhrjz2D8ztonjMkRVJXmzd9j9ep9BAcL/fpdw/DhLSlSJCzQoZk0ZNhGISLXicgGYJM73FBEJvs9sqxmnf8Zk2OICM88cw1NmlRg+fLHeO21dpYkcrAMu/AQkd+BO4DZqtrYHbdOVetnQ3znyXQXHtZlhzEBc+ZMImPH/kZwsDBgQDPAKVUkJSnBwfnvmppAuJguPHyqelLVnanqDBMzs7EcwZKEMdlqyZJ/6NlzDhs2HCAsLJgHHmhIuXKFERGCg60tIjfwJVHsFJHrABWRUOApYKN/wzLG5HYHD8YwcOBC3nsvCoCaNUsyeXJHypUrHNjAzAXzJVH0BMbhPMZ0F/At0MufQWU5a8Q2JtuoKtOnRzFgwEIOHTpNgQLBPPvs9QwefD3h4XahZW7ky3+tlqre5zlCRJoBv/gnJD+wRmxjstWHH67l0KHTtGpVjcmTO1CrVulAh2Qugi+JYgJwpQ/jcj5rnzDGL2Ji4jl2LJby5YsgIkye3IFly3Zz331X2D0ReUC6iUJErgWuA8qISD+PSUVxnliXO1i1kzF+NW/en/TuPZfLLivBwoXdEBFq1SptpYg8xFuJogBQ2J3H8/mCx3Eul80drNrJGL/Ytes4Tz+9gJkzNwBQpEgYhw6dtq438qB0E4Wq/gT8JCLTVfWfbIwp63iWJqzayZgskZiYxKRJyxg69AdOnDhDoUKhjBhxI08+2ZSQELsnIi/ypY0iRkReBeoB4ckjVbWV36LKKlaaMCZLJSUpLVpM55dfdgJw6621GTeuPZUrFwtwZMaffEn//8PpvqMa8G9gO7DMjzFlDStNGJPlgoKEtm2rU6lSUb766m5mzepqSSIf8KULjxWqepWIrFHVBu64Zap6dbZEmIrPXXhYlx3GXDRV5dNP1xMSEsTtt9cFIC4ugfj4JAoXLhDg6MyF8HcXHvHu3z0i0hHYDZTMzMYCwpKEMZny11+H6dVrLt9++xdlyhSkVatqlCgRQVhYCGHWf1++4kuiGCkixYD+OPdPFAWe9mdQxpjAiYtL4NVXf2XUqCXExiZQokQ4o0a1olix8IwXNnlSholCVb9x3x4DboSUO7NzLrt3wphM+fHH7TzxxBw2bToIQLduDRgzpi1lyxYKcGQmkLzdcBcM3IXTx9N8VV0nIp2A54AIoHH2hHiB7LkTxmRKYmISvXo5SaJWrVJMmdKRG2+sFuiwTA7grUTxDlAJ+AMYLyK7gUhgsKp+mQ2xZY5nkrD2CWO8SkpSYmMTKFgwlODgIKZM6cjixf8wcGAzwsKsAz/j8HYkRAINVDVJRMKBvUB1VT2UPaFdJEsSxni1du0+evacQ+3apXjnnS4AtGhRlRYtqgY2MJPjeEsUZ1Q1CUBVY0VkW65JEsaYdJ06dYYRI35i7NilJCQk8fffRzhy5DQlSkQEOjSTQ3lLFLVFZI37XoDq7rAAmnxPRY5ijdjGePX115vp02ceO3YcQwR69Ypk1KjWFC9uVzSZ9HlLFHWyLYqsYo3YxqQpISGJrl1n8sUXzsMpGzW6hGnTOtGkSYUAR2ZyA2+dAubOjgDB2ieMSSUkJIhixcIoXLgAL754I336NLEO/IzP/HqkiEh7EdksIltFZHA689wlIhtEZL2IfJTpjVm1kzHn+P33aH7/PTpl+NVX27BxY2+efvoaSxLmgvjt+jf3PoxJQBsgGlgmIrNVdYPHPDWBZ4FmqnpERMpmeoNW7WQMAEePxvLss98xbdoKatcuTVRUTwoUCKZUKXtOhMkcnxKFiEQAlVV18wWsuwmwVVW3ueuYAXQBNnjM8ygwSVWPAKjq/gtYf9qs2snkU6rKxx+vo1+/Bezbd4qQkCA6d65FYmISuemhlCbnyTBRiMgtwBicJ95VE5FGwAhV7ZzBohWAnR7D0UDTVPNc7m7jF5wjebiqzvctdGNMsj//PESvXnP57rttADRrVompUztRv37mC+nGJPOlRDEcp3TwI4CqRolIVt3XHwLUBFoCFYHFInKFqh71nElEHgMeA6hcuXIWbdqYvCE+PpFWrT4gOvo4JUtG8MorN/HQQ40JCpJAh2byCJ+6GVfVYyLnHHTeH2Lh2IXTBUiyiu44T9HA76oaD/wtIltwEsc5D0ZS1TeBN8F5HoUP2zYmz1NVRITQ0GBGjWrFokXbeeWVmyhTxjrwM1nLl0sf1ovIvUCwiNQUkQnArz4stwyoKSLVRKQAcDcwO9U8X+KUJhCR0jhVUdt8jN2YfGnfvpN06zaLkSMXp4x74IGGvPdeF0sSxi98SRR9cZ6XHQd8hNPd+NMZLaSqCUAfYAGwEfhUVdeLyAgRSW7fWAAcEpENwCJggHUTYkzakpKUadOWU7v2JD78cA1jxy7lxIm4QIdl8gFfqp5qq+oQYMiFrlxV5wJzU4173uO9Av3clzEmHatX76VnzzksXercF9G+fQ0mTepAkSL2qDnjf74kitdE5BJgJvCJqq7zc0zGGFd8fCLPPvs9b7yxlMREpXz5wowb15477qhLqnZDY/wmw6onVb0R58l2B4BpIrJWRIb6PTJjDCEhQaxatZekJKVv3yZs3NibO++sZ0nCZCufbrhT1b04Dy9aBAwEngdG+jMwY/KrHTuOkZiYRLVqJRARpk7tyLFjcURGXhro0Ew+lWGJQkTqiMhwEVkLJF/xVNHvkV0I6+fJ5AHx8YmMGfMrdepM4tFHv8ZpwoOaNUtZkjAB5UuJ4l3gE6Cdqu72czyZY/08mVzut9920rPnHNas2QdAyZIRxMTEU6hQgQBHZowPiUJVr82OQLKE9fNkcpkjR04zePB3vPnmSgCqVSvOpEkduPnmmgGOzJiz0k0UIvKpqt7lVjl53g2dc59wZ0wuEheXQKNG09ix4xihoUEMGHAdQ4Y0p2DB0ECHZsw5vJUonnL/dsqOQIzJb8LCQujRozHff/83U6Z0pG7dMoEOyZg0pduYrap73Le9VPUfzxfQK3vCMybviI1N4IUXFvHRR2tTxj333A38+OODliRMjuZLFx5t0hh3c1YHYkxetnDhX1xxxRRGjFjMM88s4PTpeMC5T8LuiTA5nbc2iidwSg6Xicgaj0lFgF/8HZgxecHevSfp128BH3/sdGhQr14Zpk7tRESEtUOY3MNbG8VHwDzgP4Dn865PqOphv0ZlTC6XmJjEtGkreO657zl2LI6IiBBeeKEFzzxzLQUK2NPmTO7iLVGoqm4Xkd6pJ4hISUsWxqQvMVGZMOEPjh2Lo0OHmkyceDPVqpUIdFjGZEpGJYpOwAqcy2M9K1IVuMyPcRmT65w4EUdiolK8eDgFCgTz1lu3sG/fSW67rY61Q5hcLd1Eoaqd3L9Z9dhT/7DuO0yAqSqzZm3iySfn0a5ddd55pwsA119vj+01eYMvfT01E5FC7vv7RWSsiOScb4B132ECaPv2o3TuPIPbb/+UXbtOsG7dAWJjEwIdljFZypfLY6cAMSLSEOgP/AX8169RZYZ132GyUXx8Ii+//DN1607im2+2ULRoGBMn3syvvz5MeLhPnTIbk2v4ckQnqKqKSBdgoqq+IyI9/B2YMTlVTEw811zzNmvX7gfg7rvrM3ZsW8qXLxLgyIzxD18SxQkReRboBtwgIkGAXQRu8q2CBUOJjLyUmJh4Jk/uSNu21QMdkjF+5Uui6ArcCzysqnvd9olX/RuWMTmHqvLBB6upXr1kSgP166+3o0CBYLtxzuQLvjwKdS/wP6CYiHQCYlX1A79HZkwOsHHjAW688X26d/+Kxx77mjNnEgEoVizckoTJN3y56uku4A/gTuAu4HcRucPfgRkTSKdPxzN06A80bDiVn376hzJlCvLss9cTGurL9R/G5C2+VD0NAa5W1f0AIlIG+A6Y6c/AfGL3UBg/mD9/K717z2XbtiMAPProlYwefRMlS0YEODJjAsOXRBGUnCRch/Dtslr/s3soTBY7efIM3brN4uDBGOrXL8vUqR1p1izn3DZkTCD4kijmi8gC4GN3uCsw138hZYLdQ2EuQmJiEklJSmhoMIULF2DcuPZERx/nmWeuITTUOvAzxpdnZg8QkduA691Rb6rqLP+GZUz2WLFiN48//g1dutRi2LAWANx77xUBjsqYnMXb8yhqAmOA6sBa4P9UdVd2BWaMPx0/HsewYT8wceIykpKU48fjGDz4eitBGJMGb20N7wLfALfj9CA7IVsi8pU1ZJtMUFU++2w9tWtPZPz4PxCBfv2uYeXKxy1JGJMOb1VPRVT1Lff9ZhFZmR0B+cwass0FOnEijq5dZzJv3lYAmjatwNSpnWjU6JIAR2ZMzuYtUYSLSGPOPociwnNYVQOXODxLE9aQbXxUuHAB4uISKVYsjNGjb+Kxx64iKMieE2FMRrwlij3AWI/hvR7DCrTyV1AZstKE8dHixf9QvnxhatYshYjw7rudCQ8PoVy5woEOzZhcw9uDi27MzkAyxUoTJh0HD8YwcOBC3nsvitatq7FwYTdEhCpVigc6NGNyHes43+QpSUnK9OlRDBiwkMOHT1OgQDA33FCZxEQlJMSqmYzJDL/eYS0i7UVks4hsFZHBXua7XURURCL9GY/J29av30/LltPp0WM2hw+fpnXraqxd+wQvvNCSkJCc0ZmAMbmR30oUIhIMTALaANHAMhGZraobUs1XBHgK+N1fsZi879ixWK655h1OnjxD2bKFGDu2LffeewUiVoow5mJlmCjE+abdB1ymqiPc51Fcoqp/ZLBoE2Crqm5z1zMD6AJsSDXfi8DLwIALDd4YVUVEKFYsnEGDmrFr13Feeqk1JUpYB37GZBVfyuOTgWuBe9zhEzglhYxUAHZ6DEe741KIyJVAJVX12iotIo+JyHIRWX7gwAEfNm3yul27jnPHHZ/y4YdrUsYNGXIDU6Z0siRhTBbzJVE0VdXeQCyAqh4BClzsht1Hqo4F+mc0r6q+qaqRqhpZpkyZi920ycUSEpIYN24ptWtP4vPPN/LCCz+SmJgEYNVMxviJL20U8W57g0LK8yiSfFhuF1DJY7iiOy5ZEaA+8KP7Bb8EmC0inVV1uQ/rN/nMsmW76NlzDitX7gHg1ltrM358e4KDraHaGH/yJVGMB2YBZUVkFHAHMNSH5ZYBNUWkGk6CuBvn2dsAqOoxoHTysIj8iNPxoCUJc45Tp84waNB3TJ68DFWoXLkYEybcTOfOtQIdmjH5gi/djP9PRFYArXG677hVVTf6sFyCiPQBFgDBwLuqul5ERgDLVXX2RcZu8omQkCC++24bQUFCv37X8sILLShU6KJrP40xPhJV9T6Dc5XTeVR1h18iykBkZKQuv2eFM9Dfe+wm9/rrr8MULx5OqVIFAafaKTw8hCuuKBfgyIzJnURkhapm6l41X6qe5uC0TwgQDlQDNgP1MrNBY7yJi0vg1Vd/ZdSoJdx33xW8/XZnAK6+ukIGSxpj/MWXqqdzHvflXtLay28RZeTonwHbtPGvH3/czhNPzGHTpoOAc4VTYmKSNVYbE2AXfGe2qq4Ukab+CMYnccedv9ZzbJ6xf/8pBgxYyAcfrAagVq1STJnSkRtvrBbgyIwx4Nud2f08BoOAK4HdfovIV9ZzbJ5w8GAMdepM4vDh04SFBTNkyA0MHNiMsDDrr9KYnMKXb2MRj/cJOG0Wn/snHJPflC5dkC5dahEdfZzJkztSo0bJQIdkjEnFa6Jwb7Qroqr/l03xmDzu1KkzjBjxEx07Xk7z5lUAmDy5I2FhwXZntTE5VLqJQkRC3HshmmVnQCbv+vrrzfTpM48dO44xZ86frFnzBEFBQni4VTMZk5N5+4b+gdMeESUis4HPgFPJE1X1Cz/HZvKInTuP8dRT85k1axMAjRtfwrRpnex51cbkEr78lAsHDuE8Izv5fgoFLFEYrxISkhg//neef34Rp07FU7hwAUaOvJHevZvYg4SMyUW8JYqy7hVP6zibIJLZLdEmQ8ePx/Gf//zMqVPx3H57Hd54oz0VKxYNdFjGmAvkLVEEA4U5N0Eks0Rh0nT0aCwRESGEhYVQsmQE06Z1IiwsmI4dLw90aMaYTPKWKPao6ohsi8TkaqrKxx+v45lnFtCnz9UMG9YCgNtuqxPgyIwxF8tborCWRuOTLVsO0avXHL7//m8AFi/ekfKIUmNM7uctUbTOtihMrhQbm8DLL//MSy/9zJkziZQsGcGrr7ahe/dGliSMyUPSTRSqejg7AzG5y969J2ne/D3+/NM5TLp3b8Srr7ahdOmCAY7MGJPV7E4nkynlyhWiUqVihIQEMWVKR1q0qBrokIwxfmKJwvgkKUl5660V3HhjNS6/vBQiwkcf3UaJEhEUKBAc6PCMMX5kdz2ZDK1evZdmzd6lZ8859Oo1h+SnIpYrV9iShDH5gJUoTLpOnjzD8OE/8sYbS0lMVC69tAg9e2bqSYrGmFzMEoVJ05dfbqJv33lERx8nKEjo27cJI0e2omjRsECHZozJZpYozHl27TrO3XfPJC4ukauuKs/UqZ2IjLw00GEZYwLEEoUBID4+kZCQIESEChWKMmpUKwoUCKZXr6vtmdXG5HN2BjD8+utOrrrqTT78cE3KuP79r6Nv36aWJIwxlijys8OHT/P441/TrNm7rF27n8mTl6dc0WSMMcms6ikfUlU+/HAN/ft/y4EDMYSGBjFwYDOGDLnBut4wxpzHEkU+s2/fSe6553MWLdoOQIsWVZgypSN16pQJbGDGmBzLEkU+U7x4OHv2nKR06YKMGdOGBx5oaKUIY4xXlijygYUL/+LKK8tTqlRBwsJC+OyzOylfvjClSlkHfsaYjFljdh62Z88J7rnnc9q2/ZBBg75LGV+/fllLEsYYn1mJIg9KTExi2rQVPPvs9xw/HkdERAi1apWyhwkZYzLFEkUes3LlHnr2/IZly3YD0LFjTSZO7EDVqsUDG5gxJteyRJGHbN9+lCZN3iIxUalQoQjjx9/Mv/5V20oRxpiL4tdEISLtgXFAMPC2qo5ONb0f8AiQABwAHlbVf/wZU15WtWpxHnqoEUWKhPHvf7ekSBHrwM8Yc/H81pgtIsHAJOBmoC5wj4jUTTXbKiBSVRsAM4FX/BVPXrR9+1FuueVjfvppe8q4N9+8hbFj21mSMMZkGX+WKJoAW1V1G4CIzAC6ABuSZ1DVRR7zLwXu92M8eUZ8fCJjx/7Gv//9E6dPJ3DwYAy//dYDwKqZjDFZzp+Xx1YAdnoMR7vj0tMDmJfWBBF5TESWi8jyLIwvV/r55x00bjyNwYO/5/TpBO6+uz5ffHFXoMMyxuRhOaIxW0TuByKBFmlNV9U3gTcBIitJvuy17siR0wwYsJB33lkFQPXqJZg8uSNt21YPcGTGmLzOn4liF1DJY7iiO+4cInITMARooapxfownV0tKUr76ajOhoUEMHnw9zz57PRERoYEOyxiTD/gzUSwDaopINZwEcTdwr+cMItIYmAa0V9X9fowlV9q06SDVqhUnLCyEUqUK8r//3UblysWoXbt0oEMzxuQjfmujUNUEoA+wANgIfKqq60VkhIh0dmd7FSgMfCYiUSIy21/x5CYxMfEMGfI9DRpM4ZVXfkkZ37ZtdUsSxphs59c2ClWdC8xNNe55j/c3+XP7udH8+Vvp1WsOf/99FICDB2MCG5AxJt/LEY3ZBnbvPsHTT8/ns8+cq4evuKIsU6d24rrrKmWwpDHG+Jclihxgy5ZDREa+yYkTZyhYMJThw1vw9NPXEBoaHOjQjDHGEkVOULNmSa6+ugKFCoUyYcLNVKlSPNAhGWNMCksUAXD8eBzPP7+IXr2u5vLLSyEizJ59N4UKFQh0aMYYcx5LFNlIVZk5cwNPPTWfPXtOsmnTQebPd3otsSRhjMmpLFFkk23bjtCnz1zmzdsKwDXXVOTll+2iL2NMzpc7E0W1DoGOwGdnziQyZsyvvPjiYmJjEyhePJzRo1vz6KNXERRkHfgZY3K+3JkobpsT6Ah8tnPnMUaM+Im4uETuu+8KXnutLeXKFQ50WMYY47PcmShyuCNHTlO8eDgiQvXqJRk3rj01apSkdevLAh2aMcZcMH92M57vJCUp7767iho1JvDhh2tSxj/+eKQlCWNMrmWJIousX7+fli2n06PHbA4fPp3SaG2MMbmdVT1dpJiYeF588SfGjPmNhIQkypYtxOuvt+Oee+oHOjRjjMkSliguwpYth2jX7kO2bz+KCPTseRUvvdSaEiUiAh2aMcZkGUsUF6FKlWKEh4fQsGE5pk7txDXXVAx0SCYHiY+PJzo6mtjY2ECHYvKR8PBwKlasSGho1j3YzBLFBUhISGLq1OXcc099SpUqSFhYCPPn30eFCkUJCbHmHnOu6OhoihQpQtWqVRGxe2aM/6kqhw4dIjo6mmrVqmXZeu3s5qM//thFkyZv0bfvPAYN+i5lfJUqxS1JmDTFxsZSqlQpSxIm24gIpUqVyvJSrJUoMnDsWCxDhvzA5MnLUIXKlYvRpUutQIdlcglLEia7+eOYs0SRDlXlk0/W88wzC9i79yQhIUH063cNzz/fwjrwM8bkK1Znko7Vq/dxzz2fs3fvSa67rhIrVz7Gyy+3sSRhcpXg4GAaNWpE/fr1ueWWWzh69GjKtPXr19OqVStq1apFzZo1efHFF1HVlOnz5s0jMjKSunXr0rhxY/r37x+AT+DdqlWr6NGjR6DDSFdcXBxdu3alRo0aNG3alO3bt6c537hx46hfvz716tXjjTfeOGfahAkTqF27NvXq1WPgwIEArF27lu7du/s3eE+qmqteV1VE/SUhIfGc4Weema9vvbVCExOT/LZNk3dt2LAh0CFooUKFUt4/8MADOnLkSFVVjYmJ0csuu0wXLFigqqqnTp3S9u3b68SJE1VVde3atXrZZZfpxo0bVVU1ISFBJ0+enKWxxcfHX/Q67rjjDo2KisrWbV6ISZMm6eOPP66qqh9//LHedddd582zdu1arVevnp46dUrj4+O1devW+ueff6qq6g8//KCtW7fW2NhYVVXdt29fynKtW7fWf/75J83tpnXsAcs1k+ddq3pyLVr0N716zWXatE40b14FgLFj2wU4KpNnvOantor+mvE8rmuvvZY1a5yuZT766COaNWtG27ZtAShYsCATJ06kZcuW9O7dm1deeYUhQ4ZQu3ZtwCmZPPHEE+et8+TJk/Tt25fly5cjIrzwwgvcfvvtFC5cmJMnTwIwc+ZMvvnmG6ZPn0737t0JDw9n1apVNGvWjC+++IKoqCiKFy8OQM2aNfn5558JCgqiZ8+e7NixA4A33niDZs2anbPtEydOsGbNGho2bAjAH3/8wVNPPUVsbCwRERG899571KpVi+nTp/PFF19w8uRJEhMTmTt3Ln379mXdunXEx8czfPhwunTpwvbt2+nWrRunTp0CYOLEiVx33XU+79+0fPXVVwwfPhyAO+64gz59+qCq57QjbNy4kaZNm1KwYEEAWrRowRdffMHAgQOZMmUKgwcPJiwsDICyZcumLHfLLbcwY8aMlFKGP+X7qqf9+0/x4INf0qrVB2zadJCxY38LdEjGZLnExES+//57OnfuDDjVTlddddU581SvXp2TJ09y/Phx1q1bd970tLz44osUK1aMtWvXsmbNGlq1apXhMtHR0fz666+MHTuWLl26MGvWLAB+//13qlSpQrly5Xjqqad45plnWLZsGZ9//jmPPPLIeetZvnw59euf7QGhdu3aLFmyhFWrVjFixAiee+65lGkrV65k5syZ/PTTT4waNYpWrVrxxx9/sGjRIgYMGMCpU6coW7YsCxcuZOXKlXzyySc8+eSTacZ/ww030KhRo/Ne33333Xnz7tq1i0qVKgEQEhJCsWLFOHTo0Dnz1K9fnyVLlnDo0CFiYmKYO3cuO3fuBGDLli0sWbKEpk2b0qJFC5YtW5ayXGRkJEuWLMlwf2eFfFuiSEpS3nlnJYMGfceRI7GEhQUzdGhzBgy4uF8QxqTpAn75Z6XTp0/TqFEjdu3aRZ06dWjTpk2Wrv+7775jxowZKcMlSpTIcJk777yT4OBgALp27cqIESN46KGHmDFjBl27dk1Z74YNG1KWOX78OCdPnqRw4bNd9O/Zs4cyZcqkDB87dowHH3yQP//8ExEhPj4+ZVqbNm0oWbIkAN9++y2zZ89mzJgxgHMZ844dO7j00kvp06cPUVFRBAcHs2XLljTjz+qTc506dRg0aBBt27alUKFCNGrUKGX/JCQkcPjwYZYuXcqyZcu466672LZtGyJC2bJl2b17d5bGkp58mSj+/vsI998/i19/dbJ227bVmTSpAzVqlAxwZMZkrYiICKKiooiJiaFdu3ZMmjSJJ598krp167J48eJz5t22bRuFCxemaNGi1KtXjxUrVqRU61woz6qV1Nf0FypUKOX9tddey9atWzlw4ABffvklQ4cOBSApKYmlS5cSHh7u9bN5rnvYsGHceOONzJo1i+3bt9OyZcs0t6mqfP7559Sqde5l7sOHD6dcuXKsXr2apKSkdLd9ww03cOLEifPGjxkzhptuOveplRUqVGDnzp1UrFiRhIQEjh07RqlSpc5btkePHimN8s899xwVKzq9PFSsWJHbbrsNEaFJkyYEBQVx8OBBypQpk1LFlh3yZdVT0aJhbNlyiEsuKcyMGbczf/59liRMnlawYEHGjx/Pa6+9RkJCAvfddx8///xzSnXJ6dOnefLJJ1PquwcMGMBLL72U8qs6KSmJqVOnnrfeNm3aMGnSpJThI0eOAFCuXDk2btxIUlJSStVSWkSEf/3rX/Tr1486deqknETbtm3LhAkTUuaLioo6b9k6deqwdevZXpqPHTtGhQoVAJg+fXq622zXrh0TJkxIucJr1apVKcuXL1+eoKAg/vvf/5KYmJjm8kuWLCEqKuq8V+okAdC5c2fef/99wGmradWqVZr3Oezfvx+AHTt28MUXX3DvvfcCcOutt7Jo0SLAqYY6c+YMpUuXThn2rHrzp3yTKBYs2EpcXAIApUoVZPbsu9m0qTddu9a3m6JMvtC4cWMaNGjAxx9/TEREBF999RUjR46kVq1aXHHFFVx99dX06dMHgAYNGvDGG29wzz33UKdOHerXr8+2bdvOW+fQoUM5cuQI9evXp2HDhikntdGjR9OpUyeuu+46ypcv7zWurl278uGHH6ZUOwGMHz+e5cuX06BBA+rWrZtmkqpduzbHjh1L+XU/cOBAnn32WRo3bkxCQkK62xs2bBjx8fE0aNCAevXqMWzYMAB69erF+++/T8OGDdm0adM5pZDM6tGjB4cOHaJGjRqMHTuW0aNHA7B79246dDj7SOfbb7+dunXrcssttzBp0qSUxv2HH36Ybdu2Ub9+fe6++27ef//9lPPVokWL6Nix40XH6AtJzqq5RWQl0eU7fY95585jPPnkfL78chMvvngjQ4c292N0xpy1ceNG6tSpE+gw8rTXX3+dIkWKpNnYnZfFxcXRokULfv75Z0JCzm9BSOvYE5EVqhqZme3l2RJFQkISY8f+Rp06k/jyy00ULlyAkiWt+29j8pInnngi5dLR/GTHjh2MHj06zSThD3myMXvp0mh69vyG1av3AXD77XUYN649FSoUDXBkxpisFB4eTrdu3QIdRrarWbMmNWvWzLbt5blE8fvv0Vx33TuoQtWqxZk48WY6drw80GGZfCr1zVXG+Js/mhPyXKJo0qQC7drVoHHjSxg6tDkFC2bdwzuMuRDh4eEcOnTIuho32Ubd51F4u6w4M3J9Y/affx7imWcWMHZsOy6/3Lm0LilJCQqyL6YJLHvCnQmE9J5wdzGN2bm2RBEXl8Do0T/zn//8TFxcIuHhIcyceReAJQmTI4SGhmbpU8aMCRS/XvUkIu1FZLOIbBWRwWlMDxORT9zpv4tIVV/W+/3322jQYCrDh/9EXFwiDz3UiKlTO2V5/MYYY/xYohCRYGAS0AaIBpaJyGxV3eAxWw/giKrWEJG7gZeBruev7ay/Dxfnppv+C0CdOqWZOvVsb6/GGGOynj9LFE2Araq6TVXPADOALqnm6QK8776fCbSWDFr9jsREEB4ewksvtSIqqqclCWOM8TO/NWaLyB1Ae1V9xB3uBjRV1T4e86xz54l2h/9y5zmYal2PAY+5g/WBdX4JOvcpDRzMcK78wfbFWbYvzrJ9cVYtVS2SmQVzRWO2qr4JvAkgIssz23Kf19i+OMv2xVm2L86yfXGWiCzP7LL+rHraBVTyGK7ojktzHhEJAYoBhzDGGJNj+DNRLANqikg1ESkA3A3MTjXPbOBB9/0dwA+a227sMMaYPM5vVU+qmiAifYAFQDDwrqquF5EROA/5ng28A/xXRLYCh3GSSUbe9FfMuZDti7NsX5xl++Is2xdnZXpf5Lo7s40xxmSvPNvNuDHGmKxhicIYY4xXOTZR+Kv7j9zIh33RT0Q2iMgaEfleRPLsXYgZ7QuP+W4XERWRPHtppC/7QkTuco+N9SLyUXbHmF18+I5UFpFFIrLK/Z50SGs9uZ2IvCsi+9171NKaLiIy3t1Pa0TkSp9WrKo57oXT+P0XcBlQAFgN1E01Ty9gqvv+buCTQMcdwH1xI1DQff9Eft4X7nxFgMXAUiAy0HEH8LioCawCSrjDZQMddwD3xZvAE+77usD2QMftp33RHLgSWJfO9A7APECAa4DffVlvTi1R+KX7j1wqw32hqotUNcYdXIpzz0pe5MtxAfAiTr9hebl/b1/2xaPAJFU9AqCq+7M5xuziy75QIPkRl8WA3dkYX7ZR1cU4V5CmpwvwgTqWAsVFpHxG682piaICsNNjONodl+Y8qpoAHANKZUt02cuXfeGpB84vhrwow33hFqUrqeqc7AwsAHw5Li4HLheRX0RkqYi0z7bospcv+2I4cL+IRANzgb7ZE1qOc6HnEyCXdOFhfCMi9wORQItAxxIIIhIEjAW6BziUnCIEp/qpJU4pc7GIXKGqRwMZVIDcA0xX1ddE5Fqc+7fqq2pSoAPLDXJqicK6/zjLl32BiNwEDAE6q2pcNsWW3TLaF0VwOo38UUS249TBzs6jDdq+HBfRwGxVjVfVv4EtOIkjr/FlX/QAPgVQ1d+AcJwOA/Mbn84nqeXURGHdf5yV4b4QkcbANJwkkVfroSGDfaGqx1S1tKpWVdWqOO01nVU1052h5WC+fEe+xClNICKlcaqitmVjjNnFl32xA2gNICJ1cBLFgWyNMmeYDTzgXv10DXBMVfdktFCOrHpS/3X/kev4uC9eBQoDn7nt+TtUtXPAgvYTH/dFvuDjvlgAtBWRDUAiMEBV81yp28d90R94S0SewWnY7p4Xf1iKyMc4Pw5Ku+0xLwChAKo6Fad9pgOwFYgBHvJpvXlwXxljjMlCObXqyRhjTA5hicIYY4xXliiMMcZ4ZYnCGGOMV5YojDHGeGWJwuRIIpIoIlEer6pe5j2ZBdubLiJ/u9ta6d69e6HreFtE6rrvn0s17deLjdFdT/J+WSciX4tI8Qzmb5RXe0o12ccujzU5koicVNXCWT2vl3VMB75R1Zki0hYYo6oNLmJ9Fx1TRusVkfeBLao6ysv83XF60O2T1bGY/MNKFCZXEJHC7rM2VorIWhE5r9dYESkvIos9fnHf4I5vKyK/uct+JiIZncAXAzXcZfu561onIk+74wqJyBwRWe2O7+qO/1FEIkVkNBDhxvE/d9pJ9+8MEenoEfN0EblDRIJF5FURWeY+J+BxH3bLb7gduolIE/czrhKRX0WklnuX8gigqxtLVzf2d0XkD3fetHrfNeZcge4/3V72SuuFcydxlPuahdOLQFF3WmmcO0uTS8Qn3b/9gSHu+2Ccvp9K45z4C7njBwHPp7G96cAd7vs7gd+Bq4C1QCGcO9/XA42B24G3PJYt5v79Eff5F8kxecyTHOO/gPfd9wVwevKMAB4Dhrrjw4DlQLU04jzp8fk+A9q7w0WBEPf9TcDn7vvuwESP5V8C7nffF8fp/6lQoP/f9srZrxzZhYcxwGlVbZQ8ICKhwEsi0hxIwvklXQ7Y67HMMuBdd94vVTVKRFrgPKjmF7d7kwI4v8TT8qqIDMXpA6gHTt9As1T1lBvDF8ANwHzgNRF5Gae6askFfK55wDgRCQPaA4tV9bRb3dVARO5w5yuG04Hf36mWjxCRKPfzbwQWesz/vojUxOmiIjSd7bcFOovI/7nD4UBld13GpMkShckt7gPKAFeparw4vcOGe86gqovdRNIRmC4iY4EjwEJVvceHbQxQ1ZnJAyLSOq2ZVHWLOM+96ACMFJHvVXWELx9CVWNF5EegHdAV5yE74DxxrK+qLshgFadVtZGIFMTp26g3MB7nYU2LVPVfbsP/j+ksL8DtqrrZl3iNAWujMLlHMWC/myRuBM57Lrg4zwrfp6pvAW/jPBJyKdBMRJLbHAqJyOU+bnMJcKuIFBSRQjjVRktE5FIgRlU/xOmQMa3nDse7JZu0fILTGVty6QSck/4TycuIyOXuNtOkzhMNnwT6y9lu9pO7i+7uMesJnCq4ZAuAvuIWr8TpedgYryxRmNzif0CkiKwFHgA2pTFPS2C1iKzC+bU+TlUP4Jw4PxaRNTjVTrV92aCqrsRpu/gDp83ibVVdBVwB/OFWAb0AjExj8TeBNcmN2al8i/Nwqe/UeXQnOIltA7BSRNbhdBvvtcTvxrIG56E8rwD/cT+753KLgLrJjdk4JY9QN7b17rAxXtnlscYYY7yyEoUxxhivLFEYY4zxyhKFMcYYryxRGGOM8coShTHGGK8sURhjjPHKEoUxxhiv/h8sizGhErpDJQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "plt.figure()\n", "lw = 2\n", "plt.plot(\n", " fpr[1],\n", " tpr[1],\n", " color=\"darkorange\",\n", " lw=lw,\n", " label=\"ROC curve (area = %0.2f)\" % roc_auc[1],\n", ")\n", "plt.plot([0, 1], [0, 1], color=\"navy\", lw=lw, linestyle=\"--\")\n", "plt.xlim([0.0, 1.0])\n", "plt.ylim([0.0, 1.05])\n", "plt.xlabel(\"False Positive Rate\")\n", "plt.ylabel(\"True Positive Rate\")\n", "plt.title(\"Receiver operating characteristic example\")\n", "plt.legend(loc=\"lower right\")\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 100, "id": "e74a3ed7-27d6-4db4-81e8-8c64aba808dd", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:10:51.620850Z", "iopub.status.busy": "2022-11-27T16:10:51.619870Z", "iopub.status.idle": "2022-11-27T16:10:51.642478Z", "shell.execute_reply": "2022-11-27T16:10:51.641582Z", "shell.execute_reply.started": "2022-11-27T16:10:51.620793Z" } }, "outputs": [ { "data": { "text/plain": [ "{'accuracy': 0.8018967334035827}" ] }, "execution_count": 100, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metric.compute(predictions=preds, references=predictions.label_ids)" ] }, { "cell_type": "code", "execution_count": 107, "id": "422d81ad-3bc6-4caf-b211-74747b97337f", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:25:45.327492Z", "iopub.status.busy": "2022-11-27T16:25:45.327137Z", "iopub.status.idle": "2022-11-27T16:25:45.458733Z", "shell.execute_reply": "2022-11-27T16:25:45.457726Z", "shell.execute_reply.started": "2022-11-27T16:25:45.327465Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2D0lEQVR4nO3dd5wV1fnH8c93e6E3pS9SRJSirF0iWFFsiUYgGmMviTG/aIwmsbeYmBhjYlRibAkBaxQVJBYsMRHpXQURYQEF6WVh2/P7Y2bXu8su9y67dy+793m/XvvaOzNnZp5zF+5z55yZc2RmOOecS14piQ7AOedcYnkicM65JOeJwDnnkpwnAuecS3KeCJxzLsl5InDOuSTnicDtEUkXSvpPouOoT5LOk/TvGMo9IunmhoipIUhaJumE8PVtkv6R6Jhcw/JEkEQkZUr6m6QvJG2RNFvSKYmOKxbhh1WhpK2SvpL0pKRm9XkOMxtrZifFUO5KM7uzPs9dTpJJ2hbWc6Wk+yWlxuNczpXzRJBc0oAVwLFAS+Am4FlJeYkMqhZON7NmwCFAPkH8lUhKa/Co6t/AsJ7HAiOBixMcT71qIn+jJsUTQRIxs21mdpuZLTOzMjN7FfgcGFzTPpK6SnpR0lpJ6yT9uYZyf5S0QtJmSTMkDYnYdpik6eG2ryTdH67PkvSP8LgbJU2TtE8M9VgJTAIOCo9jkn4kaTGwOFx3WnjFs1HSfyUNiFanyOYuBf4gaU0Y9zxJ5ed7UtJdEce7TNISSeslTZDUKWKbSbpS0uIwlockKVodw3ouAT4ABkUcb0/q1VPS2+G6ryWNldQqlhiqknRmeP7Nkj6TNDxcX9G8FC5XNDFJygvfh0skLQfeljRJ0tVVjj1H0nfC130lvRG+p59IOndP4nWx8USQxMIP3T7Aghq2pwKvAl8AeUBnYHwNh5tG8IHVBvgn8JykrHDbH4E/mlkLoCfwbLj+BwRXJl2BtsCVQGEMcXcFTgVmRaw+Czgc6CfpYOBx4IrwuI8CE8KmsVjrdBLwLYL3pyVwLrCumliOA34dbu8YHrfq8U4DDgUGhOVOjlbH8Nh9gSHAknB5T+ulMMZOwAEE7/dtscRQJZ7DgKeB64FWBO/Psloc4tjw/CcD44DREcfuB3QHXpOUC7xB8O+oAzAK+EtYxsWBJ4IkJSkdGAs8ZWYf11DsMIIPj+vDq4kdZlZtB7GZ/cPM1plZiZn9HsgE9g83FwO9JLUzs61m9mHE+rZALzMrNbMZZrZ5N2G/JGkj8B/gXeCeiG2/NrP1ZlYIXA48amZTw+M+BewEjqhFnYqB5kBfQGa2yMxWV1PuPOBxM5tpZjuBXwBHVmluu9fMNprZcmAKEd/wazBT0jZgEfAO8Jdw/R7Vy8yWmNkbZrbTzNYC9xN8KNfWJWFd3wivKFfu5t9OdW4LYysE/gUMktQ93HYe8GL4Hp4GLDOzJ8J/T7OAF4Dv7kHMLgaeCJKQpBTg70ARcHXE+kkKOim3SjqP4JvjF2ZWEsMxfyZpkaRN4Yd1S6BduPkSgm/WH4fNP6eF6/8OTAbGS1ol6bdhgqrJWWbWysy6m9kPww+UcisiXncHrgubTzaG8XQl+KCMqU5m9jbwZ+AhYI2kMZJaVFO0E8G38PL9thJcOXSOKPNlxOvtQDMASQsi3u8hEWUOCcuMJLjKya1LvSTtI2m8gs7nzcA/+OZvUxtdgc/2YL9yFX8jM9sCvEbwbR+Cq4Ox4evuwOFV6nkesG8dzu12wxNBkgnbp/8G7AOcbWbF5dvM7BQzaxb+jCX4j9tNUTr3wg+xnxM0e7Q2s1bAJoImCcxssZmNJrjM/w3wvKRcMys2s9vNrB9wFME3wQv2sGqRw+iuAO4Ok0b5T46ZjYu1TmHcD5rZYKAfQSK7vppiqwg+uAAImzXaAitjOP6BEe/3+1W2mZk9C/wPuKWO9bqH4P3pHzbPnU/4t6mlFQRNe9XZBuRELFf3oV11qONxwGhJRwJZBFdL5ed5t0o9m5nZVXsQs4uBJ4Lk8zBBO+3pVb5RV+cjYDVwr6RcBZ27R1dTrjlQAqwF0iTdAlR8e5Z0vqT2ZlYGbAxXl0kaJql/2L69maA5pqwulQv9FbhS0uEK5EoaIal5rHWSdGi4fzrBh9yOGmIbB1wkaZCkTIIP3almtqwe6gFwL3CZpH3rUK/mwFZgk6TOVJ/QYvE3groeLylFUuewHwNgNjBKUrqkfOCcGI43kSCJ3gE8E/77gKCvo4+k74fHSw//HgfsYdwuCk8ESSRsj72CoI36yyrNQLsws1LgdKAXsBwoIGiuqGoy8DrwKUEzyQ4qN9UMBxZI2krQcTwqTEL7As8TJIFFBO3+f69jNTGz6cBlBE07Gwg6Wy+sZZ1aEHzwbgjrtA64r5pzvQncTNCGvZrgG/OoquXqUJd5wHsEbf97Wq/bCZqbNhE0x7y4h7F8BFwE/CE81rt8czV0M0HdN4Tn+2cMx9sZxnJCZPmw2egkgvdxFUHT2m8I+p1cHMgnpnHOueTmVwTOOZfkPBE451yS80TgnHNJzhOBc84luUY3+FO7du0sLy8v0WE451yjMmPGjK/NrH112xpdIsjLy2P69OmJDsM55xoVSV/UtM2bhpxzLsl5InDOuSTnicA555KcJwLnnEtyngiccy7JxS0RSHpcwTR/82vYLkkPKpjib66kQ+IVi3POuZrF84rgSYJRJ2tyCtA7/LmcYHhk55xzDSxuzxGY2XtVpuur6kzgaQuGP/1QUitJHWuYDrDOpi1bz/ufro3HoessMz2V7x/ZnRZZu5ucyznn4iORD5R1pvKY9QXhul0SgaTLCa4a6Nat2x6dbOYXG/jTlCV7tG88lY8Cntc2lxEDOiY2GOdcUmoUTxab2RhgDEB+fv4eTaBwxbE9ueLYmmbZS5wla7Zywv3vUurzQjjnEiSRiWAlwWTY5boQwzyvTdVLs1ayaXsRn3+9nfQ0UVRSxqVD9qNzq+xEh+aca+ISmQgmAFdLGg8cDmyKV//A3qx8hri3P17D2x+vqbQtr20uPzgqL+bjbNlZwsZtxXy5eQfrt+1k685SCjZsJy1FLF6zlez0VD79agvNstJZ/NUWmmWmsb2olMy0FJ698kjaNYt9JsCyMqOwuJStO0vYtrOE7UWlrNtWRFmZsXrTDlJTYEdxGV+s205uZiqbC4O4cjLS2FlSihC/HHGAJzrn9gJxSwSSxgFDgXaSCoBbgXQAM3uEYOLqUwnmXd1OMBdq0um9T3PuP3cgPds3o3VOBm2aZVBUUsYhd76BmVFaZqzcUMjqTYUsX7+dz7/exupNO9i6s4Q1m3ewbN12NhUWRz1PWoooKTN6tMtl+fpC9t+3GVt2lNCuWSb/W7qOsR8up88+zVi2bjtlZhRsKASML9ZtJzVFLFu3jYzUFD7/ehvZ6alsKyqtVT2z0lMoKwt+t8xJZ8X6QmZ8sYFT+u/L0rXbaJ6VxsqNhZQZYFaxbmdJGUWlZdxx5oF8++AuQJD0JO3Bu+2cq06jm7M4Pz/fmvroo19v3Un+XW9GLde1TTb7tsgiMy2Vds0y2H/fFqQI9m2ZxT4tsmiRlU67Zhm0zEknMy212mP87T+fc+erC6vdliJolZNBTkYqbXMzgvM0zyA9NYUOzTNJTUmhc6ssikqNLq2Db/YdmmeSk5FGVnoKzbPSyU5PJSs9pdIH96qNhRx179sA5GSkkpoiBHRuncOWHcX0aJdLbkYamwqL2a99LmOnLgegWWYaW3eWcGhea8ZffiQlZWU11ss5V5mkGWaWX922RtFZnGxa52RUvD5rUCe6tc2lR7sc8trm0rVNDm1zM+rtG/GFR+WxamMhw/bvQHZGCu2aZdImN4PcjDRSUuLzrbtTq2w+u+dUBDGd473FaykqKWNw99ZMnPcl05ZtoOcvJwLw9MWH0bVNDuu27mRnSRkrNxYC8MW6bWSkpvLZ2q3kZgZXGzuKg6uYnSVlXHdiH77Vp9qh2Z1LOn5F4BqVSfNWc9XYmRzUuQXzV26OWj49VRSXGt3b5rC9qJSurbOZuXwjAFN+NpTOrbLJSPORVlzTt7srAk8ErlEqKzNO+eP7nHlwJzJSU+jaJoe0FLFPiyxyM9NolZ1Os6w00lN3/ZDPu/G1itcSLLpjOFnp3sTkmjZPBM5F2LazhGPvm8LXW4uAoCP92pP6UFhUyvCD9uXATi0THKFz9c8TgXPVWL5uO9+6b8ou688/ohu3nHYgm3cU0yYnI259Jc41JE8EztXgq8072LKjmH1aZNH/tn9XW+ausw7i0Lw27L9v84p1JaVlbCos5qvNO9lWVEKnVtn+TITbq3kicC5GUz5ew0VPTuN7h3fjn+Ftq+W6tslm644SCotL2VFctsu+5wwOnnO4eUQ/mmWlkdoAVxJlZcbmHcUUlZbRoXlW3M/nGi9PBM7tgRXrt/PZ2q1c+MQ0AHq0y6V5Vhrtm2XSq0MzstKD5zdufnnBLvse2KkFr10zpNK6HcWlfL11J19t3snKjYV8tWkHZWZ8/OUW0lNFemoKPzm+N5t3FFOwoZCtO0tYuaGQLzfvAOCTL7eQouDhvg3bijBge8SDfZ1aZnHnWQdxcLfWpKaIltk+mq37hicC5xrA1p0lHHTr5IrlAV1asn5bEcWlZXy1eWedj9+heSbFpWV0a5tLi6w0MtNS6LtvCzZsL6p46C7SKQfty5E92/K9w7qRVs3dUy65eCJwrgGNHvMh/1u6jvbNM8lOT6Vtswx6tW9Gq5x0Wmans0+LLLq0zmGfFpm0bZZJi6w01m0r4uh73+ayIfvRtlkGeW1zaZGdTofmmbRvnklmWkrUhwgfe38p4z5azpYdJazZUjnxXD2sFz85oXe1t9O65OCJwLkkVP7wXbknLjqUoX3as62olDWbd7A+bF4a3K213xmVBDwROJfE/jWrgJ8+M6fG7U9ffJgPt5EEdpcI/DrRuSaufNRWgO8c0plrju/NL07py6XH9ADgVy/NS1Robi/hg845lwSW3Tui2vWP/edzVqwv5NKnpvH11iKKSspYsmYrRaXB7bEzbz6RNrkZ1e7rmg6/InDO8eaiNXz61Ra27iyp1Ex0yJ1vcOMLc7nh+bmUlTWuZmQXO+8jcC7JlZZZtQ+/RQ7OV27cZUeQkSYGd2/TEKG5euSdxc65PVJaZmzdWcLA2ysPvzGwayvOP7wb3zmkS4M8Qe3qzhOBc65OtheVcMML8+jRLpcH31pcadvPh+/PiP4d6dYmx6cQ3Yt5InDO1Zv5KzexflsRFzz+UaX1PxrWk+z0VL5Yt53SMqOwuJS/nHeIJ4e9hCcC51y9MzNmLt/A6L9Opahk10H4IJgK9ZenHkBRaRk56an+4FoCeSJwzsXVH974lMN7tKF7u1z2aZ7JzOUbOffR/+1SLiMthay0FH5yQh+O6dWOtFTRs32zBEScfDwROOca3LiPlvOLF+fRu0MzFq/Zutuypw3oyL4tsvjZyfv7tKFx4onAObdX2LqzhEWrN3PD83MZ1LUVL85auUuZ9s0z6dexBempKZx3RDeG7d8hAZE2PZ4InHN7rR3FpSxdu41TH3y/2u1vXnssbXIz/AnnOvJE4JxrFHYUl1JmRk5GWqUH2lrnpDPrlpMSGFnj54POOecahaz0VHIygiHQ5tx6UsUsaxu2F7P/TZOYtXxDIsNrsvyKwDm3V3t17iqu/uesSutaZqdzyTE92FZUQnpK0JfQsWV2giJsHLxpyDnXqG3aXszAO/5d4/YWWWmcM7grIwbsy+DubTAzf5CtijonAkkdgKOBTkAhMB+YbmbVP0USR54InHPjPlpOtzY5dG+bwzG/mVJtmdMHduLio/M4uFvrBo5u77THiUDSMOBGoA0wC1gDZAF9gJ7A88DvzWxzDfsPB/4IpAKPmdm9VbZ3Bx4H2gPrgfPNrGB3lfFE4JyLNHvFRlpkpXH/G5/y6tzVZKenUlhcCkCX1tn854bjEhzh3qEuieA+4E9mtryabWnAaUCqmb1QzfZU4FPgRKAAmAaMNrOFEWWeA141s6ckHQdcZGbf311lPBE456KZunQdI8d8CMD/ndCbHx/XO+lHSd3ju4bM7PrqkkC4rcTMXqouCYQOA5aY2VIzKwLGA2dWKdMPeDt8PaWa7c45V2uH79eWvvs2B+CBNxfT85cTWb2pMMFR7b32+PZRSRdFKdIZWBGxXBCuizQH+E74+ttAc0ltqznX5ZKmS5q+du3aPQ3ZOZdEXv+/b/HCVUdWLP/iRZ+buSZ1eY7g9no4/8+AYyXNAo4FVgKlVQuZ2Rgzyzez/Pbt21fd7Jxz1RrcvQ1vX3csAO98spbFX21he1FJgqPa++x28npJc2vaBOwT5dgrga4Ry13CdRXMbBXhFYGkZsDZZrYxynGdcy5m+0WMbnriH94D4Li+HbhsyH7069iC3MxU0lKT+9na3SYCgg/7k4Gqj/MJ+G+UfacBvSX1IEgAo4DvVTqI1A5YH96G+guCO4icc65evXf9MH48biZzCjYB8PbHa3j74zWVyowY0JGHvndIIsJLuGiJ4FWgmZnNrrpB0ju729HMSiRdDUwmuH30cTNbIOkOgmcQJgBDgV9LMuA94Ee1roFzzkXRrW0OL199DAALVm3inomL+GDJukplXpu7mo9Xv8Nna7dxx5kHcsGReQmINDH8yWLnXNK7bcICnvzvskrrZt58YpMa8dSHmHDOuSjKh6WIHPV0YNdWXD2sFyf2i9Yluvfz0Uedcy6K8rGJ5t9+csW6OSs2ctnTTf+LpycC55yL0Cwzjc/uOZVFdwyvWLdxe1ECI4o/TwTOOVdFaorIzkhlYNdWAKzauCOxAcVZzIlA0pjdLTvnXFNzbJ/gAdZTH3yfw+95k7wbX+OacbNYsX57giOrX7W5Ing0yrJzzjUpVw/rVfH6q807AZgwZxVDfjuF/LveaDIJwe8acs65GM1ZsZEzH/qgYrlfxxZM/MmQBEYUu93dNRRtiIlXgBozhZmdUcfYnHOu0RjYtRXL7h0BQN6Nr7Fw9Wb+8eEXdGieydD9O5CR1ji7XaM9Wfy7BonCOecaqZtemg9Aq5x0nr3iSIpKyujaJoeW2ekJjix2MTcNScoGupnZJ/ENafe8acg5tzdYunYrz0xbwRfrtvP6gi932f7weYdwSv+OCYisevUxZ/HpBFcHGWbWQ9Ig4I5ENA15InDO7W3WbtnJMb95m977NGP+ym9m7m2bm8E71w+leVbirw7q48ni2whmHNsIEA5C16MeYnPOuUavffNMPrnrFF798ZCKPgSAdduKGP/Rit3suXeINREUm9mmKusa1+1GzjnXQJbdO4IZN50AwPJGcItprIlggaTvAamSekv6E9HnI3DOuaSVmhKMXfT3D79g0/biBEeze7Emgh8DBwI7gXHAZuD/4hSTc841eq1yMtivfS4AhcW7zMC7V4kpEZjZdjP7FXA8MMzMfmVmTXvwDeecq6OR+cFsvV+s25bgSHYvpkQg6VBJ84C5wDxJcyQNjm9ozjnXuJUPSzFyzIc89v7SBEdTs1ibhv4G/NDM8swsj2BKySfiFpVzzjUBt5zer+L1Xa8tSmAkuxdrIig1s/fLF8zsP0BJfEJyzrmmI/J20nkFVW++3DvsNhFIOkTSIcC7kh6VNFTSsZL+ArzTIBE651wj17FlFgDvLV6b4EiqF+2K4Pfhz0CgD3ArwcNlBwCD4hmYc841Fe9ePwyAV+asYm8c8Xm3g86Z2bCGCsQ555qq8JECPv5yC8vXb6d729zEBlRFtNFHK0gaQfAsQVb5OjO7Ix5BOedcU5KWmsLow7oy7qMVzFq+ca9LBLHePvoIMJLgwTIB3wW6xzEu55xrUk7qty8A81fufR3Gsd41dJSZXQBsMLPbgSMJ+gycc87FoHz+48f+8zmn/PF9ysr2nr6CWBNBYfh7u6ROQDGw9wy07Zxzeznpm9eLVm/mhhfmJi6YKmJNBK9KagXcB8wElhGMOeSccy4Gklh27whm3XwiAM/NKKC4tCzBUQViHWvoTjPbaGYvEPQN9DWzm+MbmnPONT2tczMqXt82YUECI/lGtAfKvlP1BxgBHB++3i1JwyV9ImmJpBur2d5N0hRJsyTNlXTqnlfFOecah9f/bwgAY6cuZ82WxI/fGe320dN3s82AF2vaKCkVeAg4ESgApkmaYGYLI4rdBDxrZg9L6gdMBPJiCdw55xqrvvu2qHh92N1v8ea136JXh+YJiyfaA2UX1eHYhwFLzGwpgKTxwJlAZCIwoPwdaQmsqsP5nHOu0Zh/+8kcdOtkAH7w+DQ+uPG4hMUSa2fxnugMRE7WWRCui3QbcL6kAoKrgR9XdyBJl0uaLmn62rV751gdzjlXG80y0/jPDcHgDSs3FlJYlLjJa+KZCGIxGnjSzLoApwJ/l7RLTGY2xszyzSy/ffv2DR6kc87FQ5fWORWvD7v7zYTdRRTPRLAS6Bqx3CVcF+kS4FkAM/sfwfAV7eIYk3PO7VWm/vJ4ALbsLGH6sg0JiSHWISZyJN0s6a/hcm9Jp0XZbRrQW1IPSRnAKGBClTLLCaa/RNIBBInA236cc0ljnxZZ3H/uQIC9/orgCYKJ648Ml1cCd+1uBzMrAa4GJgOLCO4OWiDpDklnhMWuAy6TNIfgAbULbW8co9U55+KoeVY6AL+e9HFCzh/r6KM9zWykpNEQTGYvRT4wXT0zm0jQCRy57paI1wuBo2sRr3PONTkDu7QEgqEnEiHWK4IiSdkEt3siqSfBFYJzzrk66tAii8N6tAES0zwUayK4DXgd6CppLPAW8PN4BeWcc8nmo8/XA9D7V5Ma/NyxjjX0b+A7wIUEbfn5ZvZO/MJyzrnkMvuWExN27ljvGnoFOAl4x8xeNbOv4xuWc84ll1Y5GRyWFzQPvTq3YQdZiLVp6HfAEGChpOclnSMpK9pOzjnnYte3YzDe0BsLv2rQ88baNPSumf0Q2A94FDgXWBPPwJxzLtncceZB5LXNiV6wntVm8vpsgtFIRwKHAE/FKyjnnHMNJ9Y+gmcJHgo7DvgzwXMF1Q4Q55xzbs8tW7edl2c3bB9BrFcEfwNGm1nihsdzzjkXF9FmKCsfIDsXOLOa2cqcc87VoyG9g3E3Jy/4ssHOGe2K4FjgbaqfqWy3M5Q555yrvaKS4MniK/4+g2X3jmiQc0aboezW8OUdZvZ55DZJPeIWlXPOJalnrjiSvBtfA2DF+u10bRP/u4hifY7ghWrWPV+fgTjnnAt8++BgMsd7G2g00t1eEUjqCxwItKzSJ9CCYO4A55xz9eyHQ3vyr1kreW3eah4sM1JTog72XCfR+gj2B04DWlG5n2ALcFmcYnLOuaTWe5/mnDmoEy/PXkUwRUsCE4GZvQy8LOnIcCpJ55xzDWDbzhIAxn20nO8fmRfXc0W7fbR8qOnvSXqw6k9cI3POuSQ2qGsrAN75JP6z90ZrGloU/p4e70Ccc8594+rjejNp/pdEnwuy7qI1Db0S/q4YV0hSCtDMzBIzp5pzzrl6FetYQ/+U1EJSLjCfYDjq6+MbmnPOuYYQ63ME/cIrgLOASUAP4PvxCso551zDiTURpEtKJ0gEE8ysmHAie+ecc41brIngUWAZweBz70nqDngfgXPONQExDUNtZg8CkbeLfiFpWHxCcs4515Bi7SxuKel+SdPDn98TXB0455xr5GJtGnqcYFiJc8OfzcAT8QrKOedcw4l1hrKeZnZ2xPLtkmbHIR7nnHOhBas2s2DVZswMxfHJslivCAolHVO+IOlooDA+ITnnnIu0M5ysJl5iTQRXAg9JWiZpGcEE9lfELSrnnHPcMLxvg5wnatOQpEFAL2AUsBIg1uElJA0H/gikAo+Z2b1Vtv8BKL/7KAfoYGatYozdOedcPYg2+ugtwLPA2cBrwMhaJIFU4CHgFKAfMFpSv8gyZvZTMxtkZoOAP+FzIDvnXIWpn68DYOHq+D62Fa1paCQwyMxGA4cCl9fi2IcBS8xsqZkVAeOBM3dTfjQwrhbHd865Jm1wt9YATPt8fVzPEy0R7DSz7QBmti6G8pE6AysilgvCdbsIn1TuAbxdw/bLy59hWLs2/mNzO+fc3uCscO7iX8d57uJofQT7SZoQvhbQM2IZMzujnuIYBTxvZqXVbTSzMcAYgPz8fB/jyDmXFLq2yQHgqJ5t43qeaImgalPO72px7JVA14jlLuG66owCflSLYzvnXFIY2KUlGWm1aYypvWgT07xbh2NPA3pL6kGQAEYB36taSFJfoDXgcyI751wCRLtr6BVJp4dDUFfdtp+kOyRdXN2+ZlYCXA1MJpjy8lkzWxDuE9mkNAoYb2be5OOccwkQrWnoMuBa4AFJ64G1QBaQB3wG/NnMXq5pZzObCEyssu6WKsu31Tpq55xz9SZa09CXwM+Bn0vKAzoSDC3xafndRM455xq3WAedw8yWEUxO45xzrgmJb1e0c865vZ4nAuecS3KeCJxzLsnF1EcQzj9wG9A93EeAmdl+8QvNOedcQ4i1s/hvwE+BGUC1w0A455xrnGJNBJvMbFJcI3HOOZcQsSaCKZLuI5gvYGf5SjObGZeonHPOAWDAfxZ/HddzxJoIDg9/50esM+C4+g3HOedcpI3biykpM7YXlZCTEfOjX7US01HNbFj0Us455+rb+Ud0456JH1MWx9HYYrp9VFJLSfeXTw4j6feSWsYvLOeccwBCcT9HrM8RPA5sAc4NfzYDT8QrKOeccw0n1gannmZ2dsTy7ZJmxyEe55xzDSzWK4JCSceUL4QPmBXGJyTnnHMNKdYrgquAp8J+AQHrgQvjFZRzzrnKdhaX0iwzPncNxXRFYGazzWwgMADob2YHm9mcuETknHOuQmpK0Fn81sdr4naO3aYXSeeb2T8kXVtlPQBmdn/cInPOOcfwg/bljlcXUhbH+0ejXWfkhr+bxy0C55xzNVL87x6NOlXlo+Hv2+MfinPOuUSI9YGy30pqISld0luS1ko6P97BOeeci79Ybx89ycw2A6cRzFvcC7g+XkE555xrOLEmgvImpBHAc2a2KU7xOOeca2Cx3pT6qqSPCR4iu0pSe2BH/MJyzjnXUGJ9juBG4Cgg38yKgW3AmfEMzDnnXMOI9hzBcWb2tqTvRKyLLPJivAJzzjn3jTcXrWHUYd3icuxoTUPHAm8Dp1ezzfBE4JxzcdWuWSYAhcUlcTtHtOcIbg1/XxS3CJxzztUoPTWFwd1bx3VeglifI7hHUquI5daS7ophv+GSPpG0RNKNNZQ5V9JCSQsk/TPmyJ1zztWLWG8fPcXMNpYvmNkG4NTd7SApFXgIOAXoB4yW1K9Kmd7AL4CjzexA4P9ijtw551y9iDURpErKLF+QlA1k7qY8wGHAEjNbamZFwHh2vdPoMuChMLFgZvEbXs8551y1Yk0EY4G3JF0i6RLgDeCpKPt0BlZELBeE6yL1AfpI+kDSh5KGV3cgSZeXz5e8du3aGEN2zjkXi5geKDOz30iaA5wQrrrTzCbX0/l7A0OBLsB7kvpHNkOF5x8DjAHIz8+P31iszjmXhGoz3c0ioMTM3pSUI6m5mW3ZTfmVQNeI5S7hukgFwNTwIbXPJX1KkBim1SIu55xzdRDrXUOXAc8Dj4arOgMvRdltGtBbUg9JGcAoYEKVMi8RXA0gqR1BU9HSWGJyzjlXP2LtI/gRcDSwGcDMFgMddreDmZUAVwOTCa4mnjWzBZLukHRGWGwysE7SQmAKcL2Zrat9NZxzzu2pWJuGdppZUfnwEpLSCJ4s3i0zmwhMrLLulojXBlwb/jjnnEuAWK8I3pX0SyBb0onAc8Ar8QvLOedcQ4k1EdwArAXmAVcQfMu/KV5BOeecazhRm4bCJ4QXmFlf4K/xD8k551xDinpFYGalwCeS4jP+qXPOuYSKtbO4NbBA0kcEk9IAYGZn1LyLc865+mBm/GfJ15hZ1Tlh6kWsieDmej+zc865mJTform5sISWOen1fvxoM5RlAVcCvQg6iv8WPh/gnHOugZw+oBOzlm+M2/Gj9RE8BeQTJIFTgN/HLRLnnHMJEa1pqJ+Z9QeQ9Dfgo/iH5JxzriFFuyIoLn/hTULOOdc0RbsiGChpc/haBE8Wbw5fm5m1iGt0zjnn4i7a5PWpDRWIc865xIh1iAnnnHNNlCcC55xLcp4InHMuyXkicM65JOeJwDnnkpwnAuecS3KeCJxzLsl5InDOuSTnicA555KcJwLnnEtyngiccy7JeSJwzrkk54nAOeeSnCcC55xLcp4InHMuyXkicM65vZyFv+eu3BiX43sicM65vdzALi0B+GLd9rgcP9pUlXUiaTjwRyAVeMzM7q2y/ULgPmBluOrPZvZYbc9TXFxMQUEBO3bsqGPEzu0qKyuLLl26kJ6enuhQXJLq1jYnrsePWyKQlAo8BJwIFADTJE0ws4VVij5jZlfX5VwFBQU0b96cvLw8JNXlUM5VYmasW7eOgoICevTokehwnIuLeDYNHQYsMbOlZlYEjAfOjMeJduzYQdu2bT0JuHonibZt2/rVpmvS4pkIOgMrIpYLwnVVnS1prqTnJXWt7kCSLpc0XdL0tWvXVnsyTwIuXvzflmvqEt1Z/AqQZ2YDgDeAp6orZGZjzCzfzPLbt2/foAE651xTF89EsBKI/IbfhW86hQEws3VmtjNcfAwYHMd44io1NZVBgwZV/Nx7b9Av/uqrr3LwwQczcOBA+vXrx6OPPgrAbbfdhiSWLFlScYwHHngASUyfPh2ArVu3csUVV9CzZ08GDx7M0KFDmTp1ap1jfeCBB9i+PfrdB0OHDq2IpapzzjmHpUuXVizPnj0bSbz++uuVypW/LwcddBDf/e53Yzrv7pgZ11xzDb169WLAgAHMnDmz2nLPPPMMAwYM4MADD+SGG26oWP/kk0/Svn37ir/TY48F9yasXbuW4cOH1yk25xqreCaCaUBvST0kZQCjgAmRBSR1jFg8A1gUx3jiKjs7m9mzZ1f83HjjjRQXF3P55ZfzyiuvMGfOHGbNmsXQoUMr9unfvz/jx4+vWH7uuec48MADK5YvvfRS2rRpw+LFi5kxYwZPPPEEX3/9daXzmhllZWW1ijXWRFCTBQsWUFpayn777Vexbty4cRxzzDGMGzeuUtny92X+/PlkZGTwyCOP7PF5ASZNmsTixYtZvHgxY8aM4aqrrtqlzLp167j++ut56623WLBgAV9++SVvvfVWxfaRI0dW/J0uvfRSANq3b0/Hjh354IMP6hSfc41R3O4aMrMSSVcDkwluH33czBZIugOYbmYTgGsknQGUAOuBC+t63ttfWcDCVZvrephK+nVqwa2nHxi9YBVbtmyhpKSEtm3bApCZmcn+++9fsf2ss87i5Zdf5qabbuKzzz6jZcuWFbcofvbZZ0ydOpWxY8eSkhLk6x49etCjRw+WLVvGySefzOGHH86MGTM499xz2bBhAw888AAAf/3rX1m4cCF33XUX5557LgUFBZSWlnLzzTfz1VdfsWrVKoYNG0a7du2YMmUKV111FdOmTaOwsJBzzjmH22+/fbf1Gjt2LGee+U2/v5nx3HPP8cYbbzBkyBB27NhBVlbWLvsNGTKEuXPn1vp9jPTyyy9zwQUXIIkjjjiCjRs3snr1ajp2/OY7xdKlS+nduzflzYgnnHACL7zwAscff/xuj33WWWcxduxYjj766DrF6FxjE9c+AjObaGZ9zKynmd0drrslTAKY2S/M7EAzG2hmw8zs43jGE0+FhYWVmoaeeeYZ2rRpwxlnnEH37t0ZPXo0Y8eOrfTtvUWLFnTt2pX58+czfvx4Ro4cWbFtwYIFDBo0iNTU1GrPt3jxYn74wx+yYMECrrvuOl555RWKi4sBeOKJJ7j44ot5/fXX6dSpE3PmzGH+/PkMHz6ca665hk6dOjFlyhSmTJkCwN1338306dOZO3cu7777btQP6w8++IDBg79pxfvvf/9Ljx496NmzJ0OHDuW1117bZZ+SkhImTZpE//79d9k2cuTISu9d+c/TTz+9S9mVK1fStes3LY5dunRh5cpKLY706tWLTz75hGXLllFSUsJLL73EihXf3LfwwgsvMGDAAM4555xK6/Pz83n//fd3W3fnmqK4PlCWCHvyzb0+lDeBVPXYY48xb9483nzzTX73u9/xxhtv8OSTT1ZsHzVqFOPHj2fy5Mm89dZbPPHEEzGdr3v37hxxxBEANGvWjOOOO45XX32VAw44gOLiYvr3709mZibXXXcdN9xwA6eddhpDhgyp9ljPPvssY8aMoaSkhNWrV7Nw4UIGDBhQ47lXr15NZKf9uHHjGDVqVEV9nn76ac4++2zgmwQJwRXBJZdcssvxnnnmmZjqHKvWrVvz8MMPM3LkSFJSUjjqqKP47LPPADj99NMZPXo0mZmZPProo/zgBz/g7bffBqBDhw6sWrWqXmNxrjFocolgb9S/f3/69+/P97//fXr06FEpEZx22mlcf/315Ofn06JFi4r1Bx54IHPmzKG0tLTaq4Lc3NxKy5deein33HMPffv25aKLLgKgT58+zJw5k4kTJ3LTTTdx/PHHc8stt1Ta7/PPP+d3v/sd06ZNo3Xr1lx44YVR75nPzs6uKFNaWsoLL7zAyy+/zN13313xANaWLVto3rx5jQky0siRI/nkk092WX/ttddywQUXVFrXuXPnSt/iCwoK6Nx517uSTz/9dE4//XQAxowZU/EeljfTQfCe/fznP69Y3rFjB9nZ2buN1bmmyBNBHG3dupXp06dXdBDPnj2b7t27VyqTk5PDb37zG/r06VNpfc+ePcnPz+fWW2/lzjvvRBLLli1jwYIFlTqUyx1++OGsWLGCmTNnVjTtrFq1ijZt2nD++efTqlWrijtkmjdvzpYtW2jXrh2bN28mNzeXli1b8tVXXzFp0qRKHdrVOeCAA1iyZAl5eXm89dZbDBgwgMmTJ1ds/8EPfsC//vWvXT7Ea1KbK4IzzjiDP//5z4waNYqpU6fSsmXLSv0D5dasWUOHDh3YsGEDf/nLX3j22WcBKvUnTJgwgQMOOKBin08//ZSDDjoo5licayo8EdSTyCYQgOHDh/OrX/2K3/72t1xxxRVkZ2eTm5tb6WqgXHmzSlWPPfYY1113Hb169SI7O5t27dpx33331RjDueeey+zZs2ndujUA8+bN4/rrryclJYX09HQefvhhAC6//HKGDx9e0Vdw8MEH07dvX7p27RpTR+mIESN45513OOGEExg3bhzf/va3K20/++yzefjhh2NOBLVx6qmnMnHiRHr16kVOTk6lprRBgwZVXH385Cc/Yc6cOQDccsstFYn2wQcfZMKECaSlpdGmTZtKf48pU6YwYsSIeo/Zub2dzCx6qb1Ifn6+Vb23fdGiRZW+2SWr0047jZ/+9KdR746pq8LCQoYNG8YHH3xQY2d2Y/Stb32Ll19+uSKRRvJ/Yy6RNm0v5hf/msuoQ7vxrT579lCtpBlmll/dtkQ/WezqwcaNG+nTpw/Z2dlxTwIQ9BHcfvvtu9yt05itXbuWa6+9ttok4FyitcxJ5y/nDd7jJBCNNw01Aa1ateLTTz9t0HOefPLJDXq+eGvfvj1nnXVWosNwLiGazBVBY2vico2H/9tyTV2TSARZWVmsW7fO/8O6eld+O2x1T0o711Q0iaahLl26UFBQQE1DVDtXF+UzlDnXVDWJRJCenu6zRznn3B5qEk1Dzjnn9pwnAuecS3KeCJxzLsk1uieLJa0FvtjD3dsBX0ct1bR4nZOD1zk51KXO3c2s2ifSGl0iqAtJ02t6xLqp8jonB69zcohXnb1pyDnnkpwnAuecS3LJlgjGJDqABPA6Jwevc3KIS52Tqo/AOefcrpLtisA551wVngiccy7JNclEIGm4pE8kLZF0YzXbMyU9E26fKikvAWHWqxjqfK2khZLmSnpLUvfqjtOYRKtzRLmzJZmkRn+rYSx1lnRu+LdeIOmfDR1jfYvh33Y3SVMkzQr/fZ+aiDjri6THJa2RNL+G7ZL0YPh+zJV0SJ1PamZN6gdIBT4D9gMygDlAvyplfgg8Er4eBTyT6LgboM7DgJzw9VXJUOewXHPgPeBDID/RcTfA37k3MAtoHS53SHTcDVDnMcBV4et+wLJEx13HOn8LOASYX8P2U4FJgIAjgKl1PWdTvCI4DFhiZkvNrAgYD5xZpcyZwFPh6+eB4yWpAWOsb1HrbGZTzGx7uPgh0NjHVY7l7wxwJ/AbYEdDBhcnsdT5MuAhM9sAYGZrGjjG+hZLnQ1oEb5uCaxqwPjqnZm9B6zfTZEzgact8CHQSlLHupyzKSaCzsCKiOWCcF21ZcysBNgEtG2Q6OIjljpHuoTgG0VjFrXO4SVzVzN7rSEDi6NY/s59gD6SPpD0oaThDRZdfMRS59uA8yUVABOBHzdMaAlT2//vUTWJ+Qhc7CSdD+QDxyY6lniSlALcD1yY4FAaWhpB89BQgqu+9yT1N7ONiQwqzkYDT5rZ7yUdCfxd0kFmVpbowBqLpnhFsBLoGrHcJVxXbRlJaQSXk+saJLr4iKXOSDoB+BVwhpntbKDY4iVanZsDBwHvSFpG0JY6oZF3GMfydy4AJphZsZl9DnxKkBgaq1jqfAnwLICZ/Q/IIhicramK6f97bTTFRDAN6C2ph6QMgs7gCVXKTAB+EL4+B3jbwl6YRipqnSUdDDxKkAQae7sxRKmzmW0ys3ZmlmdmeQT9ImeY2fTEhFsvYvm3/RLB1QCS2hE0FS1twBjrWyx1Xg4cDyDpAIJE0JTnrZ0AXBDePXQEsMnMVtflgE2uacjMSiRdDUwmuOPgcTNbIOkOYLqZTQD+RnD5uISgU2ZU4iKuuxjrfB/QDHgu7BdfbmZnJCzoOoqxzk1KjHWeDJwkaSFQClxvZo32ajfGOl8H/FXSTwk6ji9szF/sJI0jSObtwn6PW4F0ADN7hKAf5FRgCbAduKjO52zE75dzzrl60BSbhpxzztWCJwLnnEtyngiccy7JeSJwzrkk54nAOeeSnCcCF3eSSiXNljRf0iuSWtXz8ZeF98wjaWsNZbIlvSspVVKepMIwpoWSHgmfRK7NOfMlPRi+HirpqIhtV0q6oC51Co9zm6SfRSnzpKRzanHMvJpGtaxS7m5JK6q+n5KulnRxrOdzjYMnAtcQCs1skJkdRPDcxo8SEMPFwItmVhouf2Zmg4ABBCNWnlWbg5nZdDO7JlwcChwVse0RM3u6rgEn2CsEA75V9ThNfyyfpOOJwDW0/xEOkCWpp6TXJc2Q9L6kvuH6fST9S9Kc8OeocP1LYdkFki6v5XnPA16uujIcdPC/QK/w2/Lb+mbOhm7heb8bXs3MkfReuG6opFcVzGVxJfDT8ApjSPk3eUl9JX1Ufq7w+PPC14PDK5QZkiYryuiRki6TNC2M4QVJORGbT5A0XdKnkk4Ly6dKui/cZ66kK2rzZpnZh9U9rRqOYLtMUnVJwjVSnghcg5GUSjAUQPlTv2OAH5vZYOBnwF/C9Q8C75rZQIJx2ReE6y8Oy+YD10iKacTYcGiC/cxsWTXbcsKY5gF/Ap4yswHA2DAOgFuAk8N4Kj2NHR7zEeAP4VXP+xHbPgYyJPUIV40EnpGUHp7rnLA+jwN3R6nGi2Z2aBjDIoLxdcrlEXx7HwE8Iikr3L7JzA4FDgUui4ijvO6dJE2Mct7qTAeG7MF+bi/V5IaYcHulbEmzCa4EFgFvSGpG0JxSPuQFQGb4+zjgAoCwKWdTuP4aSd8OX3clGEwtluET2gEbq6zrGcZkwMtmNknS34HvhNv/Dvw2fP0B8KSkZ4EXYzhfpGcJEsC94e+RwP4EA+K9EdY9FYg2VsxBku4CWhEMFTI58hzhSJuLJS0F+gInAQMi+g9aErxfn5bvZGarCIYqqK014TlcE+GJwDWEQjMbFH77nkzQR/AksDFsp49K0lDgBOBIM9su6R2CwcViOn81ZT+L9dxmdqWkwwm+cc+QNDjG8wI8Q5DsXgwOZYsl9QcWmNmRtTjOk8BZZjZH0oWEA8uVh1g1ZILZq35sZpEJA9XPtKxZBO+payK8acg1mLB9+RqCQcK2A59L+i5UzMM6MCz6FsF0muVt3S0JvtFuCJNAX4JhpWM97wYgNWwy2Z3/8s0AhOcB74cx9DSzqWZ2C8Goll2r7LeFYNjr6s79GcHgbzcTJAWAT4D2CsbOR1K6pAOjxNYcWB02K51XZdt3JaVI6kkwpeMnBAn3qrA8kvpIyo1yjlj1AaLeeeQaD08ErkGZ2SxgLsFkIucBl0iaQ9APUD4F4U+AYWHH6gyCu3peB9IkLSJoZvmwlqf+N3BMlDI/Bi6SNBf4fhgHwH2S5oW3Xf6XYN7cSK8A3y7vLK7muM8A5/PNmPlFBMOf/yas+2wi7jqqwc3AVIJmqo+rbFsOfEQw69yVZrYDeAxYCMwM436UKi0Au+sjkPRbBSNf5kgqkHRbxOajgTeixOsaER991CUFBdNW/tTMvp/oWBozBfNaXOvvY9PiVwQuKZjZTGBKeOeS23PtCK5OXBPiVwTOOZfk/IrAOeeSnCcC55xLcp4InHMuyXkicM65JOeJwDnnktz/A8ISgZQYfGqLAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "from sklearn.metrics import PrecisionRecallDisplay\n", "display = PrecisionRecallDisplay.from_predictions(y_test, predictions_proba, name=\"ESMCrystal\")\n", "_ = display.ax_.set_title(\"2-class Precision-Recall curve\")" ] }, { "cell_type": "code", "execution_count": 110, "id": "5204096d-6734-4bfd-96e0-1d71528aa824", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:28:05.869028Z", "iopub.status.busy": "2022-11-27T16:28:05.868622Z", "iopub.status.idle": "2022-11-27T16:28:05.878144Z", "shell.execute_reply": "2022-11-27T16:28:05.877284Z", "shell.execute_reply.started": "2022-11-27T16:28:05.868999Z" } }, "outputs": [ { "data": { "text/plain": [ "0.6376660956611664" ] }, "execution_count": 110, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.metrics import matthews_corrcoef\n", "matthews_corrcoef(y_test, preds)" ] }, { "cell_type": "code", "execution_count": 111, "id": "69b33c20-901e-46cd-b407-c1672955c381", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:29:51.246676Z", "iopub.status.busy": "2022-11-27T16:29:51.246311Z", "iopub.status.idle": "2022-11-27T16:29:51.253043Z", "shell.execute_reply": "2022-11-27T16:29:51.252031Z", "shell.execute_reply.started": "2022-11-27T16:29:51.246649Z" } }, "outputs": [], "source": [ "cm = confusion_matrix(y_test, preds)" ] }, { "cell_type": "code", "execution_count": 112, "id": "3426a316-52d4-4a43-8fd7-266c597c865b", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:30:44.063373Z", "iopub.status.busy": "2022-11-27T16:30:44.062221Z", "iopub.status.idle": "2022-11-27T16:30:44.068110Z", "shell.execute_reply": "2022-11-27T16:30:44.066655Z", "shell.execute_reply.started": "2022-11-27T16:30:44.063373Z" } }, "outputs": [], "source": [ "[[tp, fp], [fn, tn]] = cm" ] }, { "cell_type": "code", "execution_count": 114, "id": "de9b3a8c-519b-40fc-ab63-d29d2003a7c0", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:30:55.421926Z", "iopub.status.busy": "2022-11-27T16:30:55.420834Z", "iopub.status.idle": "2022-11-27T16:30:55.427378Z", "shell.execute_reply": "2022-11-27T16:30:55.426490Z", "shell.execute_reply.started": "2022-11-27T16:30:55.421888Z" } }, "outputs": [ { "data": { "text/plain": [ "(979, 21, 355, 543)" ] }, "execution_count": 114, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tp, fp, fn, tn" ] }, { "cell_type": "code", "execution_count": 116, "id": "3c03880f-d002-4c7f-9f8e-b6ecf5d9c580", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:31:17.716101Z", "iopub.status.busy": "2022-11-27T16:31:17.715374Z", "iopub.status.idle": "2022-11-27T16:31:17.722395Z", "shell.execute_reply": "2022-11-27T16:31:17.721355Z", "shell.execute_reply.started": "2022-11-27T16:31:17.715990Z" } }, "outputs": [ { "data": { "text/plain": [ "0.979" ] }, "execution_count": 116, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ppv = tp/(tp+fp)\n", "ppv" ] }, { "cell_type": "code", "execution_count": 117, "id": "6e6ea5b1-057e-4739-8533-812ade453982", "metadata": { "execution": { "iopub.execute_input": "2022-11-27T16:31:46.992767Z", "iopub.status.busy": "2022-11-27T16:31:46.992034Z", "iopub.status.idle": "2022-11-27T16:31:46.999990Z", "shell.execute_reply": "2022-11-27T16:31:46.998679Z", "shell.execute_reply.started": "2022-11-27T16:31:46.992738Z" } }, "outputs": [ { "data": { "text/plain": [ "0.6046770601336303" ] }, "execution_count": 117, "metadata": {}, "output_type": "execute_result" } ], "source": [ "npv = tn/(tn+fn)\n", "npv" ] }, { "cell_type": "code", "execution_count": null, "id": "a8841805-47fe-46c2-9a12-0182472ac2f9", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6" } }, "nbformat": 4, "nbformat_minor": 5 }