diff --git "a/src/patent_train.ipynb" "b/src/patent_train.ipynb"
--- "a/src/patent_train.ipynb"
+++ "b/src/patent_train.ipynb"
@@ -1,1643 +1 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Harvard USPTO Dataset Training"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Importing Packages\n",
-    "\n",
-    "We first need to import the actual USPTO dataset."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.11.0)\n",
-      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (2022.8.2)\n",
-      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.23.3)\n",
-      "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.64.1)\n",
-      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.28.1)\n",
-      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (9.0.0)\n",
-      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (6.0)\n",
-      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (1.5.0)\n",
-      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.13.4)\n",
-      "Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\n",
-      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.2.0)\n",
-      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.6)\n",
-      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.8.4)\n",
-      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\n",
-      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.14)\n",
-      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (22.1.0)\n",
-      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
-      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.2)\n",
-      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
-      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.8.2)\n",
-      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.3)\n",
-      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (2.1.1)\n",
-      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.4.0)\n",
-      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.12.0)\n",
-      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.0.9)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
-      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.4)\n",
-      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (1.26.11)\n",
-      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
-      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2022.4)\n",
-      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install datasets"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datasets import load_dataset\n",
-    "import pandas as pd\n",
-    "import numpy as np"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Loading the Dataset"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We need to extract the dataset. We filter only for those in January 2016."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Found cached dataset hupd (/home/jovyan/.cache/huggingface/datasets/HUPD___hupd/sample-a4eeba92b4229e93/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142)\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e39fd26828774c8e9d159a8b5d91c4f5",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "  0%|          | 0/2 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "dataset_dict = load_dataset('HUPD/hupd',\n",
-    "    name='sample',\n",
-    "    data_files=\"https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather\", \n",
-    "    icpr_label=None,\n",
-    "    train_filing_start_date='2016-01-01',\n",
-    "    train_filing_end_date='2016-01-21',\n",
-    "    val_filing_start_date='2016-01-22',\n",
-    "    val_filing_end_date='2016-01-31',\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We print out the dataset to understand what exactly we want to look for"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "DatasetDict({\n",
-      "    train: Dataset({\n",
-      "        features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id'],\n",
-      "        num_rows: 16153\n",
-      "    })\n",
-      "    validation: Dataset({\n",
-      "        features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id'],\n",
-      "        num_rows: 9094\n",
-      "    })\n",
-      "})\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(dataset_dict)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We separate our data between training and validation"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_train = pd.DataFrame(dataset_dict['train'] )\n",
-    "df_val = pd.DataFrame(dataset_dict['validation'] )"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We can preview the training data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>patent_number</th>\n",
-       "      <th>decision</th>\n",
-       "      <th>title</th>\n",
-       "      <th>abstract</th>\n",
-       "      <th>claims</th>\n",
-       "      <th>background</th>\n",
-       "      <th>summary</th>\n",
-       "      <th>description</th>\n",
-       "      <th>cpc_label</th>\n",
-       "      <th>ipc_label</th>\n",
-       "      <th>filing_date</th>\n",
-       "      <th>patent_issue_date</th>\n",
-       "      <th>date_published</th>\n",
-       "      <th>examiner_id</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>13261748</td>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>MINI-OPTICAL NETWORK TERMINAL (ONT)</td>\n",
-       "      <td>The present invention relates to passive optic...</td>\n",
-       "      <td>1. A compact optical network terminal, compris...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;A netwo...</td>\n",
-       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;An aspect ...</td>\n",
-       "      <td>FIELD OF THE INVENTION The present invention r...</td>\n",
-       "      <td>H04Q110071</td>\n",
-       "      <td>H04Q1100</td>\n",
-       "      <td>20160120</td>\n",
-       "      <td>20170606</td>\n",
-       "      <td>20160526</td>\n",
-       "      <td>95191.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>13995128</td>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>APPARATUS FOR FORMING AND READING AN IDENTIFIC...</td>\n",
-       "      <td>Embodiments of the invention provide a method ...</td>\n",
-       "      <td>1. A method comprising: using a first reader t...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;Identif...</td>\n",
-       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;In accorda...</td>\n",
-       "      <td>CROSS-REFERENCE TO RELATED APPLICATIONS The pr...</td>\n",
-       "      <td>G06K500</td>\n",
-       "      <td>G06K500</td>\n",
-       "      <td>20160112</td>\n",
-       "      <td>20160322</td>\n",
-       "      <td>20140102</td>\n",
-       "      <td>59514.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>14241799</td>\n",
-       "      <td>PENDING</td>\n",
-       "      <td>PORTABLE DRUG DISPENSER</td>\n",
-       "      <td>A portable drug dispenser includes a chamber f...</td>\n",
-       "      <td>1. A portable drug dispenser, comprising: a ch...</td>\n",
-       "      <td></td>\n",
-       "      <td></td>\n",
-       "      <td>This application claims priority from U.S. app...</td>\n",
-       "      <td>A61J70084</td>\n",
-       "      <td>A61J700</td>\n",
-       "      <td>20160104</td>\n",
-       "      <td></td>\n",
-       "      <td>20171116</td>\n",
-       "      <td>95928.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>14348792</td>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>LIQUID-COOLED HEAT EXCHANGER</td>\n",
-       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
-       "      <td>1. A crystal growth furnace for growing a crys...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;1. Fiel...</td>\n",
-       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;The presen...</td>\n",
-       "      <td>CROSS-REFERENCE TO RELATED APPLICATIONS The pr...</td>\n",
-       "      <td>C30B11003</td>\n",
-       "      <td>C30B1100</td>\n",
-       "      <td>20160111</td>\n",
-       "      <td>20180529</td>\n",
-       "      <td>20160512</td>\n",
-       "      <td>63013.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>14360978</td>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>SOLE MEMBER OF FOOTWEAR</td>\n",
-       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
-       "      <td>1. A sole member of footwear comprising a base...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND ART &lt;EOH&gt;When the heel touche...</td>\n",
-       "      <td>&lt;SOH&gt; BRIEF DESCRIPTION OF THE DRAWINGS &lt;EOH&gt;F...</td>\n",
-       "      <td>TECHNICAL FIELD The present invention relates ...</td>\n",
-       "      <td>A43B13181</td>\n",
-       "      <td>A43B1318</td>\n",
-       "      <td>20160113</td>\n",
-       "      <td></td>\n",
-       "      <td>20160512</td>\n",
-       "      <td>94490.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16148</th>\n",
-       "      <td>15002394</td>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>ROBOT HAND CONTROLLING METHOD AND ROBOTICS DEVICE</td>\n",
-       "      <td>A robot hand controlling method executes calcu...</td>\n",
-       "      <td>1. A controlling method of a robot hand, the r...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;1. Fiel...</td>\n",
-       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;An object ...</td>\n",
-       "      <td>BACKGROUND OF THE INVENTION 1. Field of the In...</td>\n",
-       "      <td>B25J91612</td>\n",
-       "      <td>B25J916</td>\n",
-       "      <td>20160120</td>\n",
-       "      <td>20180710</td>\n",
-       "      <td>20160804</td>\n",
-       "      <td>66148.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16149</th>\n",
-       "      <td>15002396</td>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>IMMUNOGLOBULIN FUSION PROTEINS AND USES THEREOF</td>\n",
-       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
-       "      <td>1. A fusion protein comprising an Fc fragment ...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;An immu...</td>\n",
-       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;The presen...</td>\n",
-       "      <td>The present application is a U.S. Nonprovision...</td>\n",
-       "      <td>C07K14745</td>\n",
-       "      <td>C07K14745</td>\n",
-       "      <td>20160120</td>\n",
-       "      <td></td>\n",
-       "      <td>20161215</td>\n",
-       "      <td>95819.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16150</th>\n",
-       "      <td>15330955</td>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>PIPE EXTRACTION TOOL</td>\n",
-       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
-       "      <td>1. A pipe extraction tool for extracting a pip...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;1. Fiel...</td>\n",
-       "      <td>&lt;SOH&gt; BRIEF SUMMARY OF THE INVENTION &lt;EOH&gt;The ...</td>\n",
-       "      <td>CROSS-REFERENCES TO RELATED APPLICATIONS Not a...</td>\n",
-       "      <td>B25B2714</td>\n",
-       "      <td>B25B2714</td>\n",
-       "      <td>20160120</td>\n",
-       "      <td></td>\n",
-       "      <td>20170907</td>\n",
-       "      <td>95661.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16151</th>\n",
-       "      <td>15330961</td>\n",
-       "      <td>PENDING</td>\n",
-       "      <td>Molded parts with thermoplastic cellulose biop...</td>\n",
-       "      <td>A longitudinal extending body with oriented fi...</td>\n",
-       "      <td>1. A longitudinal body of a solidified organic...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF INVENTION &lt;EOH&gt;In the medi...</td>\n",
-       "      <td>&lt;SOH&gt; BRIEF SUMMARY OF THE PRESENT INVENTION &lt;...</td>\n",
-       "      <td>CROSS REFERENCES Application claims priority o...</td>\n",
-       "      <td>A61L3106</td>\n",
-       "      <td>A61L3106</td>\n",
-       "      <td>20160111</td>\n",
-       "      <td></td>\n",
-       "      <td>20171019</td>\n",
-       "      <td>96956.0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16152</th>\n",
-       "      <td>15330968</td>\n",
-       "      <td>PENDING</td>\n",
-       "      <td>Transmission method with double directivity</td>\n",
-       "      <td>A transmission method using a massive MIMO (Mu...</td>\n",
-       "      <td>1. Transmission method with double directivity...</td>\n",
-       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;</td>\n",
-       "      <td>&lt;SOH&gt; BRIEF SUMMARY OF THE INVENTION &lt;EOH&gt;The ...</td>\n",
-       "      <td>BACKGROUND OF THE INVENTION Field of the Inven...</td>\n",
-       "      <td>H04B7043</td>\n",
-       "      <td>H04B704</td>\n",
-       "      <td>20160114</td>\n",
-       "      <td></td>\n",
-       "      <td>20180329</td>\n",
-       "      <td>70883.0</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>16153 rows × 14 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "      patent_number  decision  \\\n",
-       "0          13261748  ACCEPTED   \n",
-       "1          13995128  ACCEPTED   \n",
-       "2          14241799   PENDING   \n",
-       "3          14348792  ACCEPTED   \n",
-       "4          14360978  REJECTED   \n",
-       "...             ...       ...   \n",
-       "16148      15002394  ACCEPTED   \n",
-       "16149      15002396  REJECTED   \n",
-       "16150      15330955  REJECTED   \n",
-       "16151      15330961   PENDING   \n",
-       "16152      15330968   PENDING   \n",
-       "\n",
-       "                                                   title  \\\n",
-       "0                    MINI-OPTICAL NETWORK TERMINAL (ONT)   \n",
-       "1      APPARATUS FOR FORMING AND READING AN IDENTIFIC...   \n",
-       "2                                PORTABLE DRUG DISPENSER   \n",
-       "3                           LIQUID-COOLED HEAT EXCHANGER   \n",
-       "4                                SOLE MEMBER OF FOOTWEAR   \n",
-       "...                                                  ...   \n",
-       "16148  ROBOT HAND CONTROLLING METHOD AND ROBOTICS DEVICE   \n",
-       "16149    IMMUNOGLOBULIN FUSION PROTEINS AND USES THEREOF   \n",
-       "16150                               PIPE EXTRACTION TOOL   \n",
-       "16151  Molded parts with thermoplastic cellulose biop...   \n",
-       "16152        Transmission method with double directivity   \n",
-       "\n",
-       "                                                abstract  \\\n",
-       "0      The present invention relates to passive optic...   \n",
-       "1      Embodiments of the invention provide a method ...   \n",
-       "2      A portable drug dispenser includes a chamber f...   \n",
-       "3      A crystal growth furnace comprising a crucible...   \n",
-       "4      A shoe midsole is composed of a base plate (1)...   \n",
-       "...                                                  ...   \n",
-       "16148  A robot hand controlling method executes calcu...   \n",
-       "16149  A fusion protein is disclosed. The fusion prot...   \n",
-       "16150  A pipe extraction tool that grips the inside o...   \n",
-       "16151  A longitudinal extending body with oriented fi...   \n",
-       "16152  A transmission method using a massive MIMO (Mu...   \n",
-       "\n",
-       "                                                  claims  \\\n",
-       "0      1. A compact optical network terminal, compris...   \n",
-       "1      1. A method comprising: using a first reader t...   \n",
-       "2      1. A portable drug dispenser, comprising: a ch...   \n",
-       "3      1. A crystal growth furnace for growing a crys...   \n",
-       "4      1. A sole member of footwear comprising a base...   \n",
-       "...                                                  ...   \n",
-       "16148  1. A controlling method of a robot hand, the r...   \n",
-       "16149  1. A fusion protein comprising an Fc fragment ...   \n",
-       "16150  1. A pipe extraction tool for extracting a pip...   \n",
-       "16151  1. A longitudinal body of a solidified organic...   \n",
-       "16152  1. Transmission method with double directivity...   \n",
-       "\n",
-       "                                              background  \\\n",
-       "0      <SOH> BACKGROUND OF THE INVENTION <EOH>A netwo...   \n",
-       "1      <SOH> BACKGROUND OF THE INVENTION <EOH>Identif...   \n",
-       "2                                                          \n",
-       "3      <SOH> BACKGROUND OF THE INVENTION <EOH>1. Fiel...   \n",
-       "4      <SOH> BACKGROUND ART <EOH>When the heel touche...   \n",
-       "...                                                  ...   \n",
-       "16148  <SOH> BACKGROUND OF THE INVENTION <EOH>1. Fiel...   \n",
-       "16149  <SOH> BACKGROUND OF THE INVENTION <EOH>An immu...   \n",
-       "16150  <SOH> BACKGROUND OF THE INVENTION <EOH>1. Fiel...   \n",
-       "16151  <SOH> BACKGROUND OF INVENTION <EOH>In the medi...   \n",
-       "16152            <SOH> BACKGROUND OF THE INVENTION <EOH>   \n",
-       "\n",
-       "                                                 summary  \\\n",
-       "0      <SOH> SUMMARY OF THE INVENTION <EOH>An aspect ...   \n",
-       "1      <SOH> SUMMARY OF THE INVENTION <EOH>In accorda...   \n",
-       "2                                                          \n",
-       "3      <SOH> SUMMARY OF THE INVENTION <EOH>The presen...   \n",
-       "4      <SOH> BRIEF DESCRIPTION OF THE DRAWINGS <EOH>F...   \n",
-       "...                                                  ...   \n",
-       "16148  <SOH> SUMMARY OF THE INVENTION <EOH>An object ...   \n",
-       "16149  <SOH> SUMMARY OF THE INVENTION <EOH>The presen...   \n",
-       "16150  <SOH> BRIEF SUMMARY OF THE INVENTION <EOH>The ...   \n",
-       "16151  <SOH> BRIEF SUMMARY OF THE PRESENT INVENTION <...   \n",
-       "16152  <SOH> BRIEF SUMMARY OF THE INVENTION <EOH>The ...   \n",
-       "\n",
-       "                                             description   cpc_label  \\\n",
-       "0      FIELD OF THE INVENTION The present invention r...  H04Q110071   \n",
-       "1      CROSS-REFERENCE TO RELATED APPLICATIONS The pr...     G06K500   \n",
-       "2      This application claims priority from U.S. app...   A61J70084   \n",
-       "3      CROSS-REFERENCE TO RELATED APPLICATIONS The pr...   C30B11003   \n",
-       "4      TECHNICAL FIELD The present invention relates ...   A43B13181   \n",
-       "...                                                  ...         ...   \n",
-       "16148  BACKGROUND OF THE INVENTION 1. Field of the In...   B25J91612   \n",
-       "16149  The present application is a U.S. Nonprovision...   C07K14745   \n",
-       "16150  CROSS-REFERENCES TO RELATED APPLICATIONS Not a...    B25B2714   \n",
-       "16151  CROSS REFERENCES Application claims priority o...    A61L3106   \n",
-       "16152  BACKGROUND OF THE INVENTION Field of the Inven...    H04B7043   \n",
-       "\n",
-       "       ipc_label filing_date patent_issue_date date_published examiner_id  \n",
-       "0       H04Q1100    20160120          20170606       20160526     95191.0  \n",
-       "1        G06K500    20160112          20160322       20140102     59514.0  \n",
-       "2        A61J700    20160104                         20171116     95928.0  \n",
-       "3       C30B1100    20160111          20180529       20160512     63013.0  \n",
-       "4       A43B1318    20160113                         20160512     94490.0  \n",
-       "...          ...         ...               ...            ...         ...  \n",
-       "16148    B25J916    20160120          20180710       20160804     66148.0  \n",
-       "16149  C07K14745    20160120                         20161215     95819.0  \n",
-       "16150   B25B2714    20160120                         20170907     95661.0  \n",
-       "16151   A61L3106    20160111                         20171019     96956.0  \n",
-       "16152    H04B704    20160114                         20180329     70883.0  \n",
-       "\n",
-       "[16153 rows x 14 columns]"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df_train"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Pre-Processing the Data"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We are interested in the following columns:\n",
-    "- Abstract\n",
-    "- Claims\n",
-    "- Decision <- our `y`\n",
-    "\n",
-    "Let's preprocess them both out of our training and validation data\n",
-    "\n",
-    "Also, consider that the \"Decision\" column has three types of values: \"Accepted\", \"Rejected\", and \"Pending\". To remove unecessary baggage, we will be only looking for \"Accepted\" and \"Rejected\"."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "necessary_columns = [\"abstract\",\"claims\",\"decision\"]\n",
-    "output_values = ['ACCEPTED','REJECTED'] "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "trainFeaturesToDrop = [col for col in list(df_train.columns) if col not in necessary_columns]\n",
-    "trainDF = df_train.dropna()\n",
-    "trainDF.drop(columns=trainFeaturesToDrop, inplace=True)\n",
-    "trainDF = trainDF[trainDF['decision'].isin(output_values)]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>decision</th>\n",
-       "      <th>abstract</th>\n",
-       "      <th>claims</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>The present invention relates to passive optic...</td>\n",
-       "      <td>1. A compact optical network terminal, compris...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>Embodiments of the invention provide a method ...</td>\n",
-       "      <td>1. A method comprising: using a first reader t...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
-       "      <td>1. A crystal growth furnace for growing a crys...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
-       "      <td>1. A sole member of footwear comprising a base...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A ratchet tool includes a shaft member, a hand...</td>\n",
-       "      <td>1. A ratchet tool, comprising a shaft member, ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16144</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A wavelength tunable laser device, including: ...</td>\n",
-       "      <td>1. A wavelength tunable laser device, comprisi...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16145</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>In one aspect, a method for use in preparing a...</td>\n",
-       "      <td>1. (canceled) 2. The method of claim 19, where...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16148</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A robot hand controlling method executes calcu...</td>\n",
-       "      <td>1. A controlling method of a robot hand, the r...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16149</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
-       "      <td>1. A fusion protein comprising an Fc fragment ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16150</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
-       "      <td>1. A pipe extraction tool for extracting a pip...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>8719 rows × 3 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "       decision                                           abstract  \\\n",
-       "0      ACCEPTED  The present invention relates to passive optic...   \n",
-       "1      ACCEPTED  Embodiments of the invention provide a method ...   \n",
-       "3      ACCEPTED  A crystal growth furnace comprising a crucible...   \n",
-       "4      REJECTED  A shoe midsole is composed of a base plate (1)...   \n",
-       "5      ACCEPTED  A ratchet tool includes a shaft member, a hand...   \n",
-       "...         ...                                                ...   \n",
-       "16144  ACCEPTED  A wavelength tunable laser device, including: ...   \n",
-       "16145  ACCEPTED  In one aspect, a method for use in preparing a...   \n",
-       "16148  ACCEPTED  A robot hand controlling method executes calcu...   \n",
-       "16149  REJECTED  A fusion protein is disclosed. The fusion prot...   \n",
-       "16150  REJECTED  A pipe extraction tool that grips the inside o...   \n",
-       "\n",
-       "                                                  claims  \n",
-       "0      1. A compact optical network terminal, compris...  \n",
-       "1      1. A method comprising: using a first reader t...  \n",
-       "3      1. A crystal growth furnace for growing a crys...  \n",
-       "4      1. A sole member of footwear comprising a base...  \n",
-       "5      1. A ratchet tool, comprising a shaft member, ...  \n",
-       "...                                                  ...  \n",
-       "16144  1. A wavelength tunable laser device, comprisi...  \n",
-       "16145  1. (canceled) 2. The method of claim 19, where...  \n",
-       "16148  1. A controlling method of a robot hand, the r...  \n",
-       "16149  1. A fusion protein comprising an Fc fragment ...  \n",
-       "16150  1. A pipe extraction tool for extracting a pip...  \n",
-       "\n",
-       "[8719 rows x 3 columns]"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "trainDF"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "valFeaturesToDrop = [col for col in list(df_val.columns) if col not in necessary_columns]\n",
-    "valDF = df_val.dropna()\n",
-    "valDF.drop(columns=valFeaturesToDrop, inplace=True)\n",
-    "valDF = valDF[valDF['decision'].isin(output_values)]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>decision</th>\n",
-       "      <th>abstract</th>\n",
-       "      <th>claims</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>Regimen for the treatment of rosacea include t...</td>\n",
-       "      <td>1. A treatment regimen comprising: cleansing a...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A clamp arrangement includes a pair of bracket...</td>\n",
-       "      <td>1. A clamp arrangement for supporting a fractu...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>A system and method for device action and conf...</td>\n",
-       "      <td>1-20. (canceled) 21. A mobile device comprisin...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>Systems and methods for managing datasets prod...</td>\n",
-       "      <td>1. A method, comprising: executing, by one or ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A scan driving circuit is provided. The scan d...</td>\n",
-       "      <td>1. A scan driving circuit for driving a scan l...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9085</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>The non-rigid gate device as described may be ...</td>\n",
-       "      <td>1; A non-rigid blocking apparatus referred to ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9090</th>\n",
-       "      <td>REJECTED</td>\n",
-       "      <td>The present invention provides an improved unc...</td>\n",
-       "      <td>1. A method for rendering a plastic surface am...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9091</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A method for detecting a software-race conditi...</td>\n",
-       "      <td>1. A method for detecting a software-race cond...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9092</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>The present application relates to multi-stage...</td>\n",
-       "      <td>1. A multi-stage amplitude modulation-based me...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9093</th>\n",
-       "      <td>ACCEPTED</td>\n",
-       "      <td>A paper feeder includes a housing, a driving u...</td>\n",
-       "      <td>1. A paper feeder, comprising: a housing; a dr...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>4888 rows × 3 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "      decision                                           abstract  \\\n",
-       "0     REJECTED  Regimen for the treatment of rosacea include t...   \n",
-       "1     ACCEPTED  A clamp arrangement includes a pair of bracket...   \n",
-       "2     REJECTED  A system and method for device action and conf...   \n",
-       "4     REJECTED  Systems and methods for managing datasets prod...   \n",
-       "9     ACCEPTED  A scan driving circuit is provided. The scan d...   \n",
-       "...        ...                                                ...   \n",
-       "9085  REJECTED  The non-rigid gate device as described may be ...   \n",
-       "9090  REJECTED  The present invention provides an improved unc...   \n",
-       "9091  ACCEPTED  A method for detecting a software-race conditi...   \n",
-       "9092  ACCEPTED  The present application relates to multi-stage...   \n",
-       "9093  ACCEPTED  A paper feeder includes a housing, a driving u...   \n",
-       "\n",
-       "                                                 claims  \n",
-       "0     1. A treatment regimen comprising: cleansing a...  \n",
-       "1     1. A clamp arrangement for supporting a fractu...  \n",
-       "2     1-20. (canceled) 21. A mobile device comprisin...  \n",
-       "4     1. A method, comprising: executing, by one or ...  \n",
-       "9     1. A scan driving circuit for driving a scan l...  \n",
-       "...                                                 ...  \n",
-       "9085  1; A non-rigid blocking apparatus referred to ...  \n",
-       "9090  1. A method for rendering a plastic surface am...  \n",
-       "9091  1. A method for detecting a software-race cond...  \n",
-       "9092  1. A multi-stage amplitude modulation-based me...  \n",
-       "9093  1. A paper feeder, comprising: a housing; a dr...  \n",
-       "\n",
-       "[4888 rows x 3 columns]"
-      ]
-     },
-     "execution_count": 11,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "valDF"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We need to replace the values in the `decision` column to numerical representations. We will set \"ACCEPTED\" as `1` and \"REJECTED\" as `0`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "yKey = {\"ACCEPTED\":1,\"REJECTED\":0}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "trainDF2 = trainDF.replace({\"decision\": yKey})\n",
-    "valDF2 = valDF.replace({\"decision\": yKey})"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>decision</th>\n",
-       "      <th>abstract</th>\n",
-       "      <th>claims</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>1</td>\n",
-       "      <td>The present invention relates to passive optic...</td>\n",
-       "      <td>1. A compact optical network terminal, compris...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>1</td>\n",
-       "      <td>Embodiments of the invention provide a method ...</td>\n",
-       "      <td>1. A method comprising: using a first reader t...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
-       "      <td>1. A crystal growth furnace for growing a crys...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
-       "      <td>1. A sole member of footwear comprising a base...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A ratchet tool includes a shaft member, a hand...</td>\n",
-       "      <td>1. A ratchet tool, comprising a shaft member, ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16144</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A wavelength tunable laser device, including: ...</td>\n",
-       "      <td>1. A wavelength tunable laser device, comprisi...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16145</th>\n",
-       "      <td>1</td>\n",
-       "      <td>In one aspect, a method for use in preparing a...</td>\n",
-       "      <td>1. (canceled) 2. The method of claim 19, where...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16148</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A robot hand controlling method executes calcu...</td>\n",
-       "      <td>1. A controlling method of a robot hand, the r...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16149</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
-       "      <td>1. A fusion protein comprising an Fc fragment ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16150</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
-       "      <td>1. A pipe extraction tool for extracting a pip...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>8719 rows × 3 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "       decision                                           abstract  \\\n",
-       "0             1  The present invention relates to passive optic...   \n",
-       "1             1  Embodiments of the invention provide a method ...   \n",
-       "3             1  A crystal growth furnace comprising a crucible...   \n",
-       "4             0  A shoe midsole is composed of a base plate (1)...   \n",
-       "5             1  A ratchet tool includes a shaft member, a hand...   \n",
-       "...         ...                                                ...   \n",
-       "16144         1  A wavelength tunable laser device, including: ...   \n",
-       "16145         1  In one aspect, a method for use in preparing a...   \n",
-       "16148         1  A robot hand controlling method executes calcu...   \n",
-       "16149         0  A fusion protein is disclosed. The fusion prot...   \n",
-       "16150         0  A pipe extraction tool that grips the inside o...   \n",
-       "\n",
-       "                                                  claims  \n",
-       "0      1. A compact optical network terminal, compris...  \n",
-       "1      1. A method comprising: using a first reader t...  \n",
-       "3      1. A crystal growth furnace for growing a crys...  \n",
-       "4      1. A sole member of footwear comprising a base...  \n",
-       "5      1. A ratchet tool, comprising a shaft member, ...  \n",
-       "...                                                  ...  \n",
-       "16144  1. A wavelength tunable laser device, comprisi...  \n",
-       "16145  1. (canceled) 2. The method of claim 19, where...  \n",
-       "16148  1. A controlling method of a robot hand, the r...  \n",
-       "16149  1. A fusion protein comprising an Fc fragment ...  \n",
-       "16150  1. A pipe extraction tool for extracting a pip...  \n",
-       "\n",
-       "[8719 rows x 3 columns]"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "trainDF2"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>decision</th>\n",
-       "      <th>abstract</th>\n",
-       "      <th>claims</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>0</td>\n",
-       "      <td>Regimen for the treatment of rosacea include t...</td>\n",
-       "      <td>1. A treatment regimen comprising: cleansing a...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A clamp arrangement includes a pair of bracket...</td>\n",
-       "      <td>1. A clamp arrangement for supporting a fractu...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A system and method for device action and conf...</td>\n",
-       "      <td>1-20. (canceled) 21. A mobile device comprisin...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>0</td>\n",
-       "      <td>Systems and methods for managing datasets prod...</td>\n",
-       "      <td>1. A method, comprising: executing, by one or ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A scan driving circuit is provided. The scan d...</td>\n",
-       "      <td>1. A scan driving circuit for driving a scan l...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9085</th>\n",
-       "      <td>0</td>\n",
-       "      <td>The non-rigid gate device as described may be ...</td>\n",
-       "      <td>1; A non-rigid blocking apparatus referred to ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9090</th>\n",
-       "      <td>0</td>\n",
-       "      <td>The present invention provides an improved unc...</td>\n",
-       "      <td>1. A method for rendering a plastic surface am...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9091</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A method for detecting a software-race conditi...</td>\n",
-       "      <td>1. A method for detecting a software-race cond...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9092</th>\n",
-       "      <td>1</td>\n",
-       "      <td>The present application relates to multi-stage...</td>\n",
-       "      <td>1. A multi-stage amplitude modulation-based me...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9093</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A paper feeder includes a housing, a driving u...</td>\n",
-       "      <td>1. A paper feeder, comprising: a housing; a dr...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>4888 rows × 3 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "      decision                                           abstract  \\\n",
-       "0            0  Regimen for the treatment of rosacea include t...   \n",
-       "1            1  A clamp arrangement includes a pair of bracket...   \n",
-       "2            0  A system and method for device action and conf...   \n",
-       "4            0  Systems and methods for managing datasets prod...   \n",
-       "9            1  A scan driving circuit is provided. The scan d...   \n",
-       "...        ...                                                ...   \n",
-       "9085         0  The non-rigid gate device as described may be ...   \n",
-       "9090         0  The present invention provides an improved unc...   \n",
-       "9091         1  A method for detecting a software-race conditi...   \n",
-       "9092         1  The present application relates to multi-stage...   \n",
-       "9093         1  A paper feeder includes a housing, a driving u...   \n",
-       "\n",
-       "                                                 claims  \n",
-       "0     1. A treatment regimen comprising: cleansing a...  \n",
-       "1     1. A clamp arrangement for supporting a fractu...  \n",
-       "2     1-20. (canceled) 21. A mobile device comprisin...  \n",
-       "4     1. A method, comprising: executing, by one or ...  \n",
-       "9     1. A scan driving circuit for driving a scan l...  \n",
-       "...                                                 ...  \n",
-       "9085  1; A non-rigid blocking apparatus referred to ...  \n",
-       "9090  1. A method for rendering a plastic surface am...  \n",
-       "9091  1. A method for detecting a software-race cond...  \n",
-       "9092  1. A multi-stage amplitude modulation-based me...  \n",
-       "9093  1. A paper feeder, comprising: a housing; a dr...  \n",
-       "\n",
-       "[4888 rows x 3 columns]"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "valDF2"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We combine the `abstract` and `claims` columns into a single `text` column. We also re-label the `decision` column to `label`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>label</th>\n",
-       "      <th>text</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>1</td>\n",
-       "      <td>The present invention relates to passive optic...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>1</td>\n",
-       "      <td>Embodiments of the invention provide a method ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>5</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A ratchet tool includes a shaft member, a hand...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16144</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A wavelength tunable laser device, including: ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16145</th>\n",
-       "      <td>1</td>\n",
-       "      <td>In one aspect, a method for use in preparing a...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16148</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A robot hand controlling method executes calcu...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16149</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>16150</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>8719 rows × 2 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "       label                                               text\n",
-       "0          1  The present invention relates to passive optic...\n",
-       "1          1  Embodiments of the invention provide a method ...\n",
-       "3          1  A crystal growth furnace comprising a crucible...\n",
-       "4          0  A shoe midsole is composed of a base plate (1)...\n",
-       "5          1  A ratchet tool includes a shaft member, a hand...\n",
-       "...      ...                                                ...\n",
-       "16144      1  A wavelength tunable laser device, including: ...\n",
-       "16145      1  In one aspect, a method for use in preparing a...\n",
-       "16148      1  A robot hand controlling method executes calcu...\n",
-       "16149      0  A fusion protein is disclosed. The fusion prot...\n",
-       "16150      0  A pipe extraction tool that grips the inside o...\n",
-       "\n",
-       "[8719 rows x 2 columns]"
-      ]
-     },
-     "execution_count": 16,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "trainDF3 = trainDF2.rename(columns={'decision': 'label'})\n",
-    "trainDF3['text'] = trainDF3['abstract'] + ' ' + trainDF3['claims']\n",
-    "trainDF3.drop(columns=[\"abstract\",\"claims\"],inplace=True)\n",
-    "trainDF3"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>label</th>\n",
-       "      <th>text</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>0</td>\n",
-       "      <td>Regimen for the treatment of rosacea include t...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A clamp arrangement includes a pair of bracket...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>0</td>\n",
-       "      <td>A system and method for device action and conf...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>0</td>\n",
-       "      <td>Systems and methods for managing datasets prod...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A scan driving circuit is provided. The scan d...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9085</th>\n",
-       "      <td>0</td>\n",
-       "      <td>The non-rigid gate device as described may be ...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9090</th>\n",
-       "      <td>0</td>\n",
-       "      <td>The present invention provides an improved unc...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9091</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A method for detecting a software-race conditi...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9092</th>\n",
-       "      <td>1</td>\n",
-       "      <td>The present application relates to multi-stage...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>9093</th>\n",
-       "      <td>1</td>\n",
-       "      <td>A paper feeder includes a housing, a driving u...</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>4888 rows × 2 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "      label                                               text\n",
-       "0         0  Regimen for the treatment of rosacea include t...\n",
-       "1         1  A clamp arrangement includes a pair of bracket...\n",
-       "2         0  A system and method for device action and conf...\n",
-       "4         0  Systems and methods for managing datasets prod...\n",
-       "9         1  A scan driving circuit is provided. The scan d...\n",
-       "...     ...                                                ...\n",
-       "9085      0  The non-rigid gate device as described may be ...\n",
-       "9090      0  The present invention provides an improved unc...\n",
-       "9091      1  A method for detecting a software-race conditi...\n",
-       "9092      1  The present application relates to multi-stage...\n",
-       "9093      1  A paper feeder includes a housing, a driving u...\n",
-       "\n",
-       "[4888 rows x 2 columns]"
-      ]
-     },
-     "execution_count": 17,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "valDF3 = valDF2.rename(columns={'decision': 'label'})\n",
-    "valDF3['text'] = valDF3['abstract'] + ' ' + valDF3['claims']\n",
-    "valDF3.drop(columns=[\"abstract\",\"claims\"],inplace=True)\n",
-    "valDF3"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We can grab the data for each column so that we have a list of values for training labels, training texts, validation labels, and validation texts."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "trainLabels = trainDF3[\"label\"].tolist()\n",
-    "trainText = trainDF3[\"text\"].tolist()\n",
-    "\n",
-    "valLabels = valDF3[\"label\"].tolist()\n",
-    "valText = valDF3[\"text\"].tolist()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Loading the Trainer\n",
-    "\n",
-    "Now we can start training! This time, we will just go with `distilbert-base-uncased` for simplicity."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Requirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (2.0.0)\n",
-      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.4.91)\n",
-      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.91)\n",
-      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch) (3.1.2)\n",
-      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch) (4.4.0)\n",
-      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /opt/conda/lib/python3.10/site-packages (from torch) (10.2.10.91)\n",
-      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /opt/conda/lib/python3.10/site-packages (from torch) (11.4.0.1)\n",
-      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /opt/conda/lib/python3.10/site-packages (from torch) (11.10.3.66)\n",
-      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /opt/conda/lib/python3.10/site-packages (from torch) (10.9.0.58)\n",
-      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch) (1.11.1)\n",
-      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.99)\n",
-      "Requirement already satisfied: triton==2.0.0 in /opt/conda/lib/python3.10/site-packages (from torch) (2.0.0)\n",
-      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch) (3.12.0)\n",
-      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch) (2.8.7)\n",
-      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /opt/conda/lib/python3.10/site-packages (from torch) (2.14.3)\n",
-      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.101)\n",
-      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.99)\n",
-      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /opt/conda/lib/python3.10/site-packages (from torch) (8.5.0.96)\n",
-      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (65.4.1)\n",
-      "Requirement already satisfied: wheel in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (0.37.1)\n",
-      "Requirement already satisfied: cmake in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch) (3.26.3)\n",
-      "Requirement already satisfied: lit in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch) (16.0.1)\n",
-      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch) (2.1.1)\n",
-      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch) (1.2.1)\n",
-      "Requirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.28.1)\n",
-      "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.4)\n",
-      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0)\n",
-      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.23.3)\n",
-      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.12.0)\n",
-      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.3.23)\n",
-      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.64.1)\n",
-      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.28.1)\n",
-      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.3)\n",
-      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\n",
-      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n",
-      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.0.9)\n",
-      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.11)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2022.9.24)\n",
-      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.4)\n",
-      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2.1.1)\n"
-     ]
-    }
-   ],
-   "source": [
-    "!pip install torch\n",
-    "!pip install transformers"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import torch\n",
-    "from torch.utils.data import Dataset\n",
-    "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n",
-    "from transformers import Trainer, TrainingArguments"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "model_name = \"distilbert-base-uncased\"\n",
-    "class USPTODataset(Dataset):\n",
-    "    def __init__(self, encodings, labels):\n",
-    "        self.encodings = encodings\n",
-    "        self.labels = labels\n",
-    "    def __getitem__(self, idx):\n",
-    "        item = {key: torch.tensor(val[idx]) for key, val in self.encoding.items()}\n",
-    "        item['labels'] = torch.tensor(self.labels[idx])\n",
-    "        return item\n",
-    "    def __len__(self):\n",
-    "        return len(self.labels)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "train_encodings = tokenizer(trainText, truncation=True, padding=True)\n",
-    "val_encodings = tokenizer(valText, truncation=True, padding=True)\n",
-    "\n",
-    "train_dataset = USPTODataset(train_encodings, trainLabels)\n",
-    "val_dataset = USPTODataset(val_encodings, valLabels)\n",
-    "\n",
-    "train_args = TrainingArguments(\n",
-    "    output_dir=\"./results\",\n",
-    "    num_train_epochs=2,\n",
-    "    per_device_train_batch_size=16,\n",
-    "    per_device_eval_batch_size=64,\n",
-    "    warmup_steps=500,\n",
-    "    learning_rate=5e-5,\n",
-    "    weight_decay=0.01,\n",
-    "    logging_dir=\"./logs\",\n",
-    "    logging_steps=10\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
+{"cells":[{"cell_type":"markdown","metadata":{"id":"BGKCW074dTy2"},"source":["# Harvard USPTO Dataset Training"]},{"cell_type":"markdown","metadata":{"id":"6IttmojFdTy4"},"source":["## Preprocessing USPTO Data"]},{"cell_type":"markdown","source":["### Importing the Dataset\n","\n","We first need to import the actual USPTO dataset."],"metadata":{"id":"rJ6oNXYiOtC3"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1UCFBK0OdTy5","executionInfo":{"status":"ok","timestamp":1682021338971,"user_tz":240,"elapsed":13759,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"87e553e1-6593-4b2d-e578-2a4e4e742d9b"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting datasets\n","  Downloading datasets-2.11.0-py3-none-any.whl (468 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.7/468.7 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.3)\n","Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.4.0)\n","Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.27.1)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.22.4)\n","Collecting multiprocess\n","  Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting responses<0.19\n","  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n","Collecting xxhash\n","  Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.1)\n","Collecting huggingface-hub<1.0.0,>=0.11.0\n","  Downloading huggingface_hub-0.13.4-py3-none-any.whl (200 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.1/200.1 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting dill<0.3.7,>=0.3.0\n","  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.65.0)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (6.0)\n","Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n","Collecting aiohttp\n","  Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m48.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.0.12)\n","Collecting async-timeout<5.0,>=4.0.0a3\n","  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (23.1.0)\n","Collecting frozenlist>=1.1.1\n","  Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting aiosignal>=1.1.2\n","  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n","Collecting yarl<2.0,>=1.0\n","  Downloading yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 kB\u001b[0m \u001b[31m21.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting multidict<7.0,>=4.5\n","  Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.11.0)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (3.4)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n","Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n","Installing collected packages: xxhash, multidict, frozenlist, dill, async-timeout, yarl, responses, multiprocess, huggingface-hub, aiosignal, aiohttp, datasets\n","Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.11.0 dill-0.3.6 frozenlist-1.3.3 huggingface-hub-0.13.4 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 xxhash-3.2.0 yarl-1.8.2\n"]}],"source":["!pip install datasets"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"V20AfVn8dTy6"},"outputs":[],"source":["from datasets import load_dataset\n","import pandas as pd\n","import numpy as np\n","import os\n","import json\n","import torch\n","import sys"]},{"cell_type":"markdown","metadata":{"id":"DALhUYBydTy7"},"source":["### Loading the Dataset\n","\n","We need to extract the dataset. We filter only for those in January 2016."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":474,"referenced_widgets":["9f321834b0bc4bc1ac089f9813fc0fb1","12e47c63e2fb4596b645c252e9756899","043c0260b1e14a129feab90cff5ef099","ddba4a3380794815a732bd1a453c925c","8d6813d2f8ec401d85e599eadc8dc093","4dca0683fdc5459e88f6687f3196af7f","0dba55c366e44537adf4048c24391786","6695e22c05d548139e71029524c0bc68","2a7b1a0f1e94446bb343979d86264f0c","155a8f30bf6f4bbcb491dda1ad722c3b","aa0b1c2f45104fc5b59e1704bce27e5f","06a520a78f5d4b95982a476b29734cbf","c5be3bfd001346af8d33caf0a64efc7f","de1f71b45aba42e4a807100c4a8bf81f","6eee3b7c1ce9453e8c73f568537d5ac8","8f895536b733460d9ad2987333e3733b","e05d3a3d645948a3b2745cac3d5aa737","da25e097845440948f1d3ec1096f2a99","5cfa97b42072490f8ab4d5f60e0a1792","22f78b57f5604f928475f9a81d723baf","87b97444ca194dd18f1a5bd6e4082fb1","de9855ff33fb41b58ef21fbb26f81b85","9a74f42973b34203af1100702e07c28f","08d60a39b36942c184664157f738c5f0","fc886230dc454a72a3b2954f9818e9e8","482f58d514734fd2aa1d82693c7d5c34","f52a0089a4494e719c54289ff33a1c04","c2f38dc161184b16ba930c2362bc8e3b","e0f8f41cd81647908c645f76483148e5","2326d9dbc48b40769dcde81195324f40","0887166927dd4805ae3346c3158be0d2","04f4e59cfdfe492c9644dd99f46910e9","5cfce12745f44e5a9c94b72120170915","d7bfd7f1e624447a9c1eea68116d915b","6a337183787b43ec8634dc0c5b95bd72","7dd3444c8a754737b4ed1c8f64f68601","d9c18d34c6b249b78fcbace6d79cbdec","52879a2ff4864ba084a15fe02ab8b1fa","a90a02a84860461789ad04c386cdf44e","8cc2a2fe26534066a3a778704cc5984b","754c5121d01646c4aa4284df0eb9bb4f","b957825a21c2412a9b0101869d1f44d8","9f145dfa3f6347a7ab46aaa5fb294ea1","85d3c680e7424ab99c5153f315ac51a0","18d0fb273b444854a97ddb941859e0bc","fedd62c855344bbc931485f1bed1bb20","3cd64accf6104e5f8a3ad59381650a61","d39610354db4493f9f33405267b50179","c82c7f9596a746b6a91435ddfe2801f5","453092a5a76540f2b7561c3f4f84f8b2","ba82d11406a2498ba8948b5729fb4935","50863c3f09e343a9970b73dc70abba90","269546507c224a049088863d503401b1","0cd095c293104c67996f52685a29b7e9","f4c9a13af4cd478bacedd30ee2d81b8d","f9ef1b3c4aa54426af822c5f8420f2ca","28f10c7180d24373ad411015ef51d68c","59139bbd088c4804a9c8213afc3ddf21","a6c68f2b636b4b10ab7f846a789b00aa","417d14969aa141c885ae6ddd6b554324","345e90b85bb542168925cc014a1780ff","0c2447ec9c2346feb9d9d34ad6f5ec89","2c3ac49e5ace478ea09109090519aa65","984011a4cf53494baca77e2847c1a6ec","0ad51ba66c3a48a2bf44a58c63d6f6b8","f71dc5f4ba95461eafff245393c29efb","422a18bc728147cfab41a3d434784b87","8e84ffdf79144dee93ec82a857d8abf8","a6de06c9f8c9494abd2c5146a151abf4","b707d410fc0b46be82513a3156c864ec","7cdeb5f8b21f42549d6919f0a140697a","8fe2841c26c947c28479cd459ae5edff","d40cdc8dac2b40429bd92e6330916ed5","f8c2274beb1d47e1a2e8d76d3f2babe1","a0ade24bceb54a7daedc48858588590a","895d67eeac0f46edb926a8bdd33f419b","21d8d59ca7304455aae43a23c7cbbbe8"]},"id":"d-bfQ8MsdTy8","executionInfo":{"status":"ok","timestamp":1682021405537,"user_tz":240,"elapsed":56565,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"734dcb4b-d924-479d-909c-ef907c2284c8"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading builder script:   0%|          | 0.00/14.7k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9f321834b0bc4bc1ac089f9813fc0fb1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading readme:   0%|          | 0.00/10.9k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"06a520a78f5d4b95982a476b29734cbf"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Downloading and preparing dataset hupd/sample to /root/.cache/huggingface/datasets/HUPD___hupd/sample-a4eeba92b4229e93/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142...\n","Loading dataset with config: PatentsConfig(name='sample', version=0.0.0, data_dir='sample', data_files={'train': ['https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather']}, description='Patent data from January 2016, for debugging')\n"]},{"output_type":"display_data","data":{"text/plain":["Downloading data:   0%|          | 0.00/6.67M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9a74f42973b34203af1100702e07c28f"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Using metadata file: /root/.cache/huggingface/datasets/downloads/bac34b767c2799633010fa78ecd401d2eeffd62eff58abdb4db75829f8932710\n"]},{"output_type":"display_data","data":{"text/plain":["Downloading data:   0%|          | 0.00/388M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d7bfd7f1e624447a9c1eea68116d915b"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Reading metadata file: /root/.cache/huggingface/datasets/downloads/bac34b767c2799633010fa78ecd401d2eeffd62eff58abdb4db75829f8932710\n","Filtering train dataset by filing start date: 2016-01-01\n","Filtering train dataset by filing end date: 2016-01-21\n","Filtering val dataset by filing start date: 2016-01-22\n","Filtering val dataset by filing end date: 2016-01-31\n"]},{"output_type":"display_data","data":{"text/plain":["Generating train split: 0 examples [00:00, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"18d0fb273b444854a97ddb941859e0bc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Generating validation split: 0 examples [00:00, ? examples/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f9ef1b3c4aa54426af822c5f8420f2ca"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Dataset hupd downloaded and prepared to /root/.cache/huggingface/datasets/HUPD___hupd/sample-a4eeba92b4229e93/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142. Subsequent calls will reuse this data.\n"]},{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/2 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"422a18bc728147cfab41a3d434784b87"}},"metadata":{}}],"source":["dataset_dict = load_dataset('HUPD/hupd',\n","    name='sample',\n","    data_files=\"https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather\", \n","    icpr_label=None,\n","    train_filing_start_date='2016-01-01',\n","    train_filing_end_date='2016-01-21',\n","    val_filing_start_date='2016-01-22',\n","    val_filing_end_date='2016-01-31',\n",")"]},{"cell_type":"markdown","metadata":{"id":"No0GXCF9dTy8"},"source":["We print out the dataset to understand what exactly we want to look for"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2vAINVw8dTy9","executionInfo":{"status":"ok","timestamp":1682021405538,"user_tz":240,"elapsed":11,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"49141a02-dbe3-40ea-e035-1b8a8034940e"},"outputs":[{"output_type":"stream","name":"stdout","text":["DatasetDict({\n","    train: Dataset({\n","        features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id'],\n","        num_rows: 16153\n","    })\n","    validation: Dataset({\n","        features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id'],\n","        num_rows: 9094\n","    })\n","})\n"]}],"source":["print(dataset_dict)"]},{"cell_type":"markdown","metadata":{"id":"nH-s94K4dTy-"},"source":["We separate our data between training and validation"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"y8G0zXJbdTy_"},"outputs":[],"source":["df_train = pd.DataFrame(dataset_dict['train'] )\n","df_val = pd.DataFrame(dataset_dict['validation'] )"]},{"cell_type":"markdown","metadata":{"id":"KZwFSLdydTzB"},"source":["### Pre-Processing the Data\n","\n","We are interested in the following columns:\n","- Patent Number <- purely for documentation purposes\n","- Abstract\n","- Claims\n","- Decision <- our `y`\n","\n","Let's preprocess them both out of our training and validation data\n","\n","Also, consider that the \"Decision\" column has three types of values: \"Accepted\", \"Rejected\", and \"Pending\". To remove unecessary baggage, we will be only looking for \"Accepted\" and \"Rejected\"."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"oEGYVesedTzB"},"outputs":[],"source":["necessary_columns = [\"patent_number\",\"abstract\",\"claims\",\"decision\"]\n","output_values = ['ACCEPTED','REJECTED'] "]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0mMznxq7dTzC"},"outputs":[],"source":["trainFeaturesToDrop = [col for col in list(df_train.columns) if col not in necessary_columns]\n","trainDF = df_train.dropna()\n","trainDF.drop(columns=trainFeaturesToDrop, inplace=True)\n","trainDF = trainDF[trainDF['decision'].isin(output_values)]"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":641},"id":"6jS-NnaqdTzC","executionInfo":{"status":"ok","timestamp":1682021420941,"user_tz":240,"elapsed":6,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"5e4ee638-d7bc-4fb6-ab75-cd5ecd3d436d"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["      patent_number  decision  \\\n","0          13261748  ACCEPTED   \n","1          13995128  ACCEPTED   \n","3          14348792  ACCEPTED   \n","4          14360978  REJECTED   \n","5          14369795  ACCEPTED   \n","...             ...       ...   \n","16144      15002390  ACCEPTED   \n","16145      15002391  ACCEPTED   \n","16148      15002394  ACCEPTED   \n","16149      15002396  REJECTED   \n","16150      15330955  REJECTED   \n","\n","                                                abstract  \\\n","0      The present invention relates to passive optic...   \n","1      Embodiments of the invention provide a method ...   \n","3      A crystal growth furnace comprising a crucible...   \n","4      A shoe midsole is composed of a base plate (1)...   \n","5      A ratchet tool includes a shaft member, a hand...   \n","...                                                  ...   \n","16144  A wavelength tunable laser device, including: ...   \n","16145  In one aspect, a method for use in preparing a...   \n","16148  A robot hand controlling method executes calcu...   \n","16149  A fusion protein is disclosed. The fusion prot...   \n","16150  A pipe extraction tool that grips the inside o...   \n","\n","                                                  claims  \n","0      1. A compact optical network terminal, compris...  \n","1      1. A method comprising: using a first reader t...  \n","3      1. A crystal growth furnace for growing a crys...  \n","4      1. A sole member of footwear comprising a base...  \n","5      1. A ratchet tool, comprising a shaft member, ...  \n","...                                                  ...  \n","16144  1. A wavelength tunable laser device, comprisi...  \n","16145  1. (canceled) 2. The method of claim 19, where...  \n","16148  1. A controlling method of a robot hand, the r...  \n","16149  1. A fusion protein comprising an Fc fragment ...  \n","16150  1. A pipe extraction tool for extracting a pip...  \n","\n","[8719 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-2498cab4-b5f5-4a3f-a55a-91ef75bd67d6\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>patent_number</th>\n","      <th>decision</th>\n","      <th>abstract</th>\n","      <th>claims</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>13261748</td>\n","      <td>ACCEPTED</td>\n","      <td>The present invention relates to passive optic...</td>\n","      <td>1. A compact optical network terminal, compris...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>13995128</td>\n","      <td>ACCEPTED</td>\n","      <td>Embodiments of the invention provide a method ...</td>\n","      <td>1. A method comprising: using a first reader t...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>14348792</td>\n","      <td>ACCEPTED</td>\n","      <td>A crystal growth furnace comprising a crucible...</td>\n","      <td>1. A crystal growth furnace for growing a crys...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>14360978</td>\n","      <td>REJECTED</td>\n","      <td>A shoe midsole is composed of a base plate (1)...</td>\n","      <td>1. A sole member of footwear comprising a base...</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>14369795</td>\n","      <td>ACCEPTED</td>\n","      <td>A ratchet tool includes a shaft member, a hand...</td>\n","      <td>1. A ratchet tool, comprising a shaft member, ...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>16144</th>\n","      <td>15002390</td>\n","      <td>ACCEPTED</td>\n","      <td>A wavelength tunable laser device, including: ...</td>\n","      <td>1. A wavelength tunable laser device, comprisi...</td>\n","    </tr>\n","    <tr>\n","      <th>16145</th>\n","      <td>15002391</td>\n","      <td>ACCEPTED</td>\n","      <td>In one aspect, a method for use in preparing a...</td>\n","      <td>1. (canceled) 2. The method of claim 19, where...</td>\n","    </tr>\n","    <tr>\n","      <th>16148</th>\n","      <td>15002394</td>\n","      <td>ACCEPTED</td>\n","      <td>A robot hand controlling method executes calcu...</td>\n","      <td>1. A controlling method of a robot hand, the r...</td>\n","    </tr>\n","    <tr>\n","      <th>16149</th>\n","      <td>15002396</td>\n","      <td>REJECTED</td>\n","      <td>A fusion protein is disclosed. The fusion prot...</td>\n","      <td>1. A fusion protein comprising an Fc fragment ...</td>\n","    </tr>\n","    <tr>\n","      <th>16150</th>\n","      <td>15330955</td>\n","      <td>REJECTED</td>\n","      <td>A pipe extraction tool that grips the inside o...</td>\n","      <td>1. A pipe extraction tool for extracting a pip...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8719 rows × 4 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2498cab4-b5f5-4a3f-a55a-91ef75bd67d6')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-2498cab4-b5f5-4a3f-a55a-91ef75bd67d6 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-2498cab4-b5f5-4a3f-a55a-91ef75bd67d6');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":8}],"source":["trainDF"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"gTW_PbL5dTzD"},"outputs":[],"source":["valFeaturesToDrop = [col for col in list(df_val.columns) if col not in necessary_columns]\n","valDF = df_val.dropna()\n","valDF.drop(columns=valFeaturesToDrop, inplace=True)\n","valDF = valDF[valDF['decision'].isin(output_values)]"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":659},"id":"RHVeHC5QdTzD","executionInfo":{"status":"ok","timestamp":1682021425508,"user_tz":240,"elapsed":5,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"f2341ef5-fb24-4f24-e0c7-0e6cd251233e"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["     patent_number  decision  \\\n","0         13144833  REJECTED   \n","1         14006524  ACCEPTED   \n","2         14365653  REJECTED   \n","4         14396367  REJECTED   \n","9         14416282  ACCEPTED   \n","...            ...       ...   \n","9085      15011551  REJECTED   \n","9090      15011556  REJECTED   \n","9091      15011557  ACCEPTED   \n","9092      15011558  ACCEPTED   \n","9093      15011559  ACCEPTED   \n","\n","                                               abstract  \\\n","0     Regimen for the treatment of rosacea include t...   \n","1     A clamp arrangement includes a pair of bracket...   \n","2     A system and method for device action and conf...   \n","4     Systems and methods for managing datasets prod...   \n","9     A scan driving circuit is provided. The scan d...   \n","...                                                 ...   \n","9085  The non-rigid gate device as described may be ...   \n","9090  The present invention provides an improved unc...   \n","9091  A method for detecting a software-race conditi...   \n","9092  The present application relates to multi-stage...   \n","9093  A paper feeder includes a housing, a driving u...   \n","\n","                                                 claims  \n","0     1. A treatment regimen comprising: cleansing a...  \n","1     1. A clamp arrangement for supporting a fractu...  \n","2     1-20. (canceled) 21. A mobile device comprisin...  \n","4     1. A method, comprising: executing, by one or ...  \n","9     1. A scan driving circuit for driving a scan l...  \n","...                                                 ...  \n","9085  1; A non-rigid blocking apparatus referred to ...  \n","9090  1. A method for rendering a plastic surface am...  \n","9091  1. A method for detecting a software-race cond...  \n","9092  1. A multi-stage amplitude modulation-based me...  \n","9093  1. A paper feeder, comprising: a housing; a dr...  \n","\n","[4888 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-f6f537bb-f0af-45e4-95e3-0c629d839227\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>patent_number</th>\n","      <th>decision</th>\n","      <th>abstract</th>\n","      <th>claims</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>13144833</td>\n","      <td>REJECTED</td>\n","      <td>Regimen for the treatment of rosacea include t...</td>\n","      <td>1. A treatment regimen comprising: cleansing a...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>14006524</td>\n","      <td>ACCEPTED</td>\n","      <td>A clamp arrangement includes a pair of bracket...</td>\n","      <td>1. A clamp arrangement for supporting a fractu...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>14365653</td>\n","      <td>REJECTED</td>\n","      <td>A system and method for device action and conf...</td>\n","      <td>1-20. (canceled) 21. A mobile device comprisin...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>14396367</td>\n","      <td>REJECTED</td>\n","      <td>Systems and methods for managing datasets prod...</td>\n","      <td>1. A method, comprising: executing, by one or ...</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>14416282</td>\n","      <td>ACCEPTED</td>\n","      <td>A scan driving circuit is provided. The scan d...</td>\n","      <td>1. A scan driving circuit for driving a scan l...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>9085</th>\n","      <td>15011551</td>\n","      <td>REJECTED</td>\n","      <td>The non-rigid gate device as described may be ...</td>\n","      <td>1; A non-rigid blocking apparatus referred to ...</td>\n","    </tr>\n","    <tr>\n","      <th>9090</th>\n","      <td>15011556</td>\n","      <td>REJECTED</td>\n","      <td>The present invention provides an improved unc...</td>\n","      <td>1. A method for rendering a plastic surface am...</td>\n","    </tr>\n","    <tr>\n","      <th>9091</th>\n","      <td>15011557</td>\n","      <td>ACCEPTED</td>\n","      <td>A method for detecting a software-race conditi...</td>\n","      <td>1. A method for detecting a software-race cond...</td>\n","    </tr>\n","    <tr>\n","      <th>9092</th>\n","      <td>15011558</td>\n","      <td>ACCEPTED</td>\n","      <td>The present application relates to multi-stage...</td>\n","      <td>1. A multi-stage amplitude modulation-based me...</td>\n","    </tr>\n","    <tr>\n","      <th>9093</th>\n","      <td>15011559</td>\n","      <td>ACCEPTED</td>\n","      <td>A paper feeder includes a housing, a driving u...</td>\n","      <td>1. A paper feeder, comprising: a housing; a dr...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4888 rows × 4 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f6f537bb-f0af-45e4-95e3-0c629d839227')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-f6f537bb-f0af-45e4-95e3-0c629d839227 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-f6f537bb-f0af-45e4-95e3-0c629d839227');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":10}],"source":["valDF"]},{"cell_type":"markdown","metadata":{"id":"YFOqWvPUdTzD"},"source":["We need to replace the values in the `decision` column to numerical representations. We will set \"ACCEPTED\" as `1` and \"REJECTED\" as `0`."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"CBxfqBL0dTzD"},"outputs":[],"source":["yKey = {\"ACCEPTED\":1,\"REJECTED\":0}"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"82I6gVrtdTzE"},"outputs":[],"source":["trainDF2 = trainDF.replace({\"decision\": yKey})\n","valDF2 = valDF.replace({\"decision\": yKey})"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":746},"id":"1XVwvlGKdTzE","executionInfo":{"status":"ok","timestamp":1682021428511,"user_tz":240,"elapsed":5,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"bb49c208-ee63-4a2c-86b1-6bea0449b583"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["      patent_number  decision  \\\n","0          13261748         1   \n","1          13995128         1   \n","3          14348792         1   \n","4          14360978         0   \n","5          14369795         1   \n","...             ...       ...   \n","16144      15002390         1   \n","16145      15002391         1   \n","16148      15002394         1   \n","16149      15002396         0   \n","16150      15330955         0   \n","\n","                                                abstract  \\\n","0      The present invention relates to passive optic...   \n","1      Embodiments of the invention provide a method ...   \n","3      A crystal growth furnace comprising a crucible...   \n","4      A shoe midsole is composed of a base plate (1)...   \n","5      A ratchet tool includes a shaft member, a hand...   \n","...                                                  ...   \n","16144  A wavelength tunable laser device, including: ...   \n","16145  In one aspect, a method for use in preparing a...   \n","16148  A robot hand controlling method executes calcu...   \n","16149  A fusion protein is disclosed. The fusion prot...   \n","16150  A pipe extraction tool that grips the inside o...   \n","\n","                                                  claims  \n","0      1. A compact optical network terminal, compris...  \n","1      1. A method comprising: using a first reader t...  \n","3      1. A crystal growth furnace for growing a crys...  \n","4      1. A sole member of footwear comprising a base...  \n","5      1. A ratchet tool, comprising a shaft member, ...  \n","...                                                  ...  \n","16144  1. A wavelength tunable laser device, comprisi...  \n","16145  1. (canceled) 2. The method of claim 19, where...  \n","16148  1. A controlling method of a robot hand, the r...  \n","16149  1. A fusion protein comprising an Fc fragment ...  \n","16150  1. A pipe extraction tool for extracting a pip...  \n","\n","[8719 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-fcb55bbd-c720-4b17-82e1-4059c671fb2f\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>patent_number</th>\n","      <th>decision</th>\n","      <th>abstract</th>\n","      <th>claims</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>13261748</td>\n","      <td>1</td>\n","      <td>The present invention relates to passive optic...</td>\n","      <td>1. A compact optical network terminal, compris...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>13995128</td>\n","      <td>1</td>\n","      <td>Embodiments of the invention provide a method ...</td>\n","      <td>1. A method comprising: using a first reader t...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>14348792</td>\n","      <td>1</td>\n","      <td>A crystal growth furnace comprising a crucible...</td>\n","      <td>1. A crystal growth furnace for growing a crys...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>14360978</td>\n","      <td>0</td>\n","      <td>A shoe midsole is composed of a base plate (1)...</td>\n","      <td>1. A sole member of footwear comprising a base...</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>14369795</td>\n","      <td>1</td>\n","      <td>A ratchet tool includes a shaft member, a hand...</td>\n","      <td>1. A ratchet tool, comprising a shaft member, ...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>16144</th>\n","      <td>15002390</td>\n","      <td>1</td>\n","      <td>A wavelength tunable laser device, including: ...</td>\n","      <td>1. A wavelength tunable laser device, comprisi...</td>\n","    </tr>\n","    <tr>\n","      <th>16145</th>\n","      <td>15002391</td>\n","      <td>1</td>\n","      <td>In one aspect, a method for use in preparing a...</td>\n","      <td>1. (canceled) 2. The method of claim 19, where...</td>\n","    </tr>\n","    <tr>\n","      <th>16148</th>\n","      <td>15002394</td>\n","      <td>1</td>\n","      <td>A robot hand controlling method executes calcu...</td>\n","      <td>1. A controlling method of a robot hand, the r...</td>\n","    </tr>\n","    <tr>\n","      <th>16149</th>\n","      <td>15002396</td>\n","      <td>0</td>\n","      <td>A fusion protein is disclosed. The fusion prot...</td>\n","      <td>1. A fusion protein comprising an Fc fragment ...</td>\n","    </tr>\n","    <tr>\n","      <th>16150</th>\n","      <td>15330955</td>\n","      <td>0</td>\n","      <td>A pipe extraction tool that grips the inside o...</td>\n","      <td>1. A pipe extraction tool for extracting a pip...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8719 rows × 4 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fcb55bbd-c720-4b17-82e1-4059c671fb2f')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-fcb55bbd-c720-4b17-82e1-4059c671fb2f button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-fcb55bbd-c720-4b17-82e1-4059c671fb2f');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":13}],"source":["trainDF2"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":659},"id":"D0eRJb_2dTzE","executionInfo":{"status":"ok","timestamp":1682021428713,"user_tz":240,"elapsed":4,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"74150843-f3b4-459c-ef34-d9cf08f0c0d6"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["     patent_number  decision  \\\n","0         13144833         0   \n","1         14006524         1   \n","2         14365653         0   \n","4         14396367         0   \n","9         14416282         1   \n","...            ...       ...   \n","9085      15011551         0   \n","9090      15011556         0   \n","9091      15011557         1   \n","9092      15011558         1   \n","9093      15011559         1   \n","\n","                                               abstract  \\\n","0     Regimen for the treatment of rosacea include t...   \n","1     A clamp arrangement includes a pair of bracket...   \n","2     A system and method for device action and conf...   \n","4     Systems and methods for managing datasets prod...   \n","9     A scan driving circuit is provided. The scan d...   \n","...                                                 ...   \n","9085  The non-rigid gate device as described may be ...   \n","9090  The present invention provides an improved unc...   \n","9091  A method for detecting a software-race conditi...   \n","9092  The present application relates to multi-stage...   \n","9093  A paper feeder includes a housing, a driving u...   \n","\n","                                                 claims  \n","0     1. A treatment regimen comprising: cleansing a...  \n","1     1. A clamp arrangement for supporting a fractu...  \n","2     1-20. (canceled) 21. A mobile device comprisin...  \n","4     1. A method, comprising: executing, by one or ...  \n","9     1. A scan driving circuit for driving a scan l...  \n","...                                                 ...  \n","9085  1; A non-rigid blocking apparatus referred to ...  \n","9090  1. A method for rendering a plastic surface am...  \n","9091  1. A method for detecting a software-race cond...  \n","9092  1. A multi-stage amplitude modulation-based me...  \n","9093  1. A paper feeder, comprising: a housing; a dr...  \n","\n","[4888 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-a77e1a79-09be-4e22-9b3c-8b1dbcc0b92d\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>patent_number</th>\n","      <th>decision</th>\n","      <th>abstract</th>\n","      <th>claims</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>13144833</td>\n","      <td>0</td>\n","      <td>Regimen for the treatment of rosacea include t...</td>\n","      <td>1. A treatment regimen comprising: cleansing a...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>14006524</td>\n","      <td>1</td>\n","      <td>A clamp arrangement includes a pair of bracket...</td>\n","      <td>1. A clamp arrangement for supporting a fractu...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>14365653</td>\n","      <td>0</td>\n","      <td>A system and method for device action and conf...</td>\n","      <td>1-20. (canceled) 21. A mobile device comprisin...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>14396367</td>\n","      <td>0</td>\n","      <td>Systems and methods for managing datasets prod...</td>\n","      <td>1. A method, comprising: executing, by one or ...</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>14416282</td>\n","      <td>1</td>\n","      <td>A scan driving circuit is provided. The scan d...</td>\n","      <td>1. A scan driving circuit for driving a scan l...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>9085</th>\n","      <td>15011551</td>\n","      <td>0</td>\n","      <td>The non-rigid gate device as described may be ...</td>\n","      <td>1; A non-rigid blocking apparatus referred to ...</td>\n","    </tr>\n","    <tr>\n","      <th>9090</th>\n","      <td>15011556</td>\n","      <td>0</td>\n","      <td>The present invention provides an improved unc...</td>\n","      <td>1. A method for rendering a plastic surface am...</td>\n","    </tr>\n","    <tr>\n","      <th>9091</th>\n","      <td>15011557</td>\n","      <td>1</td>\n","      <td>A method for detecting a software-race conditi...</td>\n","      <td>1. A method for detecting a software-race cond...</td>\n","    </tr>\n","    <tr>\n","      <th>9092</th>\n","      <td>15011558</td>\n","      <td>1</td>\n","      <td>The present application relates to multi-stage...</td>\n","      <td>1. A multi-stage amplitude modulation-based me...</td>\n","    </tr>\n","    <tr>\n","      <th>9093</th>\n","      <td>15011559</td>\n","      <td>1</td>\n","      <td>A paper feeder includes a housing, a driving u...</td>\n","      <td>1. A paper feeder, comprising: a housing; a dr...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4888 rows × 4 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a77e1a79-09be-4e22-9b3c-8b1dbcc0b92d')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-a77e1a79-09be-4e22-9b3c-8b1dbcc0b92d button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-a77e1a79-09be-4e22-9b3c-8b1dbcc0b92d');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":14}],"source":["valDF2"]},{"cell_type":"markdown","metadata":{"id":"gRb0ApxTdTzF"},"source":["We re-label the `decision` column to `label`."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":676},"id":"vRFoJ8hidTzF","executionInfo":{"status":"ok","timestamp":1682021435301,"user_tz":240,"elapsed":500,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"9444d1d4-5450-4767-aeb0-c1709889b7ac"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["      patent_number  label                                           abstract  \\\n","0          13261748      1  The present invention relates to passive optic...   \n","1          13995128      1  Embodiments of the invention provide a method ...   \n","3          14348792      1  A crystal growth furnace comprising a crucible...   \n","4          14360978      0  A shoe midsole is composed of a base plate (1)...   \n","5          14369795      1  A ratchet tool includes a shaft member, a hand...   \n","...             ...    ...                                                ...   \n","16144      15002390      1  A wavelength tunable laser device, including: ...   \n","16145      15002391      1  In one aspect, a method for use in preparing a...   \n","16148      15002394      1  A robot hand controlling method executes calcu...   \n","16149      15002396      0  A fusion protein is disclosed. The fusion prot...   \n","16150      15330955      0  A pipe extraction tool that grips the inside o...   \n","\n","                                                  claims  \n","0      1. A compact optical network terminal, compris...  \n","1      1. A method comprising: using a first reader t...  \n","3      1. A crystal growth furnace for growing a crys...  \n","4      1. A sole member of footwear comprising a base...  \n","5      1. A ratchet tool, comprising a shaft member, ...  \n","...                                                  ...  \n","16144  1. A wavelength tunable laser device, comprisi...  \n","16145  1. (canceled) 2. The method of claim 19, where...  \n","16148  1. A controlling method of a robot hand, the r...  \n","16149  1. A fusion protein comprising an Fc fragment ...  \n","16150  1. A pipe extraction tool for extracting a pip...  \n","\n","[8719 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-98f84a20-c0eb-4438-9beb-b98a63c9968e\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>patent_number</th>\n","      <th>label</th>\n","      <th>abstract</th>\n","      <th>claims</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>13261748</td>\n","      <td>1</td>\n","      <td>The present invention relates to passive optic...</td>\n","      <td>1. A compact optical network terminal, compris...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>13995128</td>\n","      <td>1</td>\n","      <td>Embodiments of the invention provide a method ...</td>\n","      <td>1. A method comprising: using a first reader t...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>14348792</td>\n","      <td>1</td>\n","      <td>A crystal growth furnace comprising a crucible...</td>\n","      <td>1. A crystal growth furnace for growing a crys...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>14360978</td>\n","      <td>0</td>\n","      <td>A shoe midsole is composed of a base plate (1)...</td>\n","      <td>1. A sole member of footwear comprising a base...</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>14369795</td>\n","      <td>1</td>\n","      <td>A ratchet tool includes a shaft member, a hand...</td>\n","      <td>1. A ratchet tool, comprising a shaft member, ...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>16144</th>\n","      <td>15002390</td>\n","      <td>1</td>\n","      <td>A wavelength tunable laser device, including: ...</td>\n","      <td>1. A wavelength tunable laser device, comprisi...</td>\n","    </tr>\n","    <tr>\n","      <th>16145</th>\n","      <td>15002391</td>\n","      <td>1</td>\n","      <td>In one aspect, a method for use in preparing a...</td>\n","      <td>1. (canceled) 2. The method of claim 19, where...</td>\n","    </tr>\n","    <tr>\n","      <th>16148</th>\n","      <td>15002394</td>\n","      <td>1</td>\n","      <td>A robot hand controlling method executes calcu...</td>\n","      <td>1. A controlling method of a robot hand, the r...</td>\n","    </tr>\n","    <tr>\n","      <th>16149</th>\n","      <td>15002396</td>\n","      <td>0</td>\n","      <td>A fusion protein is disclosed. The fusion prot...</td>\n","      <td>1. A fusion protein comprising an Fc fragment ...</td>\n","    </tr>\n","    <tr>\n","      <th>16150</th>\n","      <td>15330955</td>\n","      <td>0</td>\n","      <td>A pipe extraction tool that grips the inside o...</td>\n","      <td>1. A pipe extraction tool for extracting a pip...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>8719 rows × 4 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-98f84a20-c0eb-4438-9beb-b98a63c9968e')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-98f84a20-c0eb-4438-9beb-b98a63c9968e button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-98f84a20-c0eb-4438-9beb-b98a63c9968e');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":15}],"source":["trainDF3 = trainDF2.rename(columns={'decision': 'label'})\n","trainDF3"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":641},"id":"v3Qvaex7dTzG","executionInfo":{"status":"ok","timestamp":1682021437285,"user_tz":240,"elapsed":6,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"3c644844-db7e-4c3d-da66-afba74a1ca9a"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["     patent_number  label                                           abstract  \\\n","0         13144833      0  Regimen for the treatment of rosacea include t...   \n","1         14006524      1  A clamp arrangement includes a pair of bracket...   \n","2         14365653      0  A system and method for device action and conf...   \n","4         14396367      0  Systems and methods for managing datasets prod...   \n","9         14416282      1  A scan driving circuit is provided. The scan d...   \n","...            ...    ...                                                ...   \n","9085      15011551      0  The non-rigid gate device as described may be ...   \n","9090      15011556      0  The present invention provides an improved unc...   \n","9091      15011557      1  A method for detecting a software-race conditi...   \n","9092      15011558      1  The present application relates to multi-stage...   \n","9093      15011559      1  A paper feeder includes a housing, a driving u...   \n","\n","                                                 claims  \n","0     1. A treatment regimen comprising: cleansing a...  \n","1     1. A clamp arrangement for supporting a fractu...  \n","2     1-20. (canceled) 21. A mobile device comprisin...  \n","4     1. A method, comprising: executing, by one or ...  \n","9     1. A scan driving circuit for driving a scan l...  \n","...                                                 ...  \n","9085  1; A non-rigid blocking apparatus referred to ...  \n","9090  1. A method for rendering a plastic surface am...  \n","9091  1. A method for detecting a software-race cond...  \n","9092  1. A multi-stage amplitude modulation-based me...  \n","9093  1. A paper feeder, comprising: a housing; a dr...  \n","\n","[4888 rows x 4 columns]"],"text/html":["\n","  <div id=\"df-6782f4a5-49e1-44f0-8e06-bc644aaaed35\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>patent_number</th>\n","      <th>label</th>\n","      <th>abstract</th>\n","      <th>claims</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>13144833</td>\n","      <td>0</td>\n","      <td>Regimen for the treatment of rosacea include t...</td>\n","      <td>1. A treatment regimen comprising: cleansing a...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>14006524</td>\n","      <td>1</td>\n","      <td>A clamp arrangement includes a pair of bracket...</td>\n","      <td>1. A clamp arrangement for supporting a fractu...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>14365653</td>\n","      <td>0</td>\n","      <td>A system and method for device action and conf...</td>\n","      <td>1-20. (canceled) 21. A mobile device comprisin...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>14396367</td>\n","      <td>0</td>\n","      <td>Systems and methods for managing datasets prod...</td>\n","      <td>1. A method, comprising: executing, by one or ...</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>14416282</td>\n","      <td>1</td>\n","      <td>A scan driving circuit is provided. The scan d...</td>\n","      <td>1. A scan driving circuit for driving a scan l...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>9085</th>\n","      <td>15011551</td>\n","      <td>0</td>\n","      <td>The non-rigid gate device as described may be ...</td>\n","      <td>1; A non-rigid blocking apparatus referred to ...</td>\n","    </tr>\n","    <tr>\n","      <th>9090</th>\n","      <td>15011556</td>\n","      <td>0</td>\n","      <td>The present invention provides an improved unc...</td>\n","      <td>1. A method for rendering a plastic surface am...</td>\n","    </tr>\n","    <tr>\n","      <th>9091</th>\n","      <td>15011557</td>\n","      <td>1</td>\n","      <td>A method for detecting a software-race conditi...</td>\n","      <td>1. A method for detecting a software-race cond...</td>\n","    </tr>\n","    <tr>\n","      <th>9092</th>\n","      <td>15011558</td>\n","      <td>1</td>\n","      <td>The present application relates to multi-stage...</td>\n","      <td>1. A multi-stage amplitude modulation-based me...</td>\n","    </tr>\n","    <tr>\n","      <th>9093</th>\n","      <td>15011559</td>\n","      <td>1</td>\n","      <td>A paper feeder includes a housing, a driving u...</td>\n","      <td>1. A paper feeder, comprising: a housing; a dr...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>4888 rows × 4 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6782f4a5-49e1-44f0-8e06-bc644aaaed35')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-6782f4a5-49e1-44f0-8e06-bc644aaaed35 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-6782f4a5-49e1-44f0-8e06-bc644aaaed35');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":16}],"source":["valDF3 = valDF2.rename(columns={'decision': 'label'})\n","valDF3"]},{"cell_type":"markdown","metadata":{"id":"hJ8DMaCXdTzG"},"source":["We can grab the data for each column so that we have a list of values for training labels, training texts, validation labels, and validation texts.\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"z9omfOd0dTzG"},"outputs":[],"source":["trainData = {\n","  \"patent_numbers\":trainDF3[\"patent_number\"].tolist(),\n","  \"labels\":trainDF3[\"label\"].tolist(),\n","  \"abstracts\":trainDF3[\"abstract\"].tolist(),\n","  \"claims\":trainDF3[\"claims\"].tolist(),\n","}\n","valData = {\n","  \"patent_numbers\":valDF3[\"patent_number\"].tolist(),\n","  \"labels\":valDF3[\"label\"].tolist(),\n","  \"abstracts\":valDF3[\"abstract\"].tolist(),\n","  \"claims\":valDF3[\"claims\"].tolist(),\n","}"]},{"cell_type":"markdown","source":["We will save these dictionaries as data for later."],"metadata":{"id":"CLeEbFI_NBuK"}},{"cell_type":"code","source":["if not os.path.exists(\"./data\"):\n","  os.makedirs('./data')\n","\n","with open(\"./data/train.json\", \"w\") as outfile:\n","  json.dump(trainData, outfile, indent=2)\n","with open(\"./data/val.json\", \"w\") as outfile:\n","  json.dump(valData, outfile, indent=2)"],"metadata":{"id":"NBPNxz7qNHRq"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"pE3HG8bUdTzG"},"source":["## Loading the Trainer\n","\n","Now we can start training! This time, we will just go with `distilbert-base-uncased` for simplicity."]},{"cell_type":"markdown","source":["### Initializing Classes and Trainers"],"metadata":{"id":"YklaXlgDO6Jw"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hxE_CIT_dTzH","executionInfo":{"status":"ok","timestamp":1682021471720,"user_tz":240,"elapsed":16542,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"758b0092-d56e-47b6-852a-4a19915bfe0c"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (2.0.0+cu118)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from torch) (3.1.2)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch) (4.5.0)\n","Requirement already satisfied: sympy in /usr/local/lib/python3.9/dist-packages (from torch) (1.11.1)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from torch) (3.11.0)\n","Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.9/dist-packages (from torch) (2.0.0)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.9/dist-packages (from torch) (3.1)\n","Requirement already satisfied: lit in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch) (16.0.1)\n","Requirement already satisfied: cmake in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch) (3.25.2)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->torch) (2.1.2)\n","Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.9/dist-packages (from sympy->torch) (1.3.0)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting transformers\n","  Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m81.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.11.0)\n","Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n","  Downloading tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m100.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n","Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.4)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n","Installing collected packages: tokenizers, transformers\n","Successfully installed tokenizers-0.13.3 transformers-4.28.1\n"]}],"source":["!pip install torch\n","!pip install transformers"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"i8_0Ih_WdTzH"},"outputs":[],"source":["from torch.utils.data import Dataset, DataLoader\n","from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n","from transformers import Trainer, TrainingArguments, AdamW"]},{"cell_type":"code","source":["torch.backends.cuda.matmul.allow_tf32 = True\n","model_name = \"distilbert-base-uncased\"\n","upsto_abstracts_model_path = './models/uspto_abstracts'\n","upsto_claims_model_path = './models/uspto_claims'"],"metadata":{"id":"wXkvS5h2NrzW","executionInfo":{"status":"ok","timestamp":1682032710087,"user_tz":240,"elapsed":217,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}}},"execution_count":39,"outputs":[]},{"cell_type":"markdown","source":["We will create a Dataset class for the training"],"metadata":{"id":"awXD1_ltNxPC"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"yVi-Vhb-dTzH"},"outputs":[],"source":["class USPTODataset(Dataset):\n","    def __init__(self, encodings, labels):\n","        self.encodings = encodings\n","        self.labels = labels\n","    def __getitem__(self, idx):\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","    def __len__(self):\n","        return len(self.labels)\n"]},{"cell_type":"markdown","source":["### Double-Checking the Data\n","\n","We will do a basic check: Do we have `trainData` and `valData` cached? If not, we need to load it in!"],"metadata":{"id":"ZXqCGaTxN7qy"}},{"cell_type":"code","source":["trainDataPath = \"./data/train.json\"\n","valDataPath = \"./data/val.json\"\n","\n","if trainData is None and os.path.exists(trainDataPath):\n","  f = open(trainDataPath)\n","  trainData = json.load(f)\n","  f.close()\n","if valData is None and os.path.exists(valDataPath):\n","  f = open(valDataPath)\n","  valData = json.load(f)\n","  f.close()"],"metadata":{"id":"8Szn0TJ-N7CI"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Encoding the Data"],"metadata":{"id":"V3oKe81RPIgq"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"4Cxzzr6KdTzI","executionInfo":{"status":"ok","timestamp":1682021490811,"user_tz":240,"elapsed":1763,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"colab":{"base_uri":"https://localhost:8080/","height":257,"referenced_widgets":["37e352aeab994637887c9fce16a4fdda","7ce12e08913445429c0f44752b5f821c","6999d35a0c09459b9d0c9d47dba70320","5dc74e126ba4481e8e15ffa59b1eaf8e","3d9ca529621f46da9ed93641ae56b4ea","d9e649a7a52641b28b77037fc4713d77","3d83814aa933459dac4d493ab6c2ecf7","a08f7de9b7284616a3a6f2176804a714","2f624544ba68401491be11bb78cc8086","e7d6e1e3fb9a49e0b58281aca52517b5","51b1280a3a5e4facbeafb28923d77133","0da194d892754092ad01803ff69c9a7c","1e085de4a50e4c2685b9d24e0f289679","373fdb0d94684d44aa5e0e6293319bc6","c31e597bc5c14d14b287206ea8be2522","acf120e8d7f14a23a7a8a8f6d2c72d54","8bbb84dc028a4b62b1ea4dcd98131706","72125209dca54decaae05e5678a9eb60","659e0520847d4db5a5cf717a7be903b1","7077cb21a4b9491ab20b2af5dd7d30e5","517fdc4c1e61453f9e167dd8cc33f021","055f263ecfab430da77808fdc07699a1","cb4f082d2c384b74a54bac7e92b19772","12f8fa71da0d434a88c43ab13159fbc6","cb2d86cc73fd4a529d75aeb8e9c354ae","28bff5766c51461e8b9456c07aac9c57","8f3f4ca0a7114fb3929b2b80402c19ad","097daf3ff77f4d39809fe3a9d5bbd3c3","e53f41626ff34cbca574ef5be6b910e9","e7a1f0216c184d5e8abee0f4998f7cb7","ec7f6f10a68f4aa3b1696e4e1d59c231","041087211da7424e86b03574c00bcc7e","44c305d3e3ec44a1ac31a9e82ee00fd5","6cce9c60a7074c40ad9992597eb1f50a","87cad6102054466d8e1243da205cf506","5d1ae7f7479e485a97e80db391b6e694","789cf158a3154bba8b1091b2ec443843","f4d8392b478149949a77bf606fea3090","d457e5284b6e4ecf8efddff65b613315","abff237c84fe446f857de2c7c6fc466c","e34a8a0a27614aab95e63b221861965f","bf607dd1b0ba47c2a4b42cd934786356","f66a864297f1446d92968786100fa6ef","035fd49261424e179b16f2ae4688944e"]},"outputId":"d1afd722-6591-4860-db86-5bb9ffd58e7d"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"37e352aeab994637887c9fce16a4fdda"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0da194d892754092ad01803ff69c9a7c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)/main/tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cb4f082d2c384b74a54bac7e92b19772"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading (…)lve/main/config.json:   0%|          | 0.00/483 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6cce9c60a7074c40ad9992597eb1f50a"}},"metadata":{}}],"source":["# Initializing the Tokenizer\n","tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"suHHAY90dTzI"},"outputs":[],"source":["# Encoding the Data\n","train_abstracts_encodings = tokenizer(trainData[\"abstracts\"], truncation=True, padding=True)\n","train_claims_encodings = tokenizer(trainData[\"claims\"], truncation=True, padding=True)"]},{"cell_type":"code","source":["# Creating the Datasets from the data\n","train_abstracts_dataset = USPTODataset(train_abstracts_encodings, trainData[\"labels\"])\n","train_claims_dataset = USPTODataset(train_claims_encodings, trainData[\"labels\"])"],"metadata":{"id":"hptmNAJ1PcZN"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Model Preparation\n","\n","We need to initialize the model that we will use as a base now."],"metadata":{"id":"22RFJcEXPnZB"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":830,"referenced_widgets":["a7752b9c0c89474ab6662cbc5a19c513","2f392aea78f945d2ac8198a9be17288b","25c8b6c20fd84746a0b3b20a99d1b5bf","f0b3815c4e314899aba7256040f4cbbb","75d63216400645deab0d6e56cea0a67c","a03b5a76f061429aba6fc0935fbb46c8","8aaea1deb9ea452798858d0209668bda","a1c2445dd87f4426ba565b7483edca24","47943fe3054f42e3856dfd9d2d7b362a","e6531320c3fa445baa68aad6c20f6388","da050369b5e8464089e69a326f34fa43"]},"id":"_TuzDNWsdTzI","executionInfo":{"status":"ok","timestamp":1682021554233,"user_tz":240,"elapsed":15686,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"862da1d9-10c5-4aa8-d241-8edcb6d27b21"},"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading pytorch_model.bin:   0%|          | 0.00/268M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a7752b9c0c89474ab6662cbc5a19c513"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.weight', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_layer_norm.bias', 'vocab_transform.bias']\n","- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"output_type":"execute_result","data":{"text/plain":["DistilBertForSequenceClassification(\n","  (distilbert): DistilBertModel(\n","    (embeddings): Embeddings(\n","      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n","      (position_embeddings): Embedding(512, 768)\n","      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (transformer): Transformer(\n","      (layer): ModuleList(\n","        (0-5): 6 x TransformerBlock(\n","          (attention): MultiHeadSelfAttention(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n","            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n","          )\n","          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","          (ffn): FFN(\n","            (dropout): Dropout(p=0.1, inplace=False)\n","            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n","            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n","            (activation): GELUActivation()\n","          )\n","          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","        )\n","      )\n","    )\n","  )\n","  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n","  (classifier): Linear(in_features=768, out_features=2, bias=True)\n","  (dropout): Dropout(p=0.2, inplace=False)\n",")"]},"metadata":{},"execution_count":27}],"source":["device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n","model = DistilBertForSequenceClassification.from_pretrained(model_name)\n","model.to(device)\n","model.train()"]},{"cell_type":"markdown","source":["### Training Preparation"],"metadata":{"id":"ff23ZnIMQOPj"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"nQJhJGNmdTzI"},"outputs":[],"source":["train_abstracts_loader = DataLoader(train_abstracts_dataset, batch_size=32, shuffle=True)\n","train_claims_loader = DataLoader(train_claims_dataset, batch_size=32, shuffle=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jgFY2KNSdTzJ"},"outputs":[],"source":["optim = AdamW(model.parameters(), lr=5e-5)"]},{"cell_type":"markdown","source":["### Training!\n","\n","We will be training for 10 epochs"],"metadata":{"id":"jSfVwiFZQfyF"}},{"cell_type":"code","source":["def Train(loader, save_path, num_train_epochs=2):\n","  batch_num = len(loader)\n","  for epoch in range(num_train_epochs):\n","    print(f'\\t- Training epoch {epoch+1}/{num_train_epochs}')\n","    batch_count = 0\n","    for batch in loader:\n","      print(f'{batch_count}|{batch_num} - {round((batch_count/batch_num)*100)}%', end=\"\")\n","      #print('\\t\\t- optim zero grad')\n","      optim.zero_grad()\n","      #print('\\t\\t- input_ids')\n","      input_ids = batch['input_ids'].to(device)\n","      #print('\\t\\t- attention_mask')\n","      attention_mask = batch['attention_mask'].to(device)\n","      #print('\\t\\t- labels0')\n","      labels = batch['labels'].to(device)\n","      #print('\\t\\t- outputs')\n","      outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n","            \n","      #print('\\t\\t- loss')\n","      loss = outputs[0]\n","      #print('\\t\\t- backwards')\n","      loss.backward()\n","      #print('\\t\\t- step')\n","      optim.step()\n","\n","      batch_count += 1\n","      print(\"\\r\", end=\"\")\n","    \n","    model.save_pretrained(save_path, from_pt=True) \n","    print(f'Saved model in {save_path}!\\n')"],"metadata":{"id":"vrCBIFTOQoqH"},"execution_count":null,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"id":"6o-434bzdTzJ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1682030797217,"user_tz":240,"elapsed":9174487,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"b02fca82-05a2-48d7-9a85-5f4521cfb1b4"},"outputs":[{"output_type":"stream","name":"stdout","text":["=== TRAINING ABSTRACTS ===\n","\t- Training epoch 1/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 2/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 3/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 4/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 5/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 6/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 7/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 8/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 9/10\n","Saved model in ./models/upsto_abstracts!\n","\n","\t- Training epoch 10/10\n","Saved model in ./models/upsto_abstracts!\n","\n","----\n","=== TRAINING CLAIMS ===\n","\t- Training epoch 1/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 2/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 3/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 4/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 5/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 6/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 7/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 8/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 9/10\n","Saved model in ./models/upsto_claims!\n","\n","\t- Training epoch 10/10\n","Saved model in ./models/upsto_claims!\n","\n"]}],"source":["print(\"=== TRAINING ABSTRACTS ===\")\n","Train(train_abstracts_loader,upsto_abstracts_model_path, num_train_epochs=10)\n","print(\"----\")\n","print(\"=== TRAINING CLAIMS ===\")\n","Train(train_claims_loader,upsto_claims_model_path, num_train_epochs=10)"]},{"cell_type":"code","execution_count":40,"metadata":{"id":"IUIwKTDVdTzJ","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1682032764012,"user_tz":240,"elapsed":30306,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"70ca9022-a039-4d35-a7f1-4e1c570af2c1"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["'/content/uspto_claims.zip'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":40}],"source":["import shutil\n","shutil.make_archive(\"uspto_abstracts\", 'zip', './models/uspto_abstracts')\n","shutil.make_archive(\"uspto_claims\", 'zip', './models/uspto_claims')\n"]},{"cell_type":"code","source":[],"metadata":{"id":"c0P0PK7V9BDA"},"execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.6"},"colab":{"provenance":[],"gpuType":"T4"},"widgets":{"application/vnd.jupyter.widget-state+json":{"9f321834b0bc4bc1ac089f9813fc0fb1":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_12e47c63e2fb4596b645c252e9756899","IPY_MODEL_043c0260b1e14a129feab90cff5ef099","IPY_MODEL_ddba4a3380794815a732bd1a453c925c"],"layout":"IPY_MODEL_8d6813d2f8ec401d85e599eadc8dc093"}},"12e47c63e2fb4596b645c252e9756899":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4dca0683fdc5459e88f6687f3196af7f","placeholder":"​","style":"IPY_MODEL_0dba55c366e44537adf4048c24391786","value":"Downloading builder script: 100%"}},"043c0260b1e14a129feab90cff5ef099":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_6695e22c05d548139e71029524c0bc68","max":14711,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2a7b1a0f1e94446bb343979d86264f0c","value":14711}},"ddba4a3380794815a732bd1a453c925c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_155a8f30bf6f4bbcb491dda1ad722c3b","placeholder":"​","style":"IPY_MODEL_aa0b1c2f45104fc5b59e1704bce27e5f","value":" 14.7k/14.7k [00:00&lt;00:00, 166kB/s]"}},"8d6813d2f8ec401d85e599eadc8dc093":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4dca0683fdc5459e88f6687f3196af7f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0dba55c366e44537adf4048c24391786":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"6695e22c05d548139e71029524c0bc68":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2a7b1a0f1e94446bb343979d86264f0c":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"155a8f30bf6f4bbcb491dda1ad722c3b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"aa0b1c2f45104fc5b59e1704bce27e5f":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"06a520a78f5d4b95982a476b29734cbf":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_c5be3bfd001346af8d33caf0a64efc7f","IPY_MODEL_de1f71b45aba42e4a807100c4a8bf81f","IPY_MODEL_6eee3b7c1ce9453e8c73f568537d5ac8"],"layout":"IPY_MODEL_8f895536b733460d9ad2987333e3733b"}},"c5be3bfd001346af8d33caf0a64efc7f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e05d3a3d645948a3b2745cac3d5aa737","placeholder":"​","style":"IPY_MODEL_da25e097845440948f1d3ec1096f2a99","value":"Downloading readme: 100%"}},"de1f71b45aba42e4a807100c4a8bf81f":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_5cfa97b42072490f8ab4d5f60e0a1792","max":10898,"min":0,"orientation":"horizontal","style":"IPY_MODEL_22f78b57f5604f928475f9a81d723baf","value":10898}},"6eee3b7c1ce9453e8c73f568537d5ac8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_87b97444ca194dd18f1a5bd6e4082fb1","placeholder":"​","style":"IPY_MODEL_de9855ff33fb41b58ef21fbb26f81b85","value":" 10.9k/10.9k [00:00&lt;00:00, 274kB/s]"}},"8f895536b733460d9ad2987333e3733b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e05d3a3d645948a3b2745cac3d5aa737":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"da25e097845440948f1d3ec1096f2a99":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5cfa97b42072490f8ab4d5f60e0a1792":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"22f78b57f5604f928475f9a81d723baf":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"87b97444ca194dd18f1a5bd6e4082fb1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"de9855ff33fb41b58ef21fbb26f81b85":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9a74f42973b34203af1100702e07c28f":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_08d60a39b36942c184664157f738c5f0","IPY_MODEL_fc886230dc454a72a3b2954f9818e9e8","IPY_MODEL_482f58d514734fd2aa1d82693c7d5c34"],"layout":"IPY_MODEL_f52a0089a4494e719c54289ff33a1c04"}},"08d60a39b36942c184664157f738c5f0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c2f38dc161184b16ba930c2362bc8e3b","placeholder":"​","style":"IPY_MODEL_e0f8f41cd81647908c645f76483148e5","value":"Downloading data: 100%"}},"fc886230dc454a72a3b2954f9818e9e8":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_2326d9dbc48b40769dcde81195324f40","max":6665746,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0887166927dd4805ae3346c3158be0d2","value":6665746}},"482f58d514734fd2aa1d82693c7d5c34":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_04f4e59cfdfe492c9644dd99f46910e9","placeholder":"​","style":"IPY_MODEL_5cfce12745f44e5a9c94b72120170915","value":" 6.67M/6.67M [00:00&lt;00:00, 30.3MB/s]"}},"f52a0089a4494e719c54289ff33a1c04":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c2f38dc161184b16ba930c2362bc8e3b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e0f8f41cd81647908c645f76483148e5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2326d9dbc48b40769dcde81195324f40":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0887166927dd4805ae3346c3158be0d2":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"04f4e59cfdfe492c9644dd99f46910e9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5cfce12745f44e5a9c94b72120170915":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d7bfd7f1e624447a9c1eea68116d915b":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_6a337183787b43ec8634dc0c5b95bd72","IPY_MODEL_7dd3444c8a754737b4ed1c8f64f68601","IPY_MODEL_d9c18d34c6b249b78fcbace6d79cbdec"],"layout":"IPY_MODEL_52879a2ff4864ba084a15fe02ab8b1fa"}},"6a337183787b43ec8634dc0c5b95bd72":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a90a02a84860461789ad04c386cdf44e","placeholder":"​","style":"IPY_MODEL_8cc2a2fe26534066a3a778704cc5984b","value":"Downloading data: 100%"}},"7dd3444c8a754737b4ed1c8f64f68601":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_754c5121d01646c4aa4284df0eb9bb4f","max":387636489,"min":0,"orientation":"horizontal","style":"IPY_MODEL_b957825a21c2412a9b0101869d1f44d8","value":387636489}},"d9c18d34c6b249b78fcbace6d79cbdec":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9f145dfa3f6347a7ab46aaa5fb294ea1","placeholder":"​","style":"IPY_MODEL_85d3c680e7424ab99c5153f315ac51a0","value":" 388M/388M [00:09&lt;00:00, 43.4MB/s]"}},"52879a2ff4864ba084a15fe02ab8b1fa":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a90a02a84860461789ad04c386cdf44e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8cc2a2fe26534066a3a778704cc5984b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"754c5121d01646c4aa4284df0eb9bb4f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b957825a21c2412a9b0101869d1f44d8":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"9f145dfa3f6347a7ab46aaa5fb294ea1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"85d3c680e7424ab99c5153f315ac51a0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"18d0fb273b444854a97ddb941859e0bc":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_fedd62c855344bbc931485f1bed1bb20","IPY_MODEL_3cd64accf6104e5f8a3ad59381650a61","IPY_MODEL_d39610354db4493f9f33405267b50179"],"layout":"IPY_MODEL_c82c7f9596a746b6a91435ddfe2801f5"}},"fedd62c855344bbc931485f1bed1bb20":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_453092a5a76540f2b7561c3f4f84f8b2","placeholder":"​","style":"IPY_MODEL_ba82d11406a2498ba8948b5729fb4935","value":"Generating train split: "}},"3cd64accf6104e5f8a3ad59381650a61":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"info","description":"","description_tooltip":null,"layout":"IPY_MODEL_50863c3f09e343a9970b73dc70abba90","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_269546507c224a049088863d503401b1","value":1}},"d39610354db4493f9f33405267b50179":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0cd095c293104c67996f52685a29b7e9","placeholder":"​","style":"IPY_MODEL_f4c9a13af4cd478bacedd30ee2d81b8d","value":" 16132/0 [00:13&lt;00:00, 662.47 examples/s]"}},"c82c7f9596a746b6a91435ddfe2801f5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"453092a5a76540f2b7561c3f4f84f8b2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ba82d11406a2498ba8948b5729fb4935":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"50863c3f09e343a9970b73dc70abba90":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"269546507c224a049088863d503401b1":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"0cd095c293104c67996f52685a29b7e9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f4c9a13af4cd478bacedd30ee2d81b8d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f9ef1b3c4aa54426af822c5f8420f2ca":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_28f10c7180d24373ad411015ef51d68c","IPY_MODEL_59139bbd088c4804a9c8213afc3ddf21","IPY_MODEL_a6c68f2b636b4b10ab7f846a789b00aa"],"layout":"IPY_MODEL_417d14969aa141c885ae6ddd6b554324"}},"28f10c7180d24373ad411015ef51d68c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_345e90b85bb542168925cc014a1780ff","placeholder":"​","style":"IPY_MODEL_0c2447ec9c2346feb9d9d34ad6f5ec89","value":"Generating validation split: "}},"59139bbd088c4804a9c8213afc3ddf21":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"info","description":"","description_tooltip":null,"layout":"IPY_MODEL_2c3ac49e5ace478ea09109090519aa65","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_984011a4cf53494baca77e2847c1a6ec","value":1}},"a6c68f2b636b4b10ab7f846a789b00aa":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0ad51ba66c3a48a2bf44a58c63d6f6b8","placeholder":"​","style":"IPY_MODEL_f71dc5f4ba95461eafff245393c29efb","value":" 9094/0 [00:22&lt;00:00, 1703.84 examples/s]"}},"417d14969aa141c885ae6ddd6b554324":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":"hidden","width":null}},"345e90b85bb542168925cc014a1780ff":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0c2447ec9c2346feb9d9d34ad6f5ec89":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2c3ac49e5ace478ea09109090519aa65":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"984011a4cf53494baca77e2847c1a6ec":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"0ad51ba66c3a48a2bf44a58c63d6f6b8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f71dc5f4ba95461eafff245393c29efb":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"422a18bc728147cfab41a3d434784b87":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_8e84ffdf79144dee93ec82a857d8abf8","IPY_MODEL_a6de06c9f8c9494abd2c5146a151abf4","IPY_MODEL_b707d410fc0b46be82513a3156c864ec"],"layout":"IPY_MODEL_7cdeb5f8b21f42549d6919f0a140697a"}},"8e84ffdf79144dee93ec82a857d8abf8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8fe2841c26c947c28479cd459ae5edff","placeholder":"​","style":"IPY_MODEL_d40cdc8dac2b40429bd92e6330916ed5","value":"100%"}},"a6de06c9f8c9494abd2c5146a151abf4":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_f8c2274beb1d47e1a2e8d76d3f2babe1","max":2,"min":0,"orientation":"horizontal","style":"IPY_MODEL_a0ade24bceb54a7daedc48858588590a","value":2}},"b707d410fc0b46be82513a3156c864ec":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_895d67eeac0f46edb926a8bdd33f419b","placeholder":"​","style":"IPY_MODEL_21d8d59ca7304455aae43a23c7cbbbe8","value":" 2/2 [00:00&lt;00:00, 55.83it/s]"}},"7cdeb5f8b21f42549d6919f0a140697a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8fe2841c26c947c28479cd459ae5edff":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d40cdc8dac2b40429bd92e6330916ed5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f8c2274beb1d47e1a2e8d76d3f2babe1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a0ade24bceb54a7daedc48858588590a":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"895d67eeac0f46edb926a8bdd33f419b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"21d8d59ca7304455aae43a23c7cbbbe8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"37e352aeab994637887c9fce16a4fdda":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7ce12e08913445429c0f44752b5f821c","IPY_MODEL_6999d35a0c09459b9d0c9d47dba70320","IPY_MODEL_5dc74e126ba4481e8e15ffa59b1eaf8e"],"layout":"IPY_MODEL_3d9ca529621f46da9ed93641ae56b4ea"}},"7ce12e08913445429c0f44752b5f821c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d9e649a7a52641b28b77037fc4713d77","placeholder":"​","style":"IPY_MODEL_3d83814aa933459dac4d493ab6c2ecf7","value":"Downloading (…)okenizer_config.json: 100%"}},"6999d35a0c09459b9d0c9d47dba70320":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a08f7de9b7284616a3a6f2176804a714","max":28,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2f624544ba68401491be11bb78cc8086","value":28}},"5dc74e126ba4481e8e15ffa59b1eaf8e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e7d6e1e3fb9a49e0b58281aca52517b5","placeholder":"​","style":"IPY_MODEL_51b1280a3a5e4facbeafb28923d77133","value":" 28.0/28.0 [00:00&lt;00:00, 331B/s]"}},"3d9ca529621f46da9ed93641ae56b4ea":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d9e649a7a52641b28b77037fc4713d77":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3d83814aa933459dac4d493ab6c2ecf7":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a08f7de9b7284616a3a6f2176804a714":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2f624544ba68401491be11bb78cc8086":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"e7d6e1e3fb9a49e0b58281aca52517b5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"51b1280a3a5e4facbeafb28923d77133":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"0da194d892754092ad01803ff69c9a7c":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_1e085de4a50e4c2685b9d24e0f289679","IPY_MODEL_373fdb0d94684d44aa5e0e6293319bc6","IPY_MODEL_c31e597bc5c14d14b287206ea8be2522"],"layout":"IPY_MODEL_acf120e8d7f14a23a7a8a8f6d2c72d54"}},"1e085de4a50e4c2685b9d24e0f289679":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8bbb84dc028a4b62b1ea4dcd98131706","placeholder":"​","style":"IPY_MODEL_72125209dca54decaae05e5678a9eb60","value":"Downloading (…)solve/main/vocab.txt: 100%"}},"373fdb0d94684d44aa5e0e6293319bc6":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_659e0520847d4db5a5cf717a7be903b1","max":231508,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7077cb21a4b9491ab20b2af5dd7d30e5","value":231508}},"c31e597bc5c14d14b287206ea8be2522":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_517fdc4c1e61453f9e167dd8cc33f021","placeholder":"​","style":"IPY_MODEL_055f263ecfab430da77808fdc07699a1","value":" 232k/232k [00:00&lt;00:00, 3.72MB/s]"}},"acf120e8d7f14a23a7a8a8f6d2c72d54":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8bbb84dc028a4b62b1ea4dcd98131706":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"72125209dca54decaae05e5678a9eb60":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"659e0520847d4db5a5cf717a7be903b1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7077cb21a4b9491ab20b2af5dd7d30e5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"517fdc4c1e61453f9e167dd8cc33f021":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"055f263ecfab430da77808fdc07699a1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"cb4f082d2c384b74a54bac7e92b19772":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_12f8fa71da0d434a88c43ab13159fbc6","IPY_MODEL_cb2d86cc73fd4a529d75aeb8e9c354ae","IPY_MODEL_28bff5766c51461e8b9456c07aac9c57"],"layout":"IPY_MODEL_8f3f4ca0a7114fb3929b2b80402c19ad"}},"12f8fa71da0d434a88c43ab13159fbc6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_097daf3ff77f4d39809fe3a9d5bbd3c3","placeholder":"​","style":"IPY_MODEL_e53f41626ff34cbca574ef5be6b910e9","value":"Downloading (…)/main/tokenizer.json: 100%"}},"cb2d86cc73fd4a529d75aeb8e9c354ae":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_e7a1f0216c184d5e8abee0f4998f7cb7","max":466062,"min":0,"orientation":"horizontal","style":"IPY_MODEL_ec7f6f10a68f4aa3b1696e4e1d59c231","value":466062}},"28bff5766c51461e8b9456c07aac9c57":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_041087211da7424e86b03574c00bcc7e","placeholder":"​","style":"IPY_MODEL_44c305d3e3ec44a1ac31a9e82ee00fd5","value":" 466k/466k [00:00&lt;00:00, 5.28MB/s]"}},"8f3f4ca0a7114fb3929b2b80402c19ad":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"097daf3ff77f4d39809fe3a9d5bbd3c3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e53f41626ff34cbca574ef5be6b910e9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e7a1f0216c184d5e8abee0f4998f7cb7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ec7f6f10a68f4aa3b1696e4e1d59c231":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"041087211da7424e86b03574c00bcc7e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"44c305d3e3ec44a1ac31a9e82ee00fd5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"6cce9c60a7074c40ad9992597eb1f50a":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_87cad6102054466d8e1243da205cf506","IPY_MODEL_5d1ae7f7479e485a97e80db391b6e694","IPY_MODEL_789cf158a3154bba8b1091b2ec443843"],"layout":"IPY_MODEL_f4d8392b478149949a77bf606fea3090"}},"87cad6102054466d8e1243da205cf506":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d457e5284b6e4ecf8efddff65b613315","placeholder":"​","style":"IPY_MODEL_abff237c84fe446f857de2c7c6fc466c","value":"Downloading (…)lve/main/config.json: 100%"}},"5d1ae7f7479e485a97e80db391b6e694":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_e34a8a0a27614aab95e63b221861965f","max":483,"min":0,"orientation":"horizontal","style":"IPY_MODEL_bf607dd1b0ba47c2a4b42cd934786356","value":483}},"789cf158a3154bba8b1091b2ec443843":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f66a864297f1446d92968786100fa6ef","placeholder":"​","style":"IPY_MODEL_035fd49261424e179b16f2ae4688944e","value":" 483/483 [00:00&lt;00:00, 11.1kB/s]"}},"f4d8392b478149949a77bf606fea3090":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d457e5284b6e4ecf8efddff65b613315":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"abff237c84fe446f857de2c7c6fc466c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e34a8a0a27614aab95e63b221861965f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bf607dd1b0ba47c2a4b42cd934786356":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f66a864297f1446d92968786100fa6ef":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"035fd49261424e179b16f2ae4688944e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a7752b9c0c89474ab6662cbc5a19c513":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_2f392aea78f945d2ac8198a9be17288b","IPY_MODEL_25c8b6c20fd84746a0b3b20a99d1b5bf","IPY_MODEL_f0b3815c4e314899aba7256040f4cbbb"],"layout":"IPY_MODEL_75d63216400645deab0d6e56cea0a67c"}},"2f392aea78f945d2ac8198a9be17288b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a03b5a76f061429aba6fc0935fbb46c8","placeholder":"​","style":"IPY_MODEL_8aaea1deb9ea452798858d0209668bda","value":"Downloading pytorch_model.bin: 100%"}},"25c8b6c20fd84746a0b3b20a99d1b5bf":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a1c2445dd87f4426ba565b7483edca24","max":267967963,"min":0,"orientation":"horizontal","style":"IPY_MODEL_47943fe3054f42e3856dfd9d2d7b362a","value":267967963}},"f0b3815c4e314899aba7256040f4cbbb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e6531320c3fa445baa68aad6c20f6388","placeholder":"​","style":"IPY_MODEL_da050369b5e8464089e69a326f34fa43","value":" 268M/268M [00:07&lt;00:00, 41.4MB/s]"}},"75d63216400645deab0d6e56cea0a67c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a03b5a76f061429aba6fc0935fbb46c8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8aaea1deb9ea452798858d0209668bda":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a1c2445dd87f4426ba565b7483edca24":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"47943fe3054f42e3856dfd9d2d7b362a":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"e6531320c3fa445baa68aad6c20f6388":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"da050369b5e8464089e69a326f34fa43":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}},"accelerator":"GPU","gpuClass":"standard"},"nbformat":4,"nbformat_minor":0}
\ No newline at end of file