{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Harvard USPTO Dataset Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Packages\n",
    "\n",
    "We first need to import the actual USPTO dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.11.0)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (2022.8.2)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.23.3)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.64.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.28.1)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (9.0.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (6.0)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (1.5.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.13.4)\n",
      "Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.2.0)\n",
      "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.6)\n",
      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.8.4)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.14)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (22.1.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.8.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.3)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (2.1.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.4.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.12.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.0.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (1.26.11)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2022.4)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to extract the dataset. We filter only for those in January 2016."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset hupd (/home/jovyan/.cache/huggingface/datasets/HUPD___hupd/sample-a4eeba92b4229e93/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e39fd26828774c8e9d159a8b5d91c4f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset_dict = load_dataset('HUPD/hupd',\n",
    "    name='sample',\n",
    "    data_files=\"https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather\", \n",
    "    icpr_label=None,\n",
    "    train_filing_start_date='2016-01-01',\n",
    "    train_filing_end_date='2016-01-21',\n",
    "    val_filing_start_date='2016-01-22',\n",
    "    val_filing_end_date='2016-01-31',\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We print out the dataset to understand what exactly we want to look for"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id'],\n",
      "        num_rows: 16153\n",
      "    })\n",
      "    validation: Dataset({\n",
      "        features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id'],\n",
      "        num_rows: 9094\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(dataset_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We separate our data between training and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.DataFrame(dataset_dict['train'] )\n",
    "df_val = pd.DataFrame(dataset_dict['validation'] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can preview the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patent_number</th>\n",
       "      <th>decision</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>claims</th>\n",
       "      <th>background</th>\n",
       "      <th>summary</th>\n",
       "      <th>description</th>\n",
       "      <th>cpc_label</th>\n",
       "      <th>ipc_label</th>\n",
       "      <th>filing_date</th>\n",
       "      <th>patent_issue_date</th>\n",
       "      <th>date_published</th>\n",
       "      <th>examiner_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13261748</td>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>MINI-OPTICAL NETWORK TERMINAL (ONT)</td>\n",
       "      <td>The present invention relates to passive optic...</td>\n",
       "      <td>1. A compact optical network terminal, compris...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;A netwo...</td>\n",
       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;An aspect ...</td>\n",
       "      <td>FIELD OF THE INVENTION The present invention r...</td>\n",
       "      <td>H04Q110071</td>\n",
       "      <td>H04Q1100</td>\n",
       "      <td>20160120</td>\n",
       "      <td>20170606</td>\n",
       "      <td>20160526</td>\n",
       "      <td>95191.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13995128</td>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>APPARATUS FOR FORMING AND READING AN IDENTIFIC...</td>\n",
       "      <td>Embodiments of the invention provide a method ...</td>\n",
       "      <td>1. A method comprising: using a first reader t...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;Identif...</td>\n",
       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;In accorda...</td>\n",
       "      <td>CROSS-REFERENCE TO RELATED APPLICATIONS The pr...</td>\n",
       "      <td>G06K500</td>\n",
       "      <td>G06K500</td>\n",
       "      <td>20160112</td>\n",
       "      <td>20160322</td>\n",
       "      <td>20140102</td>\n",
       "      <td>59514.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14241799</td>\n",
       "      <td>PENDING</td>\n",
       "      <td>PORTABLE DRUG DISPENSER</td>\n",
       "      <td>A portable drug dispenser includes a chamber f...</td>\n",
       "      <td>1. A portable drug dispenser, comprising: a ch...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>This application claims priority from U.S. app...</td>\n",
       "      <td>A61J70084</td>\n",
       "      <td>A61J700</td>\n",
       "      <td>20160104</td>\n",
       "      <td></td>\n",
       "      <td>20171116</td>\n",
       "      <td>95928.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14348792</td>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>LIQUID-COOLED HEAT EXCHANGER</td>\n",
       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
       "      <td>1. A crystal growth furnace for growing a crys...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;1. Fiel...</td>\n",
       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;The presen...</td>\n",
       "      <td>CROSS-REFERENCE TO RELATED APPLICATIONS The pr...</td>\n",
       "      <td>C30B11003</td>\n",
       "      <td>C30B1100</td>\n",
       "      <td>20160111</td>\n",
       "      <td>20180529</td>\n",
       "      <td>20160512</td>\n",
       "      <td>63013.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14360978</td>\n",
       "      <td>REJECTED</td>\n",
       "      <td>SOLE MEMBER OF FOOTWEAR</td>\n",
       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
       "      <td>1. A sole member of footwear comprising a base...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND ART &lt;EOH&gt;When the heel touche...</td>\n",
       "      <td>&lt;SOH&gt; BRIEF DESCRIPTION OF THE DRAWINGS &lt;EOH&gt;F...</td>\n",
       "      <td>TECHNICAL FIELD The present invention relates ...</td>\n",
       "      <td>A43B13181</td>\n",
       "      <td>A43B1318</td>\n",
       "      <td>20160113</td>\n",
       "      <td></td>\n",
       "      <td>20160512</td>\n",
       "      <td>94490.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16148</th>\n",
       "      <td>15002394</td>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>ROBOT HAND CONTROLLING METHOD AND ROBOTICS DEVICE</td>\n",
       "      <td>A robot hand controlling method executes calcu...</td>\n",
       "      <td>1. A controlling method of a robot hand, the r...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;1. Fiel...</td>\n",
       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;An object ...</td>\n",
       "      <td>BACKGROUND OF THE INVENTION 1. Field of the In...</td>\n",
       "      <td>B25J91612</td>\n",
       "      <td>B25J916</td>\n",
       "      <td>20160120</td>\n",
       "      <td>20180710</td>\n",
       "      <td>20160804</td>\n",
       "      <td>66148.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16149</th>\n",
       "      <td>15002396</td>\n",
       "      <td>REJECTED</td>\n",
       "      <td>IMMUNOGLOBULIN FUSION PROTEINS AND USES THEREOF</td>\n",
       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
       "      <td>1. A fusion protein comprising an Fc fragment ...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;An immu...</td>\n",
       "      <td>&lt;SOH&gt; SUMMARY OF THE INVENTION &lt;EOH&gt;The presen...</td>\n",
       "      <td>The present application is a U.S. Nonprovision...</td>\n",
       "      <td>C07K14745</td>\n",
       "      <td>C07K14745</td>\n",
       "      <td>20160120</td>\n",
       "      <td></td>\n",
       "      <td>20161215</td>\n",
       "      <td>95819.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16150</th>\n",
       "      <td>15330955</td>\n",
       "      <td>REJECTED</td>\n",
       "      <td>PIPE EXTRACTION TOOL</td>\n",
       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
       "      <td>1. A pipe extraction tool for extracting a pip...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;1. Fiel...</td>\n",
       "      <td>&lt;SOH&gt; BRIEF SUMMARY OF THE INVENTION &lt;EOH&gt;The ...</td>\n",
       "      <td>CROSS-REFERENCES TO RELATED APPLICATIONS Not a...</td>\n",
       "      <td>B25B2714</td>\n",
       "      <td>B25B2714</td>\n",
       "      <td>20160120</td>\n",
       "      <td></td>\n",
       "      <td>20170907</td>\n",
       "      <td>95661.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16151</th>\n",
       "      <td>15330961</td>\n",
       "      <td>PENDING</td>\n",
       "      <td>Molded parts with thermoplastic cellulose biop...</td>\n",
       "      <td>A longitudinal extending body with oriented fi...</td>\n",
       "      <td>1. A longitudinal body of a solidified organic...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF INVENTION &lt;EOH&gt;In the medi...</td>\n",
       "      <td>&lt;SOH&gt; BRIEF SUMMARY OF THE PRESENT INVENTION &lt;...</td>\n",
       "      <td>CROSS REFERENCES Application claims priority o...</td>\n",
       "      <td>A61L3106</td>\n",
       "      <td>A61L3106</td>\n",
       "      <td>20160111</td>\n",
       "      <td></td>\n",
       "      <td>20171019</td>\n",
       "      <td>96956.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16152</th>\n",
       "      <td>15330968</td>\n",
       "      <td>PENDING</td>\n",
       "      <td>Transmission method with double directivity</td>\n",
       "      <td>A transmission method using a massive MIMO (Mu...</td>\n",
       "      <td>1. Transmission method with double directivity...</td>\n",
       "      <td>&lt;SOH&gt; BACKGROUND OF THE INVENTION &lt;EOH&gt;</td>\n",
       "      <td>&lt;SOH&gt; BRIEF SUMMARY OF THE INVENTION &lt;EOH&gt;The ...</td>\n",
       "      <td>BACKGROUND OF THE INVENTION Field of the Inven...</td>\n",
       "      <td>H04B7043</td>\n",
       "      <td>H04B704</td>\n",
       "      <td>20160114</td>\n",
       "      <td></td>\n",
       "      <td>20180329</td>\n",
       "      <td>70883.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16153 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      patent_number  decision  \\\n",
       "0          13261748  ACCEPTED   \n",
       "1          13995128  ACCEPTED   \n",
       "2          14241799   PENDING   \n",
       "3          14348792  ACCEPTED   \n",
       "4          14360978  REJECTED   \n",
       "...             ...       ...   \n",
       "16148      15002394  ACCEPTED   \n",
       "16149      15002396  REJECTED   \n",
       "16150      15330955  REJECTED   \n",
       "16151      15330961   PENDING   \n",
       "16152      15330968   PENDING   \n",
       "\n",
       "                                                   title  \\\n",
       "0                    MINI-OPTICAL NETWORK TERMINAL (ONT)   \n",
       "1      APPARATUS FOR FORMING AND READING AN IDENTIFIC...   \n",
       "2                                PORTABLE DRUG DISPENSER   \n",
       "3                           LIQUID-COOLED HEAT EXCHANGER   \n",
       "4                                SOLE MEMBER OF FOOTWEAR   \n",
       "...                                                  ...   \n",
       "16148  ROBOT HAND CONTROLLING METHOD AND ROBOTICS DEVICE   \n",
       "16149    IMMUNOGLOBULIN FUSION PROTEINS AND USES THEREOF   \n",
       "16150                               PIPE EXTRACTION TOOL   \n",
       "16151  Molded parts with thermoplastic cellulose biop...   \n",
       "16152        Transmission method with double directivity   \n",
       "\n",
       "                                                abstract  \\\n",
       "0      The present invention relates to passive optic...   \n",
       "1      Embodiments of the invention provide a method ...   \n",
       "2      A portable drug dispenser includes a chamber f...   \n",
       "3      A crystal growth furnace comprising a crucible...   \n",
       "4      A shoe midsole is composed of a base plate (1)...   \n",
       "...                                                  ...   \n",
       "16148  A robot hand controlling method executes calcu...   \n",
       "16149  A fusion protein is disclosed. The fusion prot...   \n",
       "16150  A pipe extraction tool that grips the inside o...   \n",
       "16151  A longitudinal extending body with oriented fi...   \n",
       "16152  A transmission method using a massive MIMO (Mu...   \n",
       "\n",
       "                                                  claims  \\\n",
       "0      1. A compact optical network terminal, compris...   \n",
       "1      1. A method comprising: using a first reader t...   \n",
       "2      1. A portable drug dispenser, comprising: a ch...   \n",
       "3      1. A crystal growth furnace for growing a crys...   \n",
       "4      1. A sole member of footwear comprising a base...   \n",
       "...                                                  ...   \n",
       "16148  1. A controlling method of a robot hand, the r...   \n",
       "16149  1. A fusion protein comprising an Fc fragment ...   \n",
       "16150  1. A pipe extraction tool for extracting a pip...   \n",
       "16151  1. A longitudinal body of a solidified organic...   \n",
       "16152  1. Transmission method with double directivity...   \n",
       "\n",
       "                                              background  \\\n",
       "0      <SOH> BACKGROUND OF THE INVENTION <EOH>A netwo...   \n",
       "1      <SOH> BACKGROUND OF THE INVENTION <EOH>Identif...   \n",
       "2                                                          \n",
       "3      <SOH> BACKGROUND OF THE INVENTION <EOH>1. Fiel...   \n",
       "4      <SOH> BACKGROUND ART <EOH>When the heel touche...   \n",
       "...                                                  ...   \n",
       "16148  <SOH> BACKGROUND OF THE INVENTION <EOH>1. Fiel...   \n",
       "16149  <SOH> BACKGROUND OF THE INVENTION <EOH>An immu...   \n",
       "16150  <SOH> BACKGROUND OF THE INVENTION <EOH>1. Fiel...   \n",
       "16151  <SOH> BACKGROUND OF INVENTION <EOH>In the medi...   \n",
       "16152            <SOH> BACKGROUND OF THE INVENTION <EOH>   \n",
       "\n",
       "                                                 summary  \\\n",
       "0      <SOH> SUMMARY OF THE INVENTION <EOH>An aspect ...   \n",
       "1      <SOH> SUMMARY OF THE INVENTION <EOH>In accorda...   \n",
       "2                                                          \n",
       "3      <SOH> SUMMARY OF THE INVENTION <EOH>The presen...   \n",
       "4      <SOH> BRIEF DESCRIPTION OF THE DRAWINGS <EOH>F...   \n",
       "...                                                  ...   \n",
       "16148  <SOH> SUMMARY OF THE INVENTION <EOH>An object ...   \n",
       "16149  <SOH> SUMMARY OF THE INVENTION <EOH>The presen...   \n",
       "16150  <SOH> BRIEF SUMMARY OF THE INVENTION <EOH>The ...   \n",
       "16151  <SOH> BRIEF SUMMARY OF THE PRESENT INVENTION <...   \n",
       "16152  <SOH> BRIEF SUMMARY OF THE INVENTION <EOH>The ...   \n",
       "\n",
       "                                             description   cpc_label  \\\n",
       "0      FIELD OF THE INVENTION The present invention r...  H04Q110071   \n",
       "1      CROSS-REFERENCE TO RELATED APPLICATIONS The pr...     G06K500   \n",
       "2      This application claims priority from U.S. app...   A61J70084   \n",
       "3      CROSS-REFERENCE TO RELATED APPLICATIONS The pr...   C30B11003   \n",
       "4      TECHNICAL FIELD The present invention relates ...   A43B13181   \n",
       "...                                                  ...         ...   \n",
       "16148  BACKGROUND OF THE INVENTION 1. Field of the In...   B25J91612   \n",
       "16149  The present application is a U.S. Nonprovision...   C07K14745   \n",
       "16150  CROSS-REFERENCES TO RELATED APPLICATIONS Not a...    B25B2714   \n",
       "16151  CROSS REFERENCES Application claims priority o...    A61L3106   \n",
       "16152  BACKGROUND OF THE INVENTION Field of the Inven...    H04B7043   \n",
       "\n",
       "       ipc_label filing_date patent_issue_date date_published examiner_id  \n",
       "0       H04Q1100    20160120          20170606       20160526     95191.0  \n",
       "1        G06K500    20160112          20160322       20140102     59514.0  \n",
       "2        A61J700    20160104                         20171116     95928.0  \n",
       "3       C30B1100    20160111          20180529       20160512     63013.0  \n",
       "4       A43B1318    20160113                         20160512     94490.0  \n",
       "...          ...         ...               ...            ...         ...  \n",
       "16148    B25J916    20160120          20180710       20160804     66148.0  \n",
       "16149  C07K14745    20160120                         20161215     95819.0  \n",
       "16150   B25B2714    20160120                         20170907     95661.0  \n",
       "16151   A61L3106    20160111                         20171019     96956.0  \n",
       "16152    H04B704    20160114                         20180329     70883.0  \n",
       "\n",
       "[16153 rows x 14 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-Processing the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are interested in the following columns:\n",
    "- Abstract\n",
    "- Claims\n",
    "- Decision <- our `y`\n",
    "\n",
    "Let's preprocess them both out of our training and validation data\n",
    "\n",
    "Also, consider that the \"Decision\" column has three types of values: \"Accepted\", \"Rejected\", and \"Pending\". To remove unecessary baggage, we will be only looking for \"Accepted\" and \"Rejected\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "necessary_columns = [\"abstract\",\"claims\",\"decision\"]\n",
    "output_values = ['ACCEPTED','REJECTED'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainFeaturesToDrop = [col for col in list(df_train.columns) if col not in necessary_columns]\n",
    "trainDF = df_train.dropna()\n",
    "trainDF.drop(columns=trainFeaturesToDrop, inplace=True)\n",
    "trainDF = trainDF[trainDF['decision'].isin(output_values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>decision</th>\n",
       "      <th>abstract</th>\n",
       "      <th>claims</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>The present invention relates to passive optic...</td>\n",
       "      <td>1. A compact optical network terminal, compris...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>Embodiments of the invention provide a method ...</td>\n",
       "      <td>1. A method comprising: using a first reader t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
       "      <td>1. A crystal growth furnace for growing a crys...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
       "      <td>1. A sole member of footwear comprising a base...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A ratchet tool includes a shaft member, a hand...</td>\n",
       "      <td>1. A ratchet tool, comprising a shaft member, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16144</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A wavelength tunable laser device, including: ...</td>\n",
       "      <td>1. A wavelength tunable laser device, comprisi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16145</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>In one aspect, a method for use in preparing a...</td>\n",
       "      <td>1. (canceled) 2. The method of claim 19, where...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16148</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A robot hand controlling method executes calcu...</td>\n",
       "      <td>1. A controlling method of a robot hand, the r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16149</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
       "      <td>1. A fusion protein comprising an Fc fragment ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16150</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
       "      <td>1. A pipe extraction tool for extracting a pip...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8719 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       decision                                           abstract  \\\n",
       "0      ACCEPTED  The present invention relates to passive optic...   \n",
       "1      ACCEPTED  Embodiments of the invention provide a method ...   \n",
       "3      ACCEPTED  A crystal growth furnace comprising a crucible...   \n",
       "4      REJECTED  A shoe midsole is composed of a base plate (1)...   \n",
       "5      ACCEPTED  A ratchet tool includes a shaft member, a hand...   \n",
       "...         ...                                                ...   \n",
       "16144  ACCEPTED  A wavelength tunable laser device, including: ...   \n",
       "16145  ACCEPTED  In one aspect, a method for use in preparing a...   \n",
       "16148  ACCEPTED  A robot hand controlling method executes calcu...   \n",
       "16149  REJECTED  A fusion protein is disclosed. The fusion prot...   \n",
       "16150  REJECTED  A pipe extraction tool that grips the inside o...   \n",
       "\n",
       "                                                  claims  \n",
       "0      1. A compact optical network terminal, compris...  \n",
       "1      1. A method comprising: using a first reader t...  \n",
       "3      1. A crystal growth furnace for growing a crys...  \n",
       "4      1. A sole member of footwear comprising a base...  \n",
       "5      1. A ratchet tool, comprising a shaft member, ...  \n",
       "...                                                  ...  \n",
       "16144  1. A wavelength tunable laser device, comprisi...  \n",
       "16145  1. (canceled) 2. The method of claim 19, where...  \n",
       "16148  1. A controlling method of a robot hand, the r...  \n",
       "16149  1. A fusion protein comprising an Fc fragment ...  \n",
       "16150  1. A pipe extraction tool for extracting a pip...  \n",
       "\n",
       "[8719 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "valFeaturesToDrop = [col for col in list(df_val.columns) if col not in necessary_columns]\n",
    "valDF = df_val.dropna()\n",
    "valDF.drop(columns=valFeaturesToDrop, inplace=True)\n",
    "valDF = valDF[valDF['decision'].isin(output_values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>decision</th>\n",
       "      <th>abstract</th>\n",
       "      <th>claims</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>Regimen for the treatment of rosacea include t...</td>\n",
       "      <td>1. A treatment regimen comprising: cleansing a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A clamp arrangement includes a pair of bracket...</td>\n",
       "      <td>1. A clamp arrangement for supporting a fractu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>A system and method for device action and conf...</td>\n",
       "      <td>1-20. (canceled) 21. A mobile device comprisin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>Systems and methods for managing datasets prod...</td>\n",
       "      <td>1. A method, comprising: executing, by one or ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A scan driving circuit is provided. The scan d...</td>\n",
       "      <td>1. A scan driving circuit for driving a scan l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9085</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>The non-rigid gate device as described may be ...</td>\n",
       "      <td>1; A non-rigid blocking apparatus referred to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9090</th>\n",
       "      <td>REJECTED</td>\n",
       "      <td>The present invention provides an improved unc...</td>\n",
       "      <td>1. A method for rendering a plastic surface am...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9091</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A method for detecting a software-race conditi...</td>\n",
       "      <td>1. A method for detecting a software-race cond...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9092</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>The present application relates to multi-stage...</td>\n",
       "      <td>1. A multi-stage amplitude modulation-based me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9093</th>\n",
       "      <td>ACCEPTED</td>\n",
       "      <td>A paper feeder includes a housing, a driving u...</td>\n",
       "      <td>1. A paper feeder, comprising: a housing; a dr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4888 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      decision                                           abstract  \\\n",
       "0     REJECTED  Regimen for the treatment of rosacea include t...   \n",
       "1     ACCEPTED  A clamp arrangement includes a pair of bracket...   \n",
       "2     REJECTED  A system and method for device action and conf...   \n",
       "4     REJECTED  Systems and methods for managing datasets prod...   \n",
       "9     ACCEPTED  A scan driving circuit is provided. The scan d...   \n",
       "...        ...                                                ...   \n",
       "9085  REJECTED  The non-rigid gate device as described may be ...   \n",
       "9090  REJECTED  The present invention provides an improved unc...   \n",
       "9091  ACCEPTED  A method for detecting a software-race conditi...   \n",
       "9092  ACCEPTED  The present application relates to multi-stage...   \n",
       "9093  ACCEPTED  A paper feeder includes a housing, a driving u...   \n",
       "\n",
       "                                                 claims  \n",
       "0     1. A treatment regimen comprising: cleansing a...  \n",
       "1     1. A clamp arrangement for supporting a fractu...  \n",
       "2     1-20. (canceled) 21. A mobile device comprisin...  \n",
       "4     1. A method, comprising: executing, by one or ...  \n",
       "9     1. A scan driving circuit for driving a scan l...  \n",
       "...                                                 ...  \n",
       "9085  1; A non-rigid blocking apparatus referred to ...  \n",
       "9090  1. A method for rendering a plastic surface am...  \n",
       "9091  1. A method for detecting a software-race cond...  \n",
       "9092  1. A multi-stage amplitude modulation-based me...  \n",
       "9093  1. A paper feeder, comprising: a housing; a dr...  \n",
       "\n",
       "[4888 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to replace the values in the `decision` column to numerical representations. We will set \"ACCEPTED\" as `1` and \"REJECTED\" as `0`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "yKey = {\"ACCEPTED\":1,\"REJECTED\":0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainDF2 = trainDF.replace({\"decision\": yKey})\n",
    "valDF2 = valDF.replace({\"decision\": yKey})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>decision</th>\n",
       "      <th>abstract</th>\n",
       "      <th>claims</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>The present invention relates to passive optic...</td>\n",
       "      <td>1. A compact optical network terminal, compris...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Embodiments of the invention provide a method ...</td>\n",
       "      <td>1. A method comprising: using a first reader t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
       "      <td>1. A crystal growth furnace for growing a crys...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
       "      <td>1. A sole member of footwear comprising a base...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>A ratchet tool includes a shaft member, a hand...</td>\n",
       "      <td>1. A ratchet tool, comprising a shaft member, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16144</th>\n",
       "      <td>1</td>\n",
       "      <td>A wavelength tunable laser device, including: ...</td>\n",
       "      <td>1. A wavelength tunable laser device, comprisi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16145</th>\n",
       "      <td>1</td>\n",
       "      <td>In one aspect, a method for use in preparing a...</td>\n",
       "      <td>1. (canceled) 2. The method of claim 19, where...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16148</th>\n",
       "      <td>1</td>\n",
       "      <td>A robot hand controlling method executes calcu...</td>\n",
       "      <td>1. A controlling method of a robot hand, the r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16149</th>\n",
       "      <td>0</td>\n",
       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
       "      <td>1. A fusion protein comprising an Fc fragment ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16150</th>\n",
       "      <td>0</td>\n",
       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
       "      <td>1. A pipe extraction tool for extracting a pip...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8719 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       decision                                           abstract  \\\n",
       "0             1  The present invention relates to passive optic...   \n",
       "1             1  Embodiments of the invention provide a method ...   \n",
       "3             1  A crystal growth furnace comprising a crucible...   \n",
       "4             0  A shoe midsole is composed of a base plate (1)...   \n",
       "5             1  A ratchet tool includes a shaft member, a hand...   \n",
       "...         ...                                                ...   \n",
       "16144         1  A wavelength tunable laser device, including: ...   \n",
       "16145         1  In one aspect, a method for use in preparing a...   \n",
       "16148         1  A robot hand controlling method executes calcu...   \n",
       "16149         0  A fusion protein is disclosed. The fusion prot...   \n",
       "16150         0  A pipe extraction tool that grips the inside o...   \n",
       "\n",
       "                                                  claims  \n",
       "0      1. A compact optical network terminal, compris...  \n",
       "1      1. A method comprising: using a first reader t...  \n",
       "3      1. A crystal growth furnace for growing a crys...  \n",
       "4      1. A sole member of footwear comprising a base...  \n",
       "5      1. A ratchet tool, comprising a shaft member, ...  \n",
       "...                                                  ...  \n",
       "16144  1. A wavelength tunable laser device, comprisi...  \n",
       "16145  1. (canceled) 2. The method of claim 19, where...  \n",
       "16148  1. A controlling method of a robot hand, the r...  \n",
       "16149  1. A fusion protein comprising an Fc fragment ...  \n",
       "16150  1. A pipe extraction tool for extracting a pip...  \n",
       "\n",
       "[8719 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainDF2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>decision</th>\n",
       "      <th>abstract</th>\n",
       "      <th>claims</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Regimen for the treatment of rosacea include t...</td>\n",
       "      <td>1. A treatment regimen comprising: cleansing a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A clamp arrangement includes a pair of bracket...</td>\n",
       "      <td>1. A clamp arrangement for supporting a fractu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>A system and method for device action and conf...</td>\n",
       "      <td>1-20. (canceled) 21. A mobile device comprisin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>Systems and methods for managing datasets prod...</td>\n",
       "      <td>1. A method, comprising: executing, by one or ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>A scan driving circuit is provided. The scan d...</td>\n",
       "      <td>1. A scan driving circuit for driving a scan l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9085</th>\n",
       "      <td>0</td>\n",
       "      <td>The non-rigid gate device as described may be ...</td>\n",
       "      <td>1; A non-rigid blocking apparatus referred to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9090</th>\n",
       "      <td>0</td>\n",
       "      <td>The present invention provides an improved unc...</td>\n",
       "      <td>1. A method for rendering a plastic surface am...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9091</th>\n",
       "      <td>1</td>\n",
       "      <td>A method for detecting a software-race conditi...</td>\n",
       "      <td>1. A method for detecting a software-race cond...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9092</th>\n",
       "      <td>1</td>\n",
       "      <td>The present application relates to multi-stage...</td>\n",
       "      <td>1. A multi-stage amplitude modulation-based me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9093</th>\n",
       "      <td>1</td>\n",
       "      <td>A paper feeder includes a housing, a driving u...</td>\n",
       "      <td>1. A paper feeder, comprising: a housing; a dr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4888 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      decision                                           abstract  \\\n",
       "0            0  Regimen for the treatment of rosacea include t...   \n",
       "1            1  A clamp arrangement includes a pair of bracket...   \n",
       "2            0  A system and method for device action and conf...   \n",
       "4            0  Systems and methods for managing datasets prod...   \n",
       "9            1  A scan driving circuit is provided. The scan d...   \n",
       "...        ...                                                ...   \n",
       "9085         0  The non-rigid gate device as described may be ...   \n",
       "9090         0  The present invention provides an improved unc...   \n",
       "9091         1  A method for detecting a software-race conditi...   \n",
       "9092         1  The present application relates to multi-stage...   \n",
       "9093         1  A paper feeder includes a housing, a driving u...   \n",
       "\n",
       "                                                 claims  \n",
       "0     1. A treatment regimen comprising: cleansing a...  \n",
       "1     1. A clamp arrangement for supporting a fractu...  \n",
       "2     1-20. (canceled) 21. A mobile device comprisin...  \n",
       "4     1. A method, comprising: executing, by one or ...  \n",
       "9     1. A scan driving circuit for driving a scan l...  \n",
       "...                                                 ...  \n",
       "9085  1; A non-rigid blocking apparatus referred to ...  \n",
       "9090  1. A method for rendering a plastic surface am...  \n",
       "9091  1. A method for detecting a software-race cond...  \n",
       "9092  1. A multi-stage amplitude modulation-based me...  \n",
       "9093  1. A paper feeder, comprising: a housing; a dr...  \n",
       "\n",
       "[4888 rows x 3 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valDF2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We combine the `abstract` and `claims` columns into a single `text` column. We also re-label the `decision` column to `label`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>The present invention relates to passive optic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Embodiments of the invention provide a method ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>A crystal growth furnace comprising a crucible...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>A shoe midsole is composed of a base plate (1)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>A ratchet tool includes a shaft member, a hand...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16144</th>\n",
       "      <td>1</td>\n",
       "      <td>A wavelength tunable laser device, including: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16145</th>\n",
       "      <td>1</td>\n",
       "      <td>In one aspect, a method for use in preparing a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16148</th>\n",
       "      <td>1</td>\n",
       "      <td>A robot hand controlling method executes calcu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16149</th>\n",
       "      <td>0</td>\n",
       "      <td>A fusion protein is disclosed. The fusion prot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16150</th>\n",
       "      <td>0</td>\n",
       "      <td>A pipe extraction tool that grips the inside o...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8719 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       label                                               text\n",
       "0          1  The present invention relates to passive optic...\n",
       "1          1  Embodiments of the invention provide a method ...\n",
       "3          1  A crystal growth furnace comprising a crucible...\n",
       "4          0  A shoe midsole is composed of a base plate (1)...\n",
       "5          1  A ratchet tool includes a shaft member, a hand...\n",
       "...      ...                                                ...\n",
       "16144      1  A wavelength tunable laser device, including: ...\n",
       "16145      1  In one aspect, a method for use in preparing a...\n",
       "16148      1  A robot hand controlling method executes calcu...\n",
       "16149      0  A fusion protein is disclosed. The fusion prot...\n",
       "16150      0  A pipe extraction tool that grips the inside o...\n",
       "\n",
       "[8719 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainDF3 = trainDF2.rename(columns={'decision': 'label'})\n",
    "trainDF3['text'] = trainDF3['abstract'] + ' ' + trainDF3['claims']\n",
    "trainDF3.drop(columns=[\"abstract\",\"claims\"],inplace=True)\n",
    "trainDF3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Regimen for the treatment of rosacea include t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A clamp arrangement includes a pair of bracket...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>A system and method for device action and conf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>Systems and methods for managing datasets prod...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>A scan driving circuit is provided. The scan d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9085</th>\n",
       "      <td>0</td>\n",
       "      <td>The non-rigid gate device as described may be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9090</th>\n",
       "      <td>0</td>\n",
       "      <td>The present invention provides an improved unc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9091</th>\n",
       "      <td>1</td>\n",
       "      <td>A method for detecting a software-race conditi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9092</th>\n",
       "      <td>1</td>\n",
       "      <td>The present application relates to multi-stage...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9093</th>\n",
       "      <td>1</td>\n",
       "      <td>A paper feeder includes a housing, a driving u...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4888 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                               text\n",
       "0         0  Regimen for the treatment of rosacea include t...\n",
       "1         1  A clamp arrangement includes a pair of bracket...\n",
       "2         0  A system and method for device action and conf...\n",
       "4         0  Systems and methods for managing datasets prod...\n",
       "9         1  A scan driving circuit is provided. The scan d...\n",
       "...     ...                                                ...\n",
       "9085      0  The non-rigid gate device as described may be ...\n",
       "9090      0  The present invention provides an improved unc...\n",
       "9091      1  A method for detecting a software-race conditi...\n",
       "9092      1  The present application relates to multi-stage...\n",
       "9093      1  A paper feeder includes a housing, a driving u...\n",
       "\n",
       "[4888 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valDF3 = valDF2.rename(columns={'decision': 'label'})\n",
    "valDF3['text'] = valDF3['abstract'] + ' ' + valDF3['claims']\n",
    "valDF3.drop(columns=[\"abstract\",\"claims\"],inplace=True)\n",
    "valDF3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can grab the data for each column so that we have a list of values for training labels, training texts, validation labels, and validation texts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainLabels = trainDF3[\"label\"].tolist()\n",
    "trainText = trainDF3[\"text\"].tolist()\n",
    "\n",
    "valLabels = valDF3[\"label\"].tolist()\n",
    "valText = valDF3[\"text\"].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Trainer\n",
    "\n",
    "Now we can start training! This time, we will just go with `distilbert-base-uncased` for simplicity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: torch in /opt/conda/lib/python3.10/site-packages (2.0.0)\n",
      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.4.91)\n",
      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.91)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch) (3.1.2)\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch) (4.4.0)\n",
      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /opt/conda/lib/python3.10/site-packages (from torch) (10.2.10.91)\n",
      "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /opt/conda/lib/python3.10/site-packages (from torch) (11.4.0.1)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /opt/conda/lib/python3.10/site-packages (from torch) (11.10.3.66)\n",
      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /opt/conda/lib/python3.10/site-packages (from torch) (10.9.0.58)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch) (1.11.1)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.99)\n",
      "Requirement already satisfied: triton==2.0.0 in /opt/conda/lib/python3.10/site-packages (from torch) (2.0.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch) (3.12.0)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch) (2.8.7)\n",
      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /opt/conda/lib/python3.10/site-packages (from torch) (2.14.3)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.101)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /opt/conda/lib/python3.10/site-packages (from torch) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /opt/conda/lib/python3.10/site-packages (from torch) (8.5.0.96)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (65.4.1)\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (0.37.1)\n",
      "Requirement already satisfied: cmake in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch) (3.26.3)\n",
      "Requirement already satisfied: lit in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch) (16.0.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch) (2.1.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch) (1.2.1)\n",
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.28.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.4)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (1.23.3)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.12.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.3.23)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers) (4.64.1)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers) (2.28.1)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.13.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers) (3.0.9)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (1.26.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2022.9.24)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (3.4)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers) (2.1.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install torch\n",
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n",
    "from transformers import Trainer, TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"distilbert-base-uncased\"\n",
    "class USPTODataset(Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encoding.items()}\n",
    "        item['labels'] = torch.tensor(self.labels[idx])\n",
    "        return item\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_encodings = tokenizer(trainText, truncation=True, padding=True)\n",
    "val_encodings = tokenizer(valText, truncation=True, padding=True)\n",
    "\n",
    "train_dataset = USPTODataset(train_encodings, trainLabels)\n",
    "val_dataset = USPTODataset(val_encodings, valLabels)\n",
    "\n",
    "train_args = TrainingArguments(\n",
    "    output_dir=\"./results\",\n",
    "    num_train_epochs=2,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=64,\n",
    "    warmup_steps=500,\n",
    "    learning_rate=5e-5,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir=\"./logs\",\n",
    "    logging_steps=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}