Added documentation, website details to readme.

.ipynb_checkpoints/finetunehupd-checkpoint.ipynb

@@ -2,29 +2,1225 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "1df3c609-62a6-49c3-bcc6-29c520f9501c",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# -*- coding: utf-8 -*-\n",
-    "\"\"\"FinetuneHUPD.ipynb\n",
-    "\n",
-    "Automatically generated by Colaboratory.\n",
-    "\n",
-    "Original file is located at\n",
-    "    https://colab.research.google.com/drive/17c2CQZx_kyD3-0fuQqv_pCMJ0Evd7fLN\n",
-    "\"\"\"\n",
-    "\n",
     "# Pretty print\n",
     "from pprint import pprint\n",
     "# Datasets load_dataset function\n",
     "from datasets import load_dataset\n",
     "# Transformers Autokenizer\n",
-    "from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, Trainer, TrainingArguments, AdamW\n",
+    "from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertTokenizerFast, Trainer, TrainingArguments, AdamW\n",
     "from torch.utils.data import DataLoader\n",
     "import torch"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "58167c28-eb27-4f82-b7d0-8216dbeaf650",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Found cached dataset hupd (C:/Users/calia/.cache/huggingface/datasets/HUPD___hupd/sample-5094df4de61ed3bc/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a2f090474cb148548ce3eb73698fcc6c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading is done!\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Use the tokenizer for DistilBert since we are using the pretrained base uncased model \n",
+    "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
+    "\n",
+    "# Load Harvard USPTO Patents Dataset from January 2-16 dataset using the load_dataset function\n",
+    "# Split the data into a train and validation set. For this purpose, I have split the patents as follows:\n",
+    "# train: Jan 1 - Jan 22, val: Jan 22 - Jan 31.\n",
+    "dataset_dict = load_dataset('HUPD/hupd',\n",
+    "    name='sample',\n",
+    "    data_files=\"https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather\", \n",
+    "    icpr_label=None,\n",
+    "    train_filing_start_date='2016-01-01',\n",
+    "    train_filing_end_date='2016-01-21',\n",
+    "    val_filing_start_date='2016-01-22',\n",
+    "    val_filing_end_date='2016-01-31',\n",
+    ")\n",
+    "\n",
+    "print('Loading is done!')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "e13c6ad1-a7f2-4806-80a2-e9c4655e1eed",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-9f7788eb9924fd62.arrow\n",
+      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-6c3687322fe5b556.arrow\n",
+      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-bd3b1eee4495f3ce.arrow\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/9094 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Label-to-index mapping for the decision status field\n",
+    "# Convert to binary, as we only care about whether or not a patent was rejected or accepted.\n",
+    "decision_to_str = {'REJECTED': 0, 'ACCEPTED': 1, 'PENDING': 0, 'CONT-REJECTED': 0, 'CONT-ACCEPTED': 0, 'CONT-PENDING': 0}\n",
+    "\n",
+    "# Helper function\n",
+    "def map_decision_to_string(example):\n",
+    "    return {'decision': decision_to_str[example['decision']]}\n",
+    "\n",
+    "# Re-labeling/mapping.\n",
+    "train_set = dataset_dict['train'].map(map_decision_to_string)\n",
+    "val_set = dataset_dict['validation'].map(map_decision_to_string)\n",
+    "\n",
+    "# Focus on the abstract section and tokenize the text using the tokenizer. \n",
+    "_SECTION_ = 'abstract'\n",
+    "\n",
+    "# Training set\n",
+    "train_set = train_set.map(\n",
+    "    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),\n",
+    "    batched=True)\n",
+    "\n",
+    "# Validation set\n",
+    "val_set = val_set.map(\n",
+    "    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),\n",
+    "    batched=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b5c098be-019b-42ce-9b80-4f6de93ef6a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 16153\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1e5a5390-19fe-4a73-b913-e3c1e2c2a399",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 9094\n",
+       "})"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4fb69db8-86e5-4c6c-8ac6-853d3e15fb93",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Process the train and validation sets so that only labels, input ids, and attention masks are left.\n",
+    "train_set = train_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
+    "val_set = val_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
+    "\n",
+    "train_set = train_set.rename_column(\"decision\", \"labels\")\n",
+    "val_set = val_set.rename_column(\"decision\", \"labels\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c0d17213-4b14-418c-980c-0238236096c2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['labels', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 16153\n",
+       "})"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "da2f1c16-3ba4-4e56-9455-5cd838df4dcd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['labels', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 9094\n",
+       "})"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "cfb35702-863d-4fec-83e1-44c4e5668156",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set the format to 'torch'\n",
+    "train_set.set_format(type='torch', \n",
+    "    columns=['labels', 'input_ids', 'attention_mask'])\n",
+    "\n",
+    "val_set.set_format(type='torch', \n",
+    "    columns=['labels', 'input_ids', 'attention_mask'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "b3248182-fddb-46dc-addb-26981a881d99",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_layer_norm.bias']\n",
+      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'pre_classifier.weight', 'classifier.bias', 'classifier.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "cuda\n",
+      "torch cuda is avail: \n",
+      "True\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Use GPU acceleration. Make sure you have the right cuda and pytorch versions!\n",
+    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
+    "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')\n",
+    "model.to(device)\n",
+    "print(device)\n",
+    "print(\"torch cuda is avail: \")\n",
+    "print(torch.cuda.is_available())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "abb2cf74-3cd5-4ca5-af0e-b0ee80627f2a",
+   "metadata": {},
+   "source": [
+    "HuggingFace Trainer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "99947cf9-a6cd-490f-a81d-32f65fb3cd46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "training_args = TrainingArguments(\n",
+    "    output_dir='./results/',\n",
+    "    num_train_epochs=2,\n",
+    "    per_device_train_batch_size=16,\n",
+    "    per_device_eval_batch_size=16,\n",
+    "    warmup_steps=500,\n",
+    "    learning_rate=5e-5,\n",
+    "    weight_decay=0.01,\n",
+    "    logging_dir='./logs/',\n",
+    "    logging_steps=10,\n",
+    ")\n",
+    "\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=train_set,\n",
+    "    eval_dataset=val_set,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "be865f1d-f29b-4306-8570-900386ac4570",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\calia\\anaconda3\\envs\\ai-finetuning-project\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+      "  warnings.warn(\n",
+      "***** Running training *****\n",
+      "  Num examples = 16153\n",
+      "  Num Epochs = 2\n",
+      "  Instantaneous batch size per device = 16\n",
+      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
+      "  Gradient Accumulation steps = 1\n",
+      "  Total optimization steps = 2020\n",
+      "  Number of trainable parameters = 66955010\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='2020' max='2020' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [2020/2020 11:47, Epoch 2/2]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>0.692000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>0.685100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>0.684000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>0.685100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>50</td>\n",
+       "      <td>0.678400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>60</td>\n",
+       "      <td>0.687300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>70</td>\n",
+       "      <td>0.681900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>80</td>\n",
+       "      <td>0.691100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>90</td>\n",
+       "      <td>0.683200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>100</td>\n",
+       "      <td>0.694100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>110</td>\n",
+       "      <td>0.673300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>120</td>\n",
+       "      <td>0.694100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>130</td>\n",
+       "      <td>0.669500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>140</td>\n",
+       "      <td>0.691100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>150</td>\n",
+       "      <td>0.683400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>160</td>\n",
+       "      <td>0.654900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>170</td>\n",
+       "      <td>0.684300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>180</td>\n",
+       "      <td>0.679300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>190</td>\n",
+       "      <td>0.662600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>200</td>\n",
+       "      <td>0.598400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>210</td>\n",
+       "      <td>0.717700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>220</td>\n",
+       "      <td>0.679100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>230</td>\n",
+       "      <td>0.677500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>240</td>\n",
+       "      <td>0.668800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>250</td>\n",
+       "      <td>0.678100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>260</td>\n",
+       "      <td>0.657500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>270</td>\n",
+       "      <td>0.707200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>280</td>\n",
+       "      <td>0.670300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>290</td>\n",
+       "      <td>0.659900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>300</td>\n",
+       "      <td>0.633300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>310</td>\n",
+       "      <td>0.676300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>320</td>\n",
+       "      <td>0.684800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>330</td>\n",
+       "      <td>0.673100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>340</td>\n",
+       "      <td>0.670500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>350</td>\n",
+       "      <td>0.657500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>360</td>\n",
+       "      <td>0.618100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>370</td>\n",
+       "      <td>0.670000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>380</td>\n",
+       "      <td>0.607400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>390</td>\n",
+       "      <td>0.656200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>400</td>\n",
+       "      <td>0.700000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>410</td>\n",
+       "      <td>0.644800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>420</td>\n",
+       "      <td>0.682800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>430</td>\n",
+       "      <td>0.668800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>440</td>\n",
+       "      <td>0.662600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>450</td>\n",
+       "      <td>0.647700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>460</td>\n",
+       "      <td>0.688600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>470</td>\n",
+       "      <td>0.682400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>480</td>\n",
+       "      <td>0.642900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>490</td>\n",
+       "      <td>0.726900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>500</td>\n",
+       "      <td>0.660400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>510</td>\n",
+       "      <td>0.649500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>520</td>\n",
+       "      <td>0.637200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>530</td>\n",
+       "      <td>0.669700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>540</td>\n",
+       "      <td>0.667100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>550</td>\n",
+       "      <td>0.617000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>560</td>\n",
+       "      <td>0.725300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>570</td>\n",
+       "      <td>0.656800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>580</td>\n",
+       "      <td>0.664600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>590</td>\n",
+       "      <td>0.702600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>600</td>\n",
+       "      <td>0.686300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>610</td>\n",
+       "      <td>0.668400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>620</td>\n",
+       "      <td>0.648200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>630</td>\n",
+       "      <td>0.628700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>640</td>\n",
+       "      <td>0.676700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>650</td>\n",
+       "      <td>0.652400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>660</td>\n",
+       "      <td>0.654300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>670</td>\n",
+       "      <td>0.640800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>680</td>\n",
+       "      <td>0.672000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>690</td>\n",
+       "      <td>0.636100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>700</td>\n",
+       "      <td>0.689100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>710</td>\n",
+       "      <td>0.691100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>720</td>\n",
+       "      <td>0.650300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>730</td>\n",
+       "      <td>0.655200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>740</td>\n",
+       "      <td>0.668400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>750</td>\n",
+       "      <td>0.659200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>760</td>\n",
+       "      <td>0.647800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>770</td>\n",
+       "      <td>0.662800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>780</td>\n",
+       "      <td>0.648500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>790</td>\n",
+       "      <td>0.656700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>800</td>\n",
+       "      <td>0.669400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>810</td>\n",
+       "      <td>0.607800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>820</td>\n",
+       "      <td>0.683200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>830</td>\n",
+       "      <td>0.663800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>840</td>\n",
+       "      <td>0.700900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>850</td>\n",
+       "      <td>0.648200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>860</td>\n",
+       "      <td>0.619400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>870</td>\n",
+       "      <td>0.649200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>880</td>\n",
+       "      <td>0.717500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>890</td>\n",
+       "      <td>0.669600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>900</td>\n",
+       "      <td>0.669700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>910</td>\n",
+       "      <td>0.683900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>920</td>\n",
+       "      <td>0.636900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>930</td>\n",
+       "      <td>0.656400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>940</td>\n",
+       "      <td>0.650000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>950</td>\n",
+       "      <td>0.617800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>960</td>\n",
+       "      <td>0.665600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>970</td>\n",
+       "      <td>0.642700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>980</td>\n",
+       "      <td>0.644000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>990</td>\n",
+       "      <td>0.688900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.654700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1010</td>\n",
+       "      <td>0.645800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1020</td>\n",
+       "      <td>0.609200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1030</td>\n",
+       "      <td>0.602300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1040</td>\n",
+       "      <td>0.618800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1050</td>\n",
+       "      <td>0.643500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1060</td>\n",
+       "      <td>0.611000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1070</td>\n",
+       "      <td>0.645000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1080</td>\n",
+       "      <td>0.641000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1090</td>\n",
+       "      <td>0.595400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1100</td>\n",
+       "      <td>0.635100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1110</td>\n",
+       "      <td>0.611600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1120</td>\n",
+       "      <td>0.600300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1130</td>\n",
+       "      <td>0.618100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1140</td>\n",
+       "      <td>0.617200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1150</td>\n",
+       "      <td>0.633400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1160</td>\n",
+       "      <td>0.597600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1170</td>\n",
+       "      <td>0.619400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1180</td>\n",
+       "      <td>0.584200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1190</td>\n",
+       "      <td>0.600700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1200</td>\n",
+       "      <td>0.657400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1210</td>\n",
+       "      <td>0.569600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1220</td>\n",
+       "      <td>0.575500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1230</td>\n",
+       "      <td>0.617900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1240</td>\n",
+       "      <td>0.610300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1250</td>\n",
+       "      <td>0.570600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1260</td>\n",
+       "      <td>0.545700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1270</td>\n",
+       "      <td>0.656300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1280</td>\n",
+       "      <td>0.554700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1290</td>\n",
+       "      <td>0.598200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1300</td>\n",
+       "      <td>0.606300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1310</td>\n",
+       "      <td>0.600500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1320</td>\n",
+       "      <td>0.569800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1330</td>\n",
+       "      <td>0.604700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1340</td>\n",
+       "      <td>0.628300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1350</td>\n",
+       "      <td>0.602700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1360</td>\n",
+       "      <td>0.583700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1370</td>\n",
+       "      <td>0.623800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1380</td>\n",
+       "      <td>0.670300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1390</td>\n",
+       "      <td>0.622400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1400</td>\n",
+       "      <td>0.590200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1410</td>\n",
+       "      <td>0.587000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1420</td>\n",
+       "      <td>0.555500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1430</td>\n",
+       "      <td>0.561000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1440</td>\n",
+       "      <td>0.514300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1450</td>\n",
+       "      <td>0.553100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1460</td>\n",
+       "      <td>0.692400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1470</td>\n",
+       "      <td>0.605200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1480</td>\n",
+       "      <td>0.548000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1490</td>\n",
+       "      <td>0.672600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1500</td>\n",
+       "      <td>0.531100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1510</td>\n",
+       "      <td>0.610600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1520</td>\n",
+       "      <td>0.580200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1530</td>\n",
+       "      <td>0.571300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1540</td>\n",
+       "      <td>0.644400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1550</td>\n",
+       "      <td>0.558500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1560</td>\n",
+       "      <td>0.624000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1570</td>\n",
+       "      <td>0.659200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1580</td>\n",
+       "      <td>0.580500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1590</td>\n",
+       "      <td>0.649900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1600</td>\n",
+       "      <td>0.608700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1610</td>\n",
+       "      <td>0.595100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1620</td>\n",
+       "      <td>0.592900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1630</td>\n",
+       "      <td>0.584000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1640</td>\n",
+       "      <td>0.607100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1650</td>\n",
+       "      <td>0.565800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1660</td>\n",
+       "      <td>0.568300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1670</td>\n",
+       "      <td>0.572200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1680</td>\n",
+       "      <td>0.597500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1690</td>\n",
+       "      <td>0.602700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1700</td>\n",
+       "      <td>0.692900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1710</td>\n",
+       "      <td>0.597900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1720</td>\n",
+       "      <td>0.538600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1730</td>\n",
+       "      <td>0.599400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1740</td>\n",
+       "      <td>0.704300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1750</td>\n",
+       "      <td>0.580500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1760</td>\n",
+       "      <td>0.595600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1770</td>\n",
+       "      <td>0.583100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1780</td>\n",
+       "      <td>0.569500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1790</td>\n",
+       "      <td>0.603300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1800</td>\n",
+       "      <td>0.564500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1810</td>\n",
+       "      <td>0.592100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1820</td>\n",
+       "      <td>0.617000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1830</td>\n",
+       "      <td>0.656500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1840</td>\n",
+       "      <td>0.563600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1850</td>\n",
+       "      <td>0.624800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1860</td>\n",
+       "      <td>0.686700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1870</td>\n",
+       "      <td>0.572300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1880</td>\n",
+       "      <td>0.587700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1890</td>\n",
+       "      <td>0.583000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1900</td>\n",
+       "      <td>0.601500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1910</td>\n",
+       "      <td>0.559700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1920</td>\n",
+       "      <td>0.610100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1930</td>\n",
+       "      <td>0.571300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1940</td>\n",
+       "      <td>0.549900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1950</td>\n",
+       "      <td>0.589200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1960</td>\n",
+       "      <td>0.634800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1970</td>\n",
+       "      <td>0.584200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1980</td>\n",
+       "      <td>0.557000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1990</td>\n",
+       "      <td>0.602700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.669700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2010</td>\n",
+       "      <td>0.607500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2020</td>\n",
+       "      <td>0.631800</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Saving model checkpoint to ./results/checkpoint-500\n",
+      "Configuration saved in ./results/checkpoint-500\\config.json\n",
+      "Model weights saved in ./results/checkpoint-500\\pytorch_model.bin\n",
+      "Saving model checkpoint to ./results/checkpoint-1000\n",
+      "Configuration saved in ./results/checkpoint-1000\\config.json\n",
+      "Model weights saved in ./results/checkpoint-1000\\pytorch_model.bin\n",
+      "Saving model checkpoint to ./results/checkpoint-1500\n",
+      "Configuration saved in ./results/checkpoint-1500\\config.json\n",
+      "Model weights saved in ./results/checkpoint-1500\\pytorch_model.bin\n",
+      "Saving model checkpoint to ./results/checkpoint-2000\n",
+      "Configuration saved in ./results/checkpoint-2000\\config.json\n",
+      "Model weights saved in ./results/checkpoint-2000\\pytorch_model.bin\n",
+      "\n",
+      "\n",
+      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=2020, training_loss=0.6342116433795136, metrics={'train_runtime': 708.5025, 'train_samples_per_second': 45.598, 'train_steps_per_second': 2.851, 'total_flos': 4279491780980736.0, 'train_loss': 0.6342116433795136, 'epoch': 2.0})"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainer.train()"
+   ]
   }
  ],
  "metadata": {

README.md

@@ -9,6 +9,8 @@ app_file: app.py
 pinned: false
 ---
 
+Link to Google Site: https://sites.google.com/nyu.edu/finetunedpatentanalysis/app
+Link to huggingface model repo: https://huggingface.co/saccharinedreams/finetuned-distilbert-base-uncased-for-hupd
 Link to huggingface app: https://huggingface.co/spaces/saccharinedreams/sentiment-analysis-app
 
 

app.py

@@ -59,6 +59,7 @@ def sentiment_analysis(model, tokenizer):
 # Title the Streamlit app 'Finetuned Harvard USPTO Patent Dataset (using DistilBert-Base-Uncased)'
 st.title('Finetuned Sentiment Analysis for US Patents')
 st.markdown('Link to the app - [sentiment-analysis-app](https://huggingface.co/spaces/saccharinedreams/sentiment-analysis-app)')
+st.markdown('Link to the model - [model repo](https://huggingface.co/saccharinedreams/finetuned-distilbert-base-uncased-for-hupd')
 st.markdown('This model was finetuned on the Harvard USPTO Patent Dataset and uses Distilbert-Base-Uncased.')
 
 abstracts = load_abstracts()

finetunehupd.ipynb

@@ -53,8 +53,12 @@
     }
    ],
    "source": [
+    "# Use the tokenizer for DistilBert since we are using the pretrained base uncased model \n",
     "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
     "\n",
+    "# Load Harvard USPTO Patents Dataset from January 2-16 dataset using the load_dataset function\n",
+    "# Split the data into a train and validation set. For this purpose, I have split the patents as follows:\n",
+    "# train: Jan 1 - Jan 22, val: Jan 22 - Jan 31.\n",
     "dataset_dict = load_dataset('HUPD/hupd',\n",
     "    name='sample',\n",
     "    data_files=\"https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather\", \n",
@@ -100,6 +104,7 @@
    ],
    "source": [
     "# Label-to-index mapping for the decision status field\n",
+    "# Convert to binary, as we only care about whether or not a patent was rejected or accepted.\n",
     "decision_to_str = {'REJECTED': 0, 'ACCEPTED': 1, 'PENDING': 0, 'CONT-REJECTED': 0, 'CONT-ACCEPTED': 0, 'CONT-PENDING': 0}\n",
     "\n",
     "# Helper function\n",
@@ -179,6 +184,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# Process the train and validation sets so that only labels, input ids, and attention masks are left.\n",
     "train_set = train_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
     "val_set = val_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
     "\n",
@@ -241,7 +247,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Set the format\n",
+    "# Set the format to 'torch'\n",
     "train_set.set_format(type='torch', \n",
     "    columns=['labels', 'input_ids', 'attention_mask'])\n",
     "\n",
@@ -249,18 +255,6 @@
     "    columns=['labels', 'input_ids', 'attention_mask'])"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "d7ac796a-9f6e-4213-960f-e17837c27d87",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# train_dataloader and val_data_loader\n",
-    "train_dataloader = DataLoader(train_set, batch_size=16)\n",
-    "val_dataloader = DataLoader(val_set, batch_size=16)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 11,
@@ -289,6 +283,7 @@
     }
    ],
    "source": [
+    "# Use GPU acceleration. Make sure you have the right cuda and pytorch versions!\n",
     "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
     "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')\n",
     "model.to(device)\n",
@@ -1226,97 +1221,6 @@
    "source": [
     "trainer.train()"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "304e0d65-74cf-4945-978d-b9f56c5a83b1",
-   "metadata": {},
-   "source": [
-    "PyTorch Training Loop"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e56d14fb-dfde-40fa-9dfa-1187c2e09866",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# model.train()\n",
-    "# optim = AdamW(model.parameters(), lr=5e-5)\n",
-    "# num_training_epochs = 2\n",
-    "\n",
-    "# for epoch in range(num_training_epochs):\n",
-    "#   print(\"starting epoch: \" + str(epoch))\n",
-    "#   for batch in train_dataloader:\n",
-    "#     optim.zero_grad()\n",
-    "#     input_ids = batch['input_ids'].to(device)\n",
-    "#     attention_mask = batch['attention_mask'].to(device)\n",
-    "#     labels = batch['labels'].to(device)\n",
-    "#     outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
-    "#     loss = outputs[0]\n",
-    "#     loss.backward()\n",
-    "#     optim.step()\n",
-    "# model.eval()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "9b496593-c0de-4ce2-95d5-d5d3bf09d93c",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'The present invention relates to passive optical network (PON), and in particular, to an optical network terminal (ONT) in the PON system. In one embodiment, the optical network terminal includes a first interface coupled to a communications network, a second interface coupled to a network client and a processor including a memory coupled to the first interface and to the second interface, wherein the processor is capable of converting optical signals to electric signals, such that the network client can access the communications network.'"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "dataset_dict['train']['abstract'][0]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cd6f20fc-9874-465d-9781-505415db3ffd",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "6b6ad778-15aa-492a-9484-40106269e10d",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'Embodiments of the invention provide a method of reading and verifying a tag based on inherent disorder during a manufacturing process. The method includes using a first reader to take a first reading of an inherent disorder feature of the tag, and using a second reader to take a second reading of the inherent disorder feature of the tag. The method further includes matching the first reading with the second reading, and determining one or more acceptance criteria, wherein at least one of the acceptance criteria is based on whether the first reading and the second reading match within a predetermined threshold. If the acceptance criteria are met, then the tag is accepted, and a fingerprint for the tag is recorded. The invention further provides a method of testing and characterizing a reader of inherent disorder tags during a manufacturing process. The method includes taking a reading of a known inherent disorder tag, using the reading to measure a characteristic of the reader, and storing the measured characteristic for use when reading inherent disorder tags.'"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "dataset_dict['train']['abstract'][1]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cf9d94a7-4d7e-4f6b-88f8-46c9855289f4",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

finetunehupd.py

@@ -1,92 +0,0 @@
-# -*- coding: utf-8 -*-
-"""FinetuneHUPD.ipynb
-
-Automatically generated by Colaboratory.
-
-Original file is located at
-    https://colab.research.google.com/drive/17c2CQZx_kyD3-0fuQqv_pCMJ0Evd7fLN
-"""
-
-# Pretty print
-from pprint import pprint
-# Datasets load_dataset function
-from datasets import load_dataset
-# Transformers Autokenizer
-from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, Trainer, TrainingArguments, AdamW
-from torch.utils.data import DataLoader
-import torch
-
-tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
-
-dataset_dict = load_dataset('HUPD/hupd',
-    name='sample',
-    data_files="https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather", 
-    icpr_label=None,
-    train_filing_start_date='2016-01-01',
-    train_filing_end_date='2016-01-31',
-    val_filing_start_date='2016-01-01',
-    val_filing_end_date='2016-01-31',
-)
-
-print('Loading is done!')
-
-# Label-to-index mapping for the decision status field
-decision_to_str = {'REJECTED': 0, 'ACCEPTED': 1, 'PENDING': 2, 'CONT-REJECTED': 3, 'CONT-ACCEPTED': 4, 'CONT-PENDING': 5}
-
-# Helper function
-def map_decision_to_string(example):
-    return {'decision': decision_to_str[example['decision']]}
-
-# Re-labeling/mapping.
-train_set = dataset_dict['train'].map(map_decision_to_string)
-val_set = dataset_dict['validation'].map(map_decision_to_string)
-
-# Focus on the abstract section and tokenize the text using the tokenizer. 
-_SECTION_ = 'abstract'
-
-# Training set
-train_set = train_set.map(
-    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),
-    batched=True)
-
-# Validation set
-val_set = val_set.map(
-    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),
-    batched=True)
-
-# Set the format
-train_set.set_format(type='torch', 
-    columns=['input_ids', 'attention_mask', 'decision'])
-
-val_set.set_format(type='torch', 
-    columns=['input_ids', 'attention_mask', 'decision'])
-
-#print(train_set['decision'])
-
-# train_dataloader and val_data_loader
-train_dataloader = DataLoader(train_set, batch_size=16)
-val_dataloader = DataLoader(val_set, batch_size=16)
-
-device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
-model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
-model.to(device)
-print(device)
-print("torch cuda is avail: ")
-print(torch.cuda.is_available())
-model.train()
-optim = AdamW(model.parameters(), lr=5e-5)
-num_training_epochs = 2
-
-for epoch in range(num_training_epochs):
-  for batch in train_dataloader:
-    optim.zero_grad()
-    input_ids = batch['input_ids'].to(device)
-    attention_mask = batch['attention_mask'].to(device)
-    labels = batch['decision'].to(device)
-    outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
-    loss = outputs[0]
-    loss.backward()
-    optim.step()
-    print("batch finished")
-
-model.eval()