Added finetuned model

Checkpoints in /results/ folder

.ipynb_checkpoints/finetunehupd-checkpoint.ipynb

@@ -0,0 +1,51 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1df3c609-62a6-49c3-bcc6-29c520f9501c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# -*- coding: utf-8 -*-\n",
+    "\"\"\"FinetuneHUPD.ipynb\n",
+    "\n",
+    "Automatically generated by Colaboratory.\n",
+    "\n",
+    "Original file is located at\n",
+    "    https://colab.research.google.com/drive/17c2CQZx_kyD3-0fuQqv_pCMJ0Evd7fLN\n",
+    "\"\"\"\n",
+    "\n",
+    "# Pretty print\n",
+    "from pprint import pprint\n",
+    "# Datasets load_dataset function\n",
+    "from datasets import load_dataset\n",
+    "# Transformers Autokenizer\n",
+    "from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, Trainer, TrainingArguments, AdamW\n",
+    "from torch.utils.data import DataLoader\n",
+    "import torch"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

.ipynb_checkpoints/finetunehupd-checkpoint.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+"""FinetuneHUPD.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/17c2CQZx_kyD3-0fuQqv_pCMJ0Evd7fLN
+"""
+
+# Pretty print
+from pprint import pprint
+# Datasets load_dataset function
+from datasets import load_dataset
+# Transformers Autokenizer
+from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, Trainer, TrainingArguments, AdamW
+from torch.utils.data import DataLoader
+import torch
+
+tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
+
+dataset_dict = load_dataset('HUPD/hupd',
+    name='sample',
+    data_files="https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather", 
+    icpr_label=None,
+    train_filing_start_date='2016-01-01',
+    train_filing_end_date='2016-01-31',
+    val_filing_start_date='2016-01-01',
+    val_filing_end_date='2016-01-31',
+)
+
+print('Loading is done!')
+
+# Label-to-index mapping for the decision status field
+decision_to_str = {'REJECTED': 0, 'ACCEPTED': 1, 'PENDING': 2, 'CONT-REJECTED': 3, 'CONT-ACCEPTED': 4, 'CONT-PENDING': 5}
+
+# Helper function
+def map_decision_to_string(example):
+    return {'decision': decision_to_str[example['decision']]}
+
+# Re-labeling/mapping.
+train_set = dataset_dict['train'].map(map_decision_to_string)
+val_set = dataset_dict['validation'].map(map_decision_to_string)
+
+# Focus on the abstract section and tokenize the text using the tokenizer. 
+_SECTION_ = 'abstract'
+
+# Training set
+train_set = train_set.map(
+    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),
+    batched=True)
+
+# Validation set
+val_set = val_set.map(
+    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),
+    batched=True)
+
+# Set the format
+train_set.set_format(type='torch', 
+    columns=['input_ids', 'attention_mask', 'decision'])
+
+val_set.set_format(type='torch', 
+    columns=['input_ids', 'attention_mask', 'decision'])
+
+#print(train_set['decision'])
+
+# train_dataloader and val_data_loader
+train_dataloader = DataLoader(train_set, batch_size=16)
+val_dataloader = DataLoader(val_set, batch_size=16)
+
+device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
+model.to(device)
+print(device)
+print("torch cuda is avail: ")
+print(torch.cuda.is_available())
+model.train()
+optim = AdamW(model.parameters(), lr=5e-5)
+num_training_epochs = 2
+
+for epoch in range(num_training_epochs):
+  for batch in train_dataloader:
+    optim.zero_grad()
+    input_ids = batch['input_ids'].to(device)
+    attention_mask = batch['attention_mask'].to(device)
+    labels = batch['decision'].to(device)
+    outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
+    loss = outputs[0]
+    loss.backward()
+    optim.step()
+    print("batch finished")
+
+model.eval()

finetunehupd.ipynb

@@ -0,0 +1,1293 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "1df3c609-62a6-49c3-bcc6-29c520f9501c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Pretty print\n",
+    "from pprint import pprint\n",
+    "# Datasets load_dataset function\n",
+    "from datasets import load_dataset\n",
+    "# Transformers Autokenizer\n",
+    "from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertTokenizerFast, Trainer, TrainingArguments, AdamW\n",
+    "from torch.utils.data import DataLoader\n",
+    "import torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "58167c28-eb27-4f82-b7d0-8216dbeaf650",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Found cached dataset hupd (C:/Users/calia/.cache/huggingface/datasets/HUPD___hupd/sample-5094df4de61ed3bc/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "345008775bf549b5a548948949710507",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading is done!\n"
+     ]
+    }
+   ],
+   "source": [
+    "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
+    "\n",
+    "dataset_dict = load_dataset('HUPD/hupd',\n",
+    "    name='sample',\n",
+    "    data_files=\"https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather\", \n",
+    "    icpr_label=None,\n",
+    "    train_filing_start_date='2016-01-01',\n",
+    "    train_filing_end_date='2016-01-21',\n",
+    "    val_filing_start_date='2016-01-22',\n",
+    "    val_filing_end_date='2016-01-31',\n",
+    ")\n",
+    "\n",
+    "print('Loading is done!')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "e13c6ad1-a7f2-4806-80a2-e9c4655e1eed",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-9f7788eb9924fd62.arrow\n",
+      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-6c3687322fe5b556.arrow\n",
+      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-bd3b1eee4495f3ce.arrow\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/9094 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Label-to-index mapping for the decision status field\n",
+    "decision_to_str = {'REJECTED': 0, 'ACCEPTED': 1, 'PENDING': 0, 'CONT-REJECTED': 0, 'CONT-ACCEPTED': 0, 'CONT-PENDING': 0}\n",
+    "\n",
+    "# Helper function\n",
+    "def map_decision_to_string(example):\n",
+    "    return {'decision': decision_to_str[example['decision']]}\n",
+    "\n",
+    "# Re-labeling/mapping.\n",
+    "train_set = dataset_dict['train'].map(map_decision_to_string)\n",
+    "val_set = dataset_dict['validation'].map(map_decision_to_string)\n",
+    "\n",
+    "# Focus on the abstract section and tokenize the text using the tokenizer. \n",
+    "_SECTION_ = 'abstract'\n",
+    "\n",
+    "# Training set\n",
+    "train_set = train_set.map(\n",
+    "    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),\n",
+    "    batched=True)\n",
+    "\n",
+    "# Validation set\n",
+    "val_set = val_set.map(\n",
+    "    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),\n",
+    "    batched=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b5c098be-019b-42ce-9b80-4f6de93ef6a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 16153\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1e5a5390-19fe-4a73-b913-e3c1e2c2a399",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 9094\n",
+       "})"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4fb69db8-86e5-4c6c-8ac6-853d3e15fb93",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_set = train_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
+    "val_set = val_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
+    "\n",
+    "train_set = train_set.rename_column(\"decision\", \"labels\")\n",
+    "val_set = val_set.rename_column(\"decision\", \"labels\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c0d17213-4b14-418c-980c-0238236096c2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['labels', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 16153\n",
+       "})"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "da2f1c16-3ba4-4e56-9455-5cd838df4dcd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['labels', 'input_ids', 'attention_mask'],\n",
+       "    num_rows: 9094\n",
+       "})"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val_set"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "cfb35702-863d-4fec-83e1-44c4e5668156",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set the format\n",
+    "train_set.set_format(type='torch', \n",
+    "    columns=['labels', 'input_ids', 'attention_mask'])\n",
+    "\n",
+    "val_set.set_format(type='torch', \n",
+    "    columns=['labels', 'input_ids', 'attention_mask'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "d7ac796a-9f6e-4213-960f-e17837c27d87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# train_dataloader and val_data_loader\n",
+    "train_dataloader = DataLoader(train_set, batch_size=16)\n",
+    "val_dataloader = DataLoader(val_set, batch_size=16)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "b3248182-fddb-46dc-addb-26981a881d99",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_layer_norm.bias']\n",
+      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'pre_classifier.weight', 'classifier.bias', 'classifier.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "cuda\n",
+      "torch cuda is avail: \n",
+      "True\n"
+     ]
+    }
+   ],
+   "source": [
+    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
+    "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')\n",
+    "model.to(device)\n",
+    "print(device)\n",
+    "print(\"torch cuda is avail: \")\n",
+    "print(torch.cuda.is_available())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "abb2cf74-3cd5-4ca5-af0e-b0ee80627f2a",
+   "metadata": {},
+   "source": [
+    "HuggingFace Trainer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "99947cf9-a6cd-490f-a81d-32f65fb3cd46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "training_args = TrainingArguments(\n",
+    "    output_dir='./results/',\n",
+    "    num_train_epochs=2,\n",
+    "    per_device_train_batch_size=16,\n",
+    "    per_device_eval_batch_size=16,\n",
+    "    warmup_steps=500,\n",
+    "    learning_rate=5e-5,\n",
+    "    weight_decay=0.01,\n",
+    "    logging_dir='./logs/',\n",
+    "    logging_steps=10,\n",
+    ")\n",
+    "\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=training_args,\n",
+    "    train_dataset=train_set,\n",
+    "    eval_dataset=val_set,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "be865f1d-f29b-4306-8570-900386ac4570",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\calia\\anaconda3\\envs\\ai-finetuning-project\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+      "  warnings.warn(\n",
+      "***** Running training *****\n",
+      "  Num examples = 16153\n",
+      "  Num Epochs = 2\n",
+      "  Instantaneous batch size per device = 16\n",
+      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
+      "  Gradient Accumulation steps = 1\n",
+      "  Total optimization steps = 2020\n",
+      "  Number of trainable parameters = 66955010\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='2020' max='2020' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [2020/2020 11:47, Epoch 2/2]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>0.692000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>0.685100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>0.684000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>0.685100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>50</td>\n",
+       "      <td>0.678400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>60</td>\n",
+       "      <td>0.687300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>70</td>\n",
+       "      <td>0.681900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>80</td>\n",
+       "      <td>0.691100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>90</td>\n",
+       "      <td>0.683200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>100</td>\n",
+       "      <td>0.694100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>110</td>\n",
+       "      <td>0.673300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>120</td>\n",
+       "      <td>0.694100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>130</td>\n",
+       "      <td>0.669500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>140</td>\n",
+       "      <td>0.691100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>150</td>\n",
+       "      <td>0.683400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>160</td>\n",
+       "      <td>0.654900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>170</td>\n",
+       "      <td>0.684300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>180</td>\n",
+       "      <td>0.679300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>190</td>\n",
+       "      <td>0.662600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>200</td>\n",
+       "      <td>0.598400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>210</td>\n",
+       "      <td>0.717700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>220</td>\n",
+       "      <td>0.679100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>230</td>\n",
+       "      <td>0.677500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>240</td>\n",
+       "      <td>0.668800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>250</td>\n",
+       "      <td>0.678100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>260</td>\n",
+       "      <td>0.657500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>270</td>\n",
+       "      <td>0.707200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>280</td>\n",
+       "      <td>0.670300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>290</td>\n",
+       "      <td>0.659900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>300</td>\n",
+       "      <td>0.633300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>310</td>\n",
+       "      <td>0.676300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>320</td>\n",
+       "      <td>0.684800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>330</td>\n",
+       "      <td>0.673100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>340</td>\n",
+       "      <td>0.670500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>350</td>\n",
+       "      <td>0.657500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>360</td>\n",
+       "      <td>0.618100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>370</td>\n",
+       "      <td>0.670000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>380</td>\n",
+       "      <td>0.607400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>390</td>\n",
+       "      <td>0.656200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>400</td>\n",
+       "      <td>0.700000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>410</td>\n",
+       "      <td>0.644800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>420</td>\n",
+       "      <td>0.682800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>430</td>\n",
+       "      <td>0.668800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>440</td>\n",
+       "      <td>0.662600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>450</td>\n",
+       "      <td>0.647700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>460</td>\n",
+       "      <td>0.688600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>470</td>\n",
+       "      <td>0.682400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>480</td>\n",
+       "      <td>0.642900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>490</td>\n",
+       "      <td>0.726900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>500</td>\n",
+       "      <td>0.660400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>510</td>\n",
+       "      <td>0.649500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>520</td>\n",
+       "      <td>0.637200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>530</td>\n",
+       "      <td>0.669700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>540</td>\n",
+       "      <td>0.667100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>550</td>\n",
+       "      <td>0.617000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>560</td>\n",
+       "      <td>0.725300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>570</td>\n",
+       "      <td>0.656800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>580</td>\n",
+       "      <td>0.664600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>590</td>\n",
+       "      <td>0.702600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>600</td>\n",
+       "      <td>0.686300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>610</td>\n",
+       "      <td>0.668400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>620</td>\n",
+       "      <td>0.648200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>630</td>\n",
+       "      <td>0.628700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>640</td>\n",
+       "      <td>0.676700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>650</td>\n",
+       "      <td>0.652400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>660</td>\n",
+       "      <td>0.654300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>670</td>\n",
+       "      <td>0.640800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>680</td>\n",
+       "      <td>0.672000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>690</td>\n",
+       "      <td>0.636100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>700</td>\n",
+       "      <td>0.689100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>710</td>\n",
+       "      <td>0.691100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>720</td>\n",
+       "      <td>0.650300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>730</td>\n",
+       "      <td>0.655200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>740</td>\n",
+       "      <td>0.668400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>750</td>\n",
+       "      <td>0.659200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>760</td>\n",
+       "      <td>0.647800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>770</td>\n",
+       "      <td>0.662800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>780</td>\n",
+       "      <td>0.648500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>790</td>\n",
+       "      <td>0.656700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>800</td>\n",
+       "      <td>0.669400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>810</td>\n",
+       "      <td>0.607800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>820</td>\n",
+       "      <td>0.683200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>830</td>\n",
+       "      <td>0.663800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>840</td>\n",
+       "      <td>0.700900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>850</td>\n",
+       "      <td>0.648200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>860</td>\n",
+       "      <td>0.619400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>870</td>\n",
+       "      <td>0.649200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>880</td>\n",
+       "      <td>0.717500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>890</td>\n",
+       "      <td>0.669600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>900</td>\n",
+       "      <td>0.669700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>910</td>\n",
+       "      <td>0.683900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>920</td>\n",
+       "      <td>0.636900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>930</td>\n",
+       "      <td>0.656400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>940</td>\n",
+       "      <td>0.650000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>950</td>\n",
+       "      <td>0.617800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>960</td>\n",
+       "      <td>0.665600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>970</td>\n",
+       "      <td>0.642700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>980</td>\n",
+       "      <td>0.644000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>990</td>\n",
+       "      <td>0.688900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1000</td>\n",
+       "      <td>0.654700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1010</td>\n",
+       "      <td>0.645800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1020</td>\n",
+       "      <td>0.609200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1030</td>\n",
+       "      <td>0.602300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1040</td>\n",
+       "      <td>0.618800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1050</td>\n",
+       "      <td>0.643500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1060</td>\n",
+       "      <td>0.611000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1070</td>\n",
+       "      <td>0.645000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1080</td>\n",
+       "      <td>0.641000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1090</td>\n",
+       "      <td>0.595400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1100</td>\n",
+       "      <td>0.635100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1110</td>\n",
+       "      <td>0.611600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1120</td>\n",
+       "      <td>0.600300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1130</td>\n",
+       "      <td>0.618100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1140</td>\n",
+       "      <td>0.617200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1150</td>\n",
+       "      <td>0.633400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1160</td>\n",
+       "      <td>0.597600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1170</td>\n",
+       "      <td>0.619400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1180</td>\n",
+       "      <td>0.584200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1190</td>\n",
+       "      <td>0.600700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1200</td>\n",
+       "      <td>0.657400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1210</td>\n",
+       "      <td>0.569600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1220</td>\n",
+       "      <td>0.575500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1230</td>\n",
+       "      <td>0.617900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1240</td>\n",
+       "      <td>0.610300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1250</td>\n",
+       "      <td>0.570600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1260</td>\n",
+       "      <td>0.545700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1270</td>\n",
+       "      <td>0.656300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1280</td>\n",
+       "      <td>0.554700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1290</td>\n",
+       "      <td>0.598200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1300</td>\n",
+       "      <td>0.606300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1310</td>\n",
+       "      <td>0.600500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1320</td>\n",
+       "      <td>0.569800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1330</td>\n",
+       "      <td>0.604700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1340</td>\n",
+       "      <td>0.628300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1350</td>\n",
+       "      <td>0.602700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1360</td>\n",
+       "      <td>0.583700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1370</td>\n",
+       "      <td>0.623800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1380</td>\n",
+       "      <td>0.670300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1390</td>\n",
+       "      <td>0.622400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1400</td>\n",
+       "      <td>0.590200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1410</td>\n",
+       "      <td>0.587000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1420</td>\n",
+       "      <td>0.555500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1430</td>\n",
+       "      <td>0.561000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1440</td>\n",
+       "      <td>0.514300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1450</td>\n",
+       "      <td>0.553100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1460</td>\n",
+       "      <td>0.692400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1470</td>\n",
+       "      <td>0.605200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1480</td>\n",
+       "      <td>0.548000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1490</td>\n",
+       "      <td>0.672600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1500</td>\n",
+       "      <td>0.531100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1510</td>\n",
+       "      <td>0.610600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1520</td>\n",
+       "      <td>0.580200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1530</td>\n",
+       "      <td>0.571300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1540</td>\n",
+       "      <td>0.644400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1550</td>\n",
+       "      <td>0.558500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1560</td>\n",
+       "      <td>0.624000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1570</td>\n",
+       "      <td>0.659200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1580</td>\n",
+       "      <td>0.580500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1590</td>\n",
+       "      <td>0.649900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1600</td>\n",
+       "      <td>0.608700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1610</td>\n",
+       "      <td>0.595100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1620</td>\n",
+       "      <td>0.592900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1630</td>\n",
+       "      <td>0.584000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1640</td>\n",
+       "      <td>0.607100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1650</td>\n",
+       "      <td>0.565800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1660</td>\n",
+       "      <td>0.568300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1670</td>\n",
+       "      <td>0.572200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1680</td>\n",
+       "      <td>0.597500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1690</td>\n",
+       "      <td>0.602700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1700</td>\n",
+       "      <td>0.692900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1710</td>\n",
+       "      <td>0.597900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1720</td>\n",
+       "      <td>0.538600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1730</td>\n",
+       "      <td>0.599400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1740</td>\n",
+       "      <td>0.704300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1750</td>\n",
+       "      <td>0.580500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1760</td>\n",
+       "      <td>0.595600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1770</td>\n",
+       "      <td>0.583100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1780</td>\n",
+       "      <td>0.569500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1790</td>\n",
+       "      <td>0.603300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1800</td>\n",
+       "      <td>0.564500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1810</td>\n",
+       "      <td>0.592100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1820</td>\n",
+       "      <td>0.617000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1830</td>\n",
+       "      <td>0.656500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1840</td>\n",
+       "      <td>0.563600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1850</td>\n",
+       "      <td>0.624800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1860</td>\n",
+       "      <td>0.686700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1870</td>\n",
+       "      <td>0.572300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1880</td>\n",
+       "      <td>0.587700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1890</td>\n",
+       "      <td>0.583000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1900</td>\n",
+       "      <td>0.601500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1910</td>\n",
+       "      <td>0.559700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1920</td>\n",
+       "      <td>0.610100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1930</td>\n",
+       "      <td>0.571300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1940</td>\n",
+       "      <td>0.549900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1950</td>\n",
+       "      <td>0.589200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1960</td>\n",
+       "      <td>0.634800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1970</td>\n",
+       "      <td>0.584200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1980</td>\n",
+       "      <td>0.557000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>1990</td>\n",
+       "      <td>0.602700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2000</td>\n",
+       "      <td>0.669700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2010</td>\n",
+       "      <td>0.607500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2020</td>\n",
+       "      <td>0.631800</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Saving model checkpoint to ./results/checkpoint-500\n",
+      "Configuration saved in ./results/checkpoint-500\\config.json\n",
+      "Model weights saved in ./results/checkpoint-500\\pytorch_model.bin\n",
+      "Saving model checkpoint to ./results/checkpoint-1000\n",
+      "Configuration saved in ./results/checkpoint-1000\\config.json\n",
+      "Model weights saved in ./results/checkpoint-1000\\pytorch_model.bin\n",
+      "Saving model checkpoint to ./results/checkpoint-1500\n",
+      "Configuration saved in ./results/checkpoint-1500\\config.json\n",
+      "Model weights saved in ./results/checkpoint-1500\\pytorch_model.bin\n",
+      "Saving model checkpoint to ./results/checkpoint-2000\n",
+      "Configuration saved in ./results/checkpoint-2000\\config.json\n",
+      "Model weights saved in ./results/checkpoint-2000\\pytorch_model.bin\n",
+      "\n",
+      "\n",
+      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=2020, training_loss=0.6342116433795136, metrics={'train_runtime': 708.5025, 'train_samples_per_second': 45.598, 'train_steps_per_second': 2.851, 'total_flos': 4279491780980736.0, 'train_loss': 0.6342116433795136, 'epoch': 2.0})"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "304e0d65-74cf-4945-978d-b9f56c5a83b1",
+   "metadata": {},
+   "source": [
+    "PyTorch Training Loop"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e56d14fb-dfde-40fa-9dfa-1187c2e09866",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model.train()\n",
+    "# optim = AdamW(model.parameters(), lr=5e-5)\n",
+    "# num_training_epochs = 2\n",
+    "\n",
+    "# for epoch in range(num_training_epochs):\n",
+    "#   print(\"starting epoch: \" + str(epoch))\n",
+    "#   for batch in train_dataloader:\n",
+    "#     optim.zero_grad()\n",
+    "#     input_ids = batch['input_ids'].to(device)\n",
+    "#     attention_mask = batch['attention_mask'].to(device)\n",
+    "#     labels = batch['labels'].to(device)\n",
+    "#     outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
+    "#     loss = outputs[0]\n",
+    "#     loss.backward()\n",
+    "#     optim.step()\n",
+    "# model.eval()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9b496593-c0de-4ce2-95d5-d5d3bf09d93c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

finetunehupd.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+"""FinetuneHUPD.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/17c2CQZx_kyD3-0fuQqv_pCMJ0Evd7fLN
+"""
+
+# Pretty print
+from pprint import pprint
+# Datasets load_dataset function
+from datasets import load_dataset
+# Transformers Autokenizer
+from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, Trainer, TrainingArguments, AdamW
+from torch.utils.data import DataLoader
+import torch
+
+tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
+
+dataset_dict = load_dataset('HUPD/hupd',
+    name='sample',
+    data_files="https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather", 
+    icpr_label=None,
+    train_filing_start_date='2016-01-01',
+    train_filing_end_date='2016-01-31',
+    val_filing_start_date='2016-01-01',
+    val_filing_end_date='2016-01-31',
+)
+
+print('Loading is done!')
+
+# Label-to-index mapping for the decision status field
+decision_to_str = {'REJECTED': 0, 'ACCEPTED': 1, 'PENDING': 2, 'CONT-REJECTED': 3, 'CONT-ACCEPTED': 4, 'CONT-PENDING': 5}
+
+# Helper function
+def map_decision_to_string(example):
+    return {'decision': decision_to_str[example['decision']]}
+
+# Re-labeling/mapping.
+train_set = dataset_dict['train'].map(map_decision_to_string)
+val_set = dataset_dict['validation'].map(map_decision_to_string)
+
+# Focus on the abstract section and tokenize the text using the tokenizer. 
+_SECTION_ = 'abstract'
+
+# Training set
+train_set = train_set.map(
+    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),
+    batched=True)
+
+# Validation set
+val_set = val_set.map(
+    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),
+    batched=True)
+
+# Set the format
+train_set.set_format(type='torch', 
+    columns=['input_ids', 'attention_mask', 'decision'])
+
+val_set.set_format(type='torch', 
+    columns=['input_ids', 'attention_mask', 'decision'])
+
+#print(train_set['decision'])
+
+# train_dataloader and val_data_loader
+train_dataloader = DataLoader(train_set, batch_size=16)
+val_dataloader = DataLoader(val_set, batch_size=16)
+
+device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
+model.to(device)
+print(device)
+print("torch cuda is avail: ")
+print(torch.cuda.is_available())
+model.train()
+optim = AdamW(model.parameters(), lr=5e-5)
+num_training_epochs = 2
+
+for epoch in range(num_training_epochs):
+  for batch in train_dataloader:
+    optim.zero_grad()
+    input_ids = batch['input_ids'].to(device)
+    attention_mask = batch['attention_mask'].to(device)
+    labels = batch['decision'].to(device)
+    outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
+    loss = outputs[0]
+    loss.backward()
+    optim.step()
+    print("batch finished")
+
+model.eval()