{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1df3c609-62a6-49c3-bcc6-29c520f9501c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pretty print\n",
    "from pprint import pprint\n",
    "# Datasets load_dataset function\n",
    "from datasets import load_dataset\n",
    "# Transformers Autokenizer\n",
    "from transformers import AutoTokenizer, DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertTokenizerFast, Trainer, TrainingArguments, AdamW\n",
    "from torch.utils.data import DataLoader\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "58167c28-eb27-4f82-b7d0-8216dbeaf650",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset hupd (C:/Users/calia/.cache/huggingface/datasets/HUPD___hupd/sample-5094df4de61ed3bc/0.0.0/6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2f090474cb148548ce3eb73698fcc6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading is done!\n"
     ]
    }
   ],
   "source": [
    "# Use the tokenizer for DistilBert since we are using the pretrained base uncased model \n",
    "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
    "\n",
    "# Load Harvard USPTO Patents Dataset from January 2-16 dataset using the load_dataset function\n",
    "# Split the data into a train and validation set. For this purpose, I have split the patents as follows:\n",
    "# train: Jan 1 - Jan 22, val: Jan 22 - Jan 31.\n",
    "dataset_dict = load_dataset('HUPD/hupd',\n",
    "    name='sample',\n",
    "    data_files=\"https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather\", \n",
    "    icpr_label=None,\n",
    "    train_filing_start_date='2016-01-01',\n",
    "    train_filing_end_date='2016-01-21',\n",
    "    val_filing_start_date='2016-01-22',\n",
    "    val_filing_end_date='2016-01-31',\n",
    ")\n",
    "\n",
    "print('Loading is done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e13c6ad1-a7f2-4806-80a2-e9c4655e1eed",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-9f7788eb9924fd62.arrow\n",
      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-6c3687322fe5b556.arrow\n",
      "Loading cached processed dataset at C:\\Users\\calia\\.cache\\huggingface\\datasets\\HUPD___hupd\\sample-5094df4de61ed3bc\\0.0.0\\6920d2def8fd7767046c0470603357f76866e5a09c97e19571896bfdca521142\\cache-bd3b1eee4495f3ce.arrow\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/9094 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Label-to-index mapping for the decision status field\n",
    "# Convert to binary, as we only care about whether or not a patent was rejected or accepted.\n",
    "decision_to_str = {'REJECTED': 0, 'ACCEPTED': 1, 'PENDING': 0, 'CONT-REJECTED': 0, 'CONT-ACCEPTED': 0, 'CONT-PENDING': 0}\n",
    "\n",
    "# Helper function\n",
    "def map_decision_to_string(example):\n",
    "    return {'decision': decision_to_str[example['decision']]}\n",
    "\n",
    "# Re-labeling/mapping.\n",
    "train_set = dataset_dict['train'].map(map_decision_to_string)\n",
    "val_set = dataset_dict['validation'].map(map_decision_to_string)\n",
    "\n",
    "# Focus on the abstract section and tokenize the text using the tokenizer. \n",
    "_SECTION_ = 'abstract'\n",
    "\n",
    "# Training set\n",
    "train_set = train_set.map(\n",
    "    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),\n",
    "    batched=True)\n",
    "\n",
    "# Validation set\n",
    "val_set = val_set.map(\n",
    "    lambda e: tokenizer((e[_SECTION_]), truncation=True, padding='max_length'),\n",
    "    batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b5c098be-019b-42ce-9b80-4f6de93ef6a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id', 'input_ids', 'attention_mask'],\n",
       "    num_rows: 16153\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1e5a5390-19fe-4a73-b913-e3c1e2c2a399",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['patent_number', 'decision', 'title', 'abstract', 'claims', 'background', 'summary', 'description', 'cpc_label', 'ipc_label', 'filing_date', 'patent_issue_date', 'date_published', 'examiner_id', 'input_ids', 'attention_mask'],\n",
       "    num_rows: 9094\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4fb69db8-86e5-4c6c-8ac6-853d3e15fb93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process the train and validation sets so that only labels, input ids, and attention masks are left.\n",
    "train_set = train_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
    "val_set = val_set.remove_columns([\"patent_number\", \"title\", \"abstract\", \"claims\", \"background\", \"summary\", \"description\", \"cpc_label\", \"ipc_label\", \"filing_date\", \"patent_issue_date\", \"date_published\", \"examiner_id\"])\n",
    "\n",
    "train_set = train_set.rename_column(\"decision\", \"labels\")\n",
    "val_set = val_set.rename_column(\"decision\", \"labels\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c0d17213-4b14-418c-980c-0238236096c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['labels', 'input_ids', 'attention_mask'],\n",
       "    num_rows: 16153\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "da2f1c16-3ba4-4e56-9455-5cd838df4dcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['labels', 'input_ids', 'attention_mask'],\n",
       "    num_rows: 9094\n",
       "})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cfb35702-863d-4fec-83e1-44c4e5668156",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the format to 'torch'\n",
    "train_set.set_format(type='torch', \n",
    "    columns=['labels', 'input_ids', 'attention_mask'])\n",
    "\n",
    "val_set.set_format(type='torch', \n",
    "    columns=['labels', 'input_ids', 'attention_mask'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b3248182-fddb-46dc-addb-26981a881d99",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_layer_norm.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'pre_classifier.weight', 'classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n",
      "torch cuda is avail: \n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# Use GPU acceleration. Make sure you have the right cuda and pytorch versions!\n",
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')\n",
    "model.to(device)\n",
    "print(device)\n",
    "print(\"torch cuda is avail: \")\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abb2cf74-3cd5-4ca5-af0e-b0ee80627f2a",
   "metadata": {},
   "source": [
    "HuggingFace Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "99947cf9-a6cd-490f-a81d-32f65fb3cd46",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir='./results/',\n",
    "    num_train_epochs=2,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    warmup_steps=500,\n",
    "    learning_rate=5e-5,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs/',\n",
    "    logging_steps=10,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_set,\n",
    "    eval_dataset=val_set,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "be865f1d-f29b-4306-8570-900386ac4570",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\calia\\anaconda3\\envs\\ai-finetuning-project\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 16153\n",
      "  Num Epochs = 2\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2020\n",
      "  Number of trainable parameters = 66955010\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2020' max='2020' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2020/2020 11:47, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.692000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.685100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.684000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.685100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.678400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60</td>\n",
       "      <td>0.687300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70</td>\n",
       "      <td>0.681900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80</td>\n",
       "      <td>0.691100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90</td>\n",
       "      <td>0.683200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.694100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>110</td>\n",
       "      <td>0.673300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>120</td>\n",
       "      <td>0.694100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>0.669500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>140</td>\n",
       "      <td>0.691100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.683400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>160</td>\n",
       "      <td>0.654900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170</td>\n",
       "      <td>0.684300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>180</td>\n",
       "      <td>0.679300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>190</td>\n",
       "      <td>0.662600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.598400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>210</td>\n",
       "      <td>0.717700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>0.679100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>230</td>\n",
       "      <td>0.677500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>0.668800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>0.678100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>260</td>\n",
       "      <td>0.657500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>270</td>\n",
       "      <td>0.707200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>280</td>\n",
       "      <td>0.670300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>290</td>\n",
       "      <td>0.659900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.633300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>310</td>\n",
       "      <td>0.676300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>320</td>\n",
       "      <td>0.684800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>330</td>\n",
       "      <td>0.673100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>340</td>\n",
       "      <td>0.670500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>0.657500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>360</td>\n",
       "      <td>0.618100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>370</td>\n",
       "      <td>0.670000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>380</td>\n",
       "      <td>0.607400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>390</td>\n",
       "      <td>0.656200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.700000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>410</td>\n",
       "      <td>0.644800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>420</td>\n",
       "      <td>0.682800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>430</td>\n",
       "      <td>0.668800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>440</td>\n",
       "      <td>0.662600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>0.647700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>460</td>\n",
       "      <td>0.688600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>470</td>\n",
       "      <td>0.682400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>480</td>\n",
       "      <td>0.642900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>490</td>\n",
       "      <td>0.726900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.660400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>510</td>\n",
       "      <td>0.649500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>520</td>\n",
       "      <td>0.637200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>530</td>\n",
       "      <td>0.669700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>540</td>\n",
       "      <td>0.667100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>0.617000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>560</td>\n",
       "      <td>0.725300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>570</td>\n",
       "      <td>0.656800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>580</td>\n",
       "      <td>0.664600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>590</td>\n",
       "      <td>0.702600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.686300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>610</td>\n",
       "      <td>0.668400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>620</td>\n",
       "      <td>0.648200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>630</td>\n",
       "      <td>0.628700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>640</td>\n",
       "      <td>0.676700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>650</td>\n",
       "      <td>0.652400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>660</td>\n",
       "      <td>0.654300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>670</td>\n",
       "      <td>0.640800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>680</td>\n",
       "      <td>0.672000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>690</td>\n",
       "      <td>0.636100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>0.689100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>710</td>\n",
       "      <td>0.691100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>720</td>\n",
       "      <td>0.650300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>730</td>\n",
       "      <td>0.655200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>740</td>\n",
       "      <td>0.668400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>750</td>\n",
       "      <td>0.659200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>760</td>\n",
       "      <td>0.647800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>770</td>\n",
       "      <td>0.662800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>780</td>\n",
       "      <td>0.648500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>790</td>\n",
       "      <td>0.656700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.669400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>810</td>\n",
       "      <td>0.607800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>820</td>\n",
       "      <td>0.683200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>830</td>\n",
       "      <td>0.663800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>840</td>\n",
       "      <td>0.700900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>850</td>\n",
       "      <td>0.648200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>860</td>\n",
       "      <td>0.619400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>870</td>\n",
       "      <td>0.649200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>880</td>\n",
       "      <td>0.717500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>890</td>\n",
       "      <td>0.669600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>0.669700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>910</td>\n",
       "      <td>0.683900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>920</td>\n",
       "      <td>0.636900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>930</td>\n",
       "      <td>0.656400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>940</td>\n",
       "      <td>0.650000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>950</td>\n",
       "      <td>0.617800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>960</td>\n",
       "      <td>0.665600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>970</td>\n",
       "      <td>0.642700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>980</td>\n",
       "      <td>0.644000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>990</td>\n",
       "      <td>0.688900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.654700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1010</td>\n",
       "      <td>0.645800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1020</td>\n",
       "      <td>0.609200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1030</td>\n",
       "      <td>0.602300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1040</td>\n",
       "      <td>0.618800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1050</td>\n",
       "      <td>0.643500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1060</td>\n",
       "      <td>0.611000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1070</td>\n",
       "      <td>0.645000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1080</td>\n",
       "      <td>0.641000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1090</td>\n",
       "      <td>0.595400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>0.635100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1110</td>\n",
       "      <td>0.611600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1120</td>\n",
       "      <td>0.600300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1130</td>\n",
       "      <td>0.618100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1140</td>\n",
       "      <td>0.617200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1150</td>\n",
       "      <td>0.633400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1160</td>\n",
       "      <td>0.597600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1170</td>\n",
       "      <td>0.619400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1180</td>\n",
       "      <td>0.584200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1190</td>\n",
       "      <td>0.600700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.657400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1210</td>\n",
       "      <td>0.569600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1220</td>\n",
       "      <td>0.575500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1230</td>\n",
       "      <td>0.617900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1240</td>\n",
       "      <td>0.610300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1250</td>\n",
       "      <td>0.570600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1260</td>\n",
       "      <td>0.545700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1270</td>\n",
       "      <td>0.656300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1280</td>\n",
       "      <td>0.554700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1290</td>\n",
       "      <td>0.598200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>0.606300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1310</td>\n",
       "      <td>0.600500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1320</td>\n",
       "      <td>0.569800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1330</td>\n",
       "      <td>0.604700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1340</td>\n",
       "      <td>0.628300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1350</td>\n",
       "      <td>0.602700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1360</td>\n",
       "      <td>0.583700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1370</td>\n",
       "      <td>0.623800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1380</td>\n",
       "      <td>0.670300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1390</td>\n",
       "      <td>0.622400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.590200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1410</td>\n",
       "      <td>0.587000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1420</td>\n",
       "      <td>0.555500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1430</td>\n",
       "      <td>0.561000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1440</td>\n",
       "      <td>0.514300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1450</td>\n",
       "      <td>0.553100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1460</td>\n",
       "      <td>0.692400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1470</td>\n",
       "      <td>0.605200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1480</td>\n",
       "      <td>0.548000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1490</td>\n",
       "      <td>0.672600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.531100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1510</td>\n",
       "      <td>0.610600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1520</td>\n",
       "      <td>0.580200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1530</td>\n",
       "      <td>0.571300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1540</td>\n",
       "      <td>0.644400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1550</td>\n",
       "      <td>0.558500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1560</td>\n",
       "      <td>0.624000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1570</td>\n",
       "      <td>0.659200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1580</td>\n",
       "      <td>0.580500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1590</td>\n",
       "      <td>0.649900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>0.608700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1610</td>\n",
       "      <td>0.595100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1620</td>\n",
       "      <td>0.592900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1630</td>\n",
       "      <td>0.584000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1640</td>\n",
       "      <td>0.607100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1650</td>\n",
       "      <td>0.565800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1660</td>\n",
       "      <td>0.568300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1670</td>\n",
       "      <td>0.572200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1680</td>\n",
       "      <td>0.597500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1690</td>\n",
       "      <td>0.602700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>0.692900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1710</td>\n",
       "      <td>0.597900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1720</td>\n",
       "      <td>0.538600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1730</td>\n",
       "      <td>0.599400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1740</td>\n",
       "      <td>0.704300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1750</td>\n",
       "      <td>0.580500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1760</td>\n",
       "      <td>0.595600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1770</td>\n",
       "      <td>0.583100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1780</td>\n",
       "      <td>0.569500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1790</td>\n",
       "      <td>0.603300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.564500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1810</td>\n",
       "      <td>0.592100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1820</td>\n",
       "      <td>0.617000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1830</td>\n",
       "      <td>0.656500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1840</td>\n",
       "      <td>0.563600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1850</td>\n",
       "      <td>0.624800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1860</td>\n",
       "      <td>0.686700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1870</td>\n",
       "      <td>0.572300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1880</td>\n",
       "      <td>0.587700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1890</td>\n",
       "      <td>0.583000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>0.601500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1910</td>\n",
       "      <td>0.559700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1920</td>\n",
       "      <td>0.610100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1930</td>\n",
       "      <td>0.571300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1940</td>\n",
       "      <td>0.549900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1950</td>\n",
       "      <td>0.589200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1960</td>\n",
       "      <td>0.634800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1970</td>\n",
       "      <td>0.584200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1980</td>\n",
       "      <td>0.557000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1990</td>\n",
       "      <td>0.602700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.669700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2010</td>\n",
       "      <td>0.607500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2020</td>\n",
       "      <td>0.631800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./results/checkpoint-500\n",
      "Configuration saved in ./results/checkpoint-500\\config.json\n",
      "Model weights saved in ./results/checkpoint-500\\pytorch_model.bin\n",
      "Saving model checkpoint to ./results/checkpoint-1000\n",
      "Configuration saved in ./results/checkpoint-1000\\config.json\n",
      "Model weights saved in ./results/checkpoint-1000\\pytorch_model.bin\n",
      "Saving model checkpoint to ./results/checkpoint-1500\n",
      "Configuration saved in ./results/checkpoint-1500\\config.json\n",
      "Model weights saved in ./results/checkpoint-1500\\pytorch_model.bin\n",
      "Saving model checkpoint to ./results/checkpoint-2000\n",
      "Configuration saved in ./results/checkpoint-2000\\config.json\n",
      "Model weights saved in ./results/checkpoint-2000\\pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=2020, training_loss=0.6342116433795136, metrics={'train_runtime': 708.5025, 'train_samples_per_second': 45.598, 'train_steps_per_second': 2.851, 'total_flos': 4279491780980736.0, 'train_loss': 0.6342116433795136, 'epoch': 2.0})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}