{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import (\n",
    "    RobertaTokenizerFast,\n",
    "    RobertaForSequenceClassification,\n",
    "    TrainingArguments,\n",
    "    Trainer,\n",
    "    AutoConfig,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# Load your preprocessed dataset into a pandas DataFrame\n",
    "df = pd.read_csv(r'D:\\Thesis\\Datasets\\preprocessed_dataset.csv')\n",
    "df = df.dropna()\n",
    "df.head()\n",
    "\n",
    "#independent variable\n",
    "X = list(df['sentence'])\n",
    "#dependent variable\n",
    "y = list(df['label'])\n",
    "\n",
    "#splitting\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.20, random_state= 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = RobertaTokenizerFast.from_pretrained(\"jcblaise/roberta-tagalog-base\")\n",
    "def tokenize(batch):\n",
    "    return tokenizer(batch[\"text\"], padding=True, truncation=True, max_length=97)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum sequence length: 146\n",
      "Minimum sequence length: 5\n",
      "Average sequence length: 34.3118799903218\n"
     ]
    }
   ],
   "source": [
    "# Calculate the length of each input sequence\n",
    "sequence_lengths = [len(tokenizer.tokenize(text)) for text in X_train]\n",
    "\n",
    "# Print statistics about sequence lengths\n",
    "print(\"Maximum sequence length:\", max(sequence_lengths))\n",
    "print(\"Minimum sequence length:\", min(sequence_lengths))\n",
    "print(\"Average sequence length:\", sum(sequence_lengths) / len(sequence_lengths))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "Map: 100%|██████████| 8266/8266 [00:00<00:00, 14158.16 examples/s]\n",
      "\n",
      "\u001b[A\n",
      "Map: 100%|██████████| 2067/2067 [00:00<00:00, 15252.23 examples/s]\n"
     ]
    }
   ],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "# Create datasets using the Dataset class\n",
    "train_dataset = Dataset.from_dict({\"text\": X_train, \"label\": y_train})\n",
    "val_dataset = Dataset.from_dict({\"text\": X_test, \"label\": y_test})\n",
    "\n",
    "# Tokenize datasets using the tokenize function\n",
    "train_dataset = train_dataset.map(tokenize, batched=True)\n",
    "val_dataset = val_dataset.map(tokenize, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at jcblaise/roberta-tagalog-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# Define your model configuration\n",
    "model_config = RobertaForSequenceClassification.from_pretrained(\"jcblaise/roberta-tagalog-base\")\n",
    "\n",
    "# Define your training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    per_device_train_batch_size=8,\n",
    "    per_device_eval_batch_size=8,\n",
    "    num_train_epochs=3,\n",
    "    logging_dir='./logs',\n",
    "    logging_steps=100,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    eval_steps=500,\n",
    ")\n",
    "\n",
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=model_config,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/3102 [02:45<?, ?it/s]\n",
      "                                                    \n",
      "  3%|▎         | 100/3102 [03:56<2:00:25,  2.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.6868, 'learning_rate': 4.8388136686009026e-05, 'epoch': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      "  6%|▋         | 200/3102 [07:51<1:49:08,  2.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5552, 'learning_rate': 4.677627337201805e-05, 'epoch': 0.19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      " 10%|▉         | 300/3102 [11:40<2:00:36,  2.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.5657, 'learning_rate': 4.516441005802708e-05, 'epoch': 0.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      " 13%|█▎        | 400/3102 [15:19<1:34:58,  2.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4919, 'learning_rate': 4.355254674403611e-05, 'epoch': 0.39}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      " 16%|█▌        | 500/3102 [18:53<1:32:59,  2.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4673, 'learning_rate': 4.1940683430045136e-05, 'epoch': 0.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\n",
      "                                                    \n",
      "\u001b[A                                     \n",
      " 16%|█▌        | 500/3102 [21:31<1:32:59,  2.14s/it]\n",
      "\u001b[ACheckpoint destination directory ./results\\checkpoint-500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.4286854565143585, 'eval_runtime': 157.876, 'eval_samples_per_second': 13.093, 'eval_steps_per_second': 1.641, 'epoch': 0.48}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 19%|█▉        | 600/3102 [25:10<1:28:58,  2.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4705, 'learning_rate': 4.032882011605416e-05, 'epoch': 0.58}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      " 23%|██▎       | 700/3102 [28:43<1:24:43,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4717, 'learning_rate': 3.8716956802063184e-05, 'epoch': 0.68}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      " 26%|██▌       | 800/3102 [32:17<1:21:12,  2.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4523, 'learning_rate': 3.7105093488072215e-05, 'epoch': 0.77}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      " 29%|██▉       | 900/3102 [35:51<1:20:16,  2.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4333, 'learning_rate': 3.549323017408124e-05, 'epoch': 0.87}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 32%|███▏      | 1000/3102 [39:26<1:14:50,  2.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4765, 'learning_rate': 3.388136686009026e-05, 'epoch': 0.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\n",
      "                                                     \n",
      "\u001b[A                                     \n",
      " 32%|███▏      | 1000/3102 [42:01<1:14:50,  2.14s/it]\n",
      "\u001b[ACheckpoint destination directory ./results\\checkpoint-1000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3951832354068756, 'eval_runtime': 154.675, 'eval_samples_per_second': 13.364, 'eval_steps_per_second': 1.674, 'epoch': 0.97}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                      \n",
      " 35%|███▌      | 1100/3102 [45:32<1:08:44,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3246, 'learning_rate': 3.226950354609929e-05, 'epoch': 1.06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 39%|███▊      | 1200/3102 [49:00<1:05:30,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3084, 'learning_rate': 3.065764023210832e-05, 'epoch': 1.16}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 42%|████▏     | 1300/3102 [52:27<1:02:14,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3059, 'learning_rate': 2.9045776918117345e-05, 'epoch': 1.26}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 45%|████▌     | 1400/3102 [55:54<58:39,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3234, 'learning_rate': 2.7433913604126372e-05, 'epoch': 1.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 48%|████▊     | 1500/3102 [59:21<55:20,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3353, 'learning_rate': 2.5822050290135396e-05, 'epoch': 1.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\n",
      "                                                   \n",
      "\u001b[A                                     \n",
      " 48%|████▊     | 1500/3102 [1:01:54<55:20,  2.07s/it]\n",
      "\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5979052186012268, 'eval_runtime': 152.068, 'eval_samples_per_second': 13.593, 'eval_steps_per_second': 1.703, 'epoch': 1.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                        \n",
      " 52%|█████▏    | 1600/3102 [1:05:26<52:05,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3539, 'learning_rate': 2.4210186976144423e-05, 'epoch': 1.55}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 55%|█████▍    | 1700/3102 [1:08:53<48:18,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3427, 'learning_rate': 2.259832366215345e-05, 'epoch': 1.64}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 58%|█████▊    | 1800/3102 [1:12:20<45:02,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2742, 'learning_rate': 2.0986460348162475e-05, 'epoch': 1.74}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 61%|██████▏   | 1900/3102 [1:15:48<41:43,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2958, 'learning_rate': 1.9374597034171506e-05, 'epoch': 1.84}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 64%|██████▍   | 2000/3102 [1:19:15<37:45,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3052, 'learning_rate': 1.776273372018053e-05, 'epoch': 1.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\n",
      "                                                     \n",
      "\u001b[A                                       \n",
      " 64%|██████▍   | 2000/3102 [1:21:47<37:45,  2.06s/it]\n",
      "\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5818384885787964, 'eval_runtime': 151.883, 'eval_samples_per_second': 13.609, 'eval_steps_per_second': 1.705, 'epoch': 1.93}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                        \n",
      " 68%|██████▊   | 2100/3102 [1:25:18<34:23,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3323, 'learning_rate': 1.6150870406189557e-05, 'epoch': 2.03}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 71%|███████   | 2200/3102 [1:28:46<31:23,  2.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1706, 'learning_rate': 1.4539007092198581e-05, 'epoch': 2.13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 74%|███████▍  | 2300/3102 [1:32:13<27:51,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2092, 'learning_rate': 1.292714377820761e-05, 'epoch': 2.22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 77%|███████▋  | 2400/3102 [1:35:41<24:20,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1928, 'learning_rate': 1.1315280464216636e-05, 'epoch': 2.32}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 81%|████████  | 2500/3102 [1:39:07<20:46,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2238, 'learning_rate': 9.703417150225661e-06, 'epoch': 2.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\n",
      "                                                     \n",
      "\u001b[A                                       \n",
      " 81%|████████  | 2500/3102 [1:41:40<20:46,  2.07s/it]\n",
      "\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6637986302375793, 'eval_runtime': 152.045, 'eval_samples_per_second': 13.595, 'eval_steps_per_second': 1.703, 'epoch': 2.42}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                       \n",
      " 84%|████████▍ | 2600/3102 [1:45:14<17:15,  2.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1736, 'learning_rate': 8.091553836234687e-06, 'epoch': 2.51}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 87%|████████▋ | 2700/3102 [1:48:42<13:51,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2078, 'learning_rate': 6.479690522243714e-06, 'epoch': 2.61}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 90%|█████████ | 2800/3102 [1:52:09<10:25,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.1574, 'learning_rate': 4.867827208252741e-06, 'epoch': 2.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 93%|█████████▎| 2900/3102 [1:55:37<06:57,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2115, 'learning_rate': 3.255963894261767e-06, 'epoch': 2.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 97%|█████████▋| 3000/3102 [1:59:04<03:32,  2.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.147, 'learning_rate': 1.644100580270793e-06, 'epoch': 2.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\n",
      "                                                     \n",
      "\u001b[A                                       \n",
      " 97%|█████████▋| 3000/3102 [2:01:38<03:32,  2.08s/it]\n",
      "\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7667654156684875, 'eval_runtime': 153.515, 'eval_samples_per_second': 13.464, 'eval_steps_per_second': 1.687, 'epoch': 2.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                       \n",
      "100%|█████████▉| 3100/3102 [2:05:16<00:04,  2.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.182, 'learning_rate': 3.2237266279819476e-08, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      "100%|██████████| 3102/3102 [2:05:19<00:00,  2.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 7519.6848, 'train_samples_per_second': 3.298, 'train_steps_per_second': 0.413, 'train_loss': 0.3369909449403014, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=3102, training_loss=0.3369909449403014, metrics={'train_runtime': 7519.6848, 'train_samples_per_second': 3.298, 'train_steps_per_second': 0.413, 'train_loss': 0.3369909449403014, 'epoch': 3.0})"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 259/259 [02:31<00:00,  1.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7732343077659607, 'eval_runtime': 152.402, 'eval_samples_per_second': 13.563, 'eval_steps_per_second': 1.699, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model\n",
    "results = trainer.evaluate()\n",
    "\n",
    "# Print evaluation results\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./fine_tuned_model\\\\tokenizer_config.json',\n",
       " './fine_tuned_model\\\\special_tokens_map.json',\n",
       " './fine_tuned_model\\\\vocab.json',\n",
       " './fine_tuned_model\\\\merges.txt',\n",
       " './fine_tuned_model\\\\added_tokens.json',\n",
       " './fine_tuned_model\\\\tokenizer.json')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the fine-tuned model\n",
    "trainer.save_model(\"./fine_tuned_model\")\n",
    "\n",
    "# Save the tokenizer\n",
    "tokenizer.save_pretrained(\"./fine_tuned_model\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 259/259 [02:32<00:00,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 81.8577648766328\n",
      "Precision: 82.02415819804327\n",
      "Recall: 81.8577648766328\n",
      "F1-score: 81.84955953514542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "# Evaluate the model\n",
    "predictions = trainer.predict(val_dataset)\n",
    "predicted_labels = predictions.predictions.argmax(-1)\n",
    "true_labels = val_dataset['label']\n",
    "\n",
    "# Compute evaluation metrics\n",
    "accuracy = accuracy_score(true_labels, predicted_labels)\n",
    "precision = precision_score(true_labels, predicted_labels, average='weighted')\n",
    "recall = recall_score(true_labels, predicted_labels, average='weighted')\n",
    "f1 = f1_score(true_labels, predicted_labels, average='weighted')\n",
    "\n",
    "accuracy_percentage = accuracy * 100\n",
    "precision_percentage = precision * 100\n",
    "recall_percentage = recall * 100\n",
    "f1_percentage = f1 * 100\n",
    "\n",
    "print(\"Accuracy:\", accuracy_percentage)\n",
    "print(\"Precision:\", precision_percentage)\n",
    "print(\"Recall:\", recall_percentage)\n",
    "print(\"F1-score:\", f1_percentage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 81.8577648766328\n",
      "Precision: 82.02415819804327\n",
      "Recall: 81.8577648766328\n",
      "F1-score: 81.84955953514542\n",
      "----------------------------\n",
      "Sentence is grammatically correct.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import RobertaTokenizerFast, RobertaForSequenceClassification\n",
    "\n",
    "# Load the fine-tuned model and tokenizer\n",
    "model_path = \"./fine_tuned_model\"\n",
    "tokenizer = RobertaTokenizerFast.from_pretrained(model_path)\n",
    "model = RobertaForSequenceClassification.from_pretrained(model_path)\n",
    "\n",
    "# Set the model to evaluation mode\n",
    "model.eval()\n",
    "\n",
    "# Sample sentence for testing\n",
    "sample_sentence = \"Takbo sya nang takbo kanina kaya sya nadapa\"\n",
    "\n",
    "# Tokenize the input sentence\n",
    "inputs = tokenizer(sample_sentence, return_tensors=\"pt\")\n",
    "\n",
    "# Forward pass through the model\n",
    "with torch.no_grad():\n",
    "    outputs = model(**inputs)\n",
    "\n",
    "# Get the predicted class (label) from the model output\n",
    "predicted_class = torch.argmax(outputs.logits).item()\n",
    "\n",
    "print(\"Accuracy:\", accuracy_percentage)\n",
    "print(\"Precision:\", precision_percentage)\n",
    "print(\"Recall:\", recall_percentage)\n",
    "print(\"F1-score:\", f1_percentage)\n",
    "print(\"----------------------------\")\n",
    "\n",
    "if predicted_class == 1:\n",
    "    print(\"Sentence is grammatically correct.\")\n",
    "else:\n",
    "    print(\"Sentence is grammatically wrong.\")\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}