added testing code

src/finetune.ipynb

@@ -22,13 +22,14 @@
         "import torch\n",
         "from torch.utils.data import Dataset, DataLoader\n",
         "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
         "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n",
         "from transformers import Trainer, TrainingArguments"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 2,
       "metadata": {
         "id": "6JovL-hI8JKQ"
       },
@@ -58,7 +59,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 3,
       "metadata": {
         "id": "Q0OqpFmZ0xHD"
       },
@@ -117,9 +118,9 @@
       },
       "outputs": [],
       "source": [
-        "# Saving encoded and tokenized data to files\n",
-        "torch.save(train_encodings, '../data/tokenized_encodings/train_encodings.pt')\n",
-        "torch.save(val_encodings, '../data/tokenized_encodings/val_encodings.pt')"
+        "# # Saving encoded and tokenized data to files\n",
+        "# torch.save(train_encodings, '../data/tokenized_encodings/train_encodings.pt')\n",
+        "# torch.save(val_encodings, '../data/tokenized_encodings/val_encodings.pt')"
       ]
     },
     {
@@ -130,9 +131,9 @@
       },
       "outputs": [],
       "source": [
-        "# Creating training and validation datasets\n",
-        "train_encodings = torch.load('../data/tokenized_encodings/train_encodings.pt').to(device)\n",
-        "val_encodings = torch.load('../data/tokenized_encodings/val_encodings.pt').to(device)"
+        "# # Loading encoded and tokenized data from files\n",
+        "# train_encodings = torch.load('../data/tokenized_encodings/train_encodings.pt').to(device)\n",
+        "# val_encodings = torch.load('../data/tokenized_encodings/val_encodings.pt').to(device)"
       ]
     },
     {
@@ -152,6 +153,14 @@
         "model.to(device)"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Training Setup & Process"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -186,6 +195,146 @@
         "# Training model\n",
         "trainer.train()"
       ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "### Testing The Fine-Tuned Model\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Loading test dataset\n",
+        "test_size = 10000\n",
+        "test_df = pd.read_csv(\"../data/raw/test.csv\")\n",
+        "test_label_df = pd.read_csv(\"../data/raw/test_labels.csv\")\n",
+        "\n",
+        "# Comments as list of strings for testing texts\n",
+        "test_texts = test_df[\"comment_text\"].tolist()[:test_size]\n",
+        "# Labels extracted from dataframe as list of lists\n",
+        "test_labels = test_label_df[[\"toxic\",\"severe_toxic\",\"obscene\",\"threat\",\"insult\",\"identity_hate\"]].values.tolist()[:test_size]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Loading model and tokenizer\n",
+        "model = DistilBertForSequenceClassification.from_pretrained(\"sergey-hovhannisyan/fine-tuned-toxic-tweets\")\n",
+        "tokenizer = DistilBertTokenizerFast.from_pretrained(\"distilbert-base-uncased\")\n",
+        "model.eval()\n",
+        "\n",
+        "# Tokenizing and encoding test set\n",
+        "test_encodings = tokenizer.batch_encode_plus(test_texts, truncation=True, padding=True, return_tensors='pt')\n",
+        "\n",
+        "# Creating datasets for testing set\n",
+        "test_dataset = ToxicTweetsDataset(test_encodings, test_labels)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "This code sets a batch size for evaluation, creates a DataLoader object for the test dataset, and then iterates over the batches in the test set. For each batch, it moves the data to the GPU (if available), uses the trained model to make predictions, and then appends the batch predictions and labels to two lists. Finally, it combines all batch predictions and labels into one array each."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Batch size for evaluation\n",
+        "batch_size = 32\n",
+        "\n",
+        "# Create the DataLoader for our test set\n",
+        "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
+        "\n",
+        "# Create lists for preds and labels\n",
+        "all_preds = []\n",
+        "all_labels = []\n",
+        "\n",
+        "# Loop over batches\n",
+        "for batch in test_loader:\n",
+        "    # move batch to GPU if available\n",
+        "    batch = {k: v.to(device) for k, v in batch.items()}\n",
+        "    with torch.no_grad():\n",
+        "        # make predictions\n",
+        "        outputs = model(**batch)\n",
+        "        logits = outputs.logits\n",
+        "        preds = torch.sigmoid(logits)\n",
+        "        preds = (preds > 0.5).int()\n",
+        "    # append predictions and labels to lists\n",
+        "    all_preds.append(preds.cpu().numpy())\n",
+        "    all_labels.append(batch['labels'].cpu().numpy())\n",
+        "\n",
+        "# Combine all predictions and labels\n",
+        "all_preds = np.concatenate(all_preds, axis=0)\n",
+        "all_labels = np.concatenate(all_labels, axis=0)"
+      ]
+    },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "In the current scenario, we are only evaluating the \"toxic\" label column since the test label dataset assigns a value of -1 to all labels if any form of toxicity is detected."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Getting only toxic label from predictions & labels\n",
+        "all_preds_toxic = -1*all_preds[:,0]\n",
+        "all_labels_toxic = all_labels[:,0]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# Calculating metrics\n",
+        "def compute_metrics(labels, predictions):\n",
+        "    accuracy = accuracy_score(labels, predictions)\n",
+        "    precision = precision_score(labels, predictions, average='weighted')\n",
+        "    recall = recall_score(labels, predictions, average='weighted')\n",
+        "    f1 = f1_score(labels, predictions, average='weighted')\n",
+        "    return {\n",
+        "        'accuracy': accuracy,\n",
+        "        'precision': precision,\n",
+        "        'recall': recall,\n",
+        "        'f1': f1 }"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# calculate evaluation metrics\n",
+        "metrics = compute_metrics(all_labels_toxic, all_preds_toxic)\n",
+        "\n",
+        "# print evaluation metrics\n",
+        "print('Recall: ', round(metrics['recall'],4))\n",
+        "print('Precision: ', round(metrics['precision'],4))\n",
+        "print('Accuracy: ', round(metrics['accuracy'],4))\n",
+        "print('F1: ', round(metrics['f1'],4))"
+      ]
     }
   ],
   "metadata": {