finetune_toxictweets.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Toxic Tweets Finetuning\n",
        "\n",
        "This code is run on colab and finetunes tweets according to the toxic tweets kaggle dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YQqdqC2IJ6mZ",
        "outputId": "0cee2ef3-14ed-4c8b-ad27-4e30d84b1c56"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.28.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.11.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n"
          ]
        }
      ],
      "source": [
        "# !ls\n",
        "# !pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EbJOwNb7UTVf",
        "outputId": "b9b072d4-9a32-4a9e-899d-ffbd80bb8b6e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "# PATH = \"/content/drive/MyDrive/data\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AbuSkKXDKoJ7"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "The below defines a custom dataset class ToxicCommentsDataset that inherits from torch.utils.data.Dataset. It takes in the following arguments:\n",
        "\n",
        "comments: The list of comments to be used as input\n",
        "labels: The list of labels corresponding to each comment\n",
        "tokenizer: The tokenizer to be used to preprocess the comments\n",
        "max_length: The maximum length of the tokenized comments\n",
        "The class implements the __len__ and __getitem__ methods required for PyTorch datasets. In the __getitem__ method, each comment is tokenized using the provided tokenizer, truncated to max_length, and padded to max_length using the padding argument. The resulting token IDs, attention mask, and label are returned as a dictionary with keys 'input_ids', 'attention_mask', and 'labels', respectively."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AO-AiK4aNBgh"
      },
      "outputs": [],
      "source": [
        "# Create a custom dataset class for the comments and labels\n",
        "class ToxicCommentsDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, comments, labels, tokenizer, max_length):\n",
        "        self.comments = comments\n",
        "        self.labels = labels\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_length = max_length\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.comments)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        comment =  str(self.comments[index])\n",
        "        label = self.labels[index]\n",
        "\n",
        "        encoding = self.tokenizer.encode_plus(\n",
        "            comment,\n",
        "            add_special_tokens=True,\n",
        "            truncation=True,\n",
        "            max_length=self.max_length,\n",
        "            return_token_type_ids=False,\n",
        "            padding='max_length',\n",
        "            return_attention_mask=True,\n",
        "            return_tensors='pt'\n",
        "        )\n",
        "\n",
        "        return {\n",
        "            'input_ids': encoding['input_ids'].flatten(),\n",
        "            'attention_mask': encoding['attention_mask'].flatten(),\n",
        "            'labels': torch.tensor(label, dtype=torch.float32)\n",
        "        }"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nGmuzGHQXEeX"
      },
      "source": [
        "### loading train and test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JPbtgyxsZKlD"
      },
      "outputs": [],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "# training parameters\n",
        "batch_size = 8\n",
        "num_epochs = 10\n",
        "learning_rate = 0.0001\n",
        "max_length = 512"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Load training from a CSV file located at /content/drive/MyDrive/data/train.csv into a pandas DataFrame train_texts_df. It then randomly samples 80% of the rows from the DataFrame using sample method and sets them as the training data.\n",
        "\n",
        "Next, the code sets the training labels by extracting the 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate' columns from train_texts_df and converting them to a list using the values.tolist() method. The comments are also extracted from the 'comment_text' column of train_texts_df and stored in train_comments.\n",
        "\n",
        "The code then loads the pre-trained DistilBERT model and tokenizer from the Hugging Face Transformers library using the DistilBertForSequenceClassification.from_pretrained and DistilBertTokenizerFast.from_pretrained methods, respectively. The num_labels argument is set to 6 to indicate that the model should be trained for multi-label classification.\n",
        "\n",
        "Finally, the train_comments and train_labels lists, along with the tokenizer and max_length, are passed to the ToxicCommentsDataset class to create the train_dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dYhlIni5XA2y",
        "outputId": "b0edeae3-8871-4fa7-ebd8-7ec2591faa5e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.weight', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight', 'vocab_transform.weight', 'vocab_projector.bias', 'vocab_transform.bias']\n",
            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'classifier.bias', 'classifier.weight', 'pre_classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "train_texts_df = pd.read_csv('/content/drive/MyDrive/data/train.csv')\n",
        "\n",
        "train_texts_df = train_texts_df.sample(frac=0.8, random_state=42)\n",
        "\n",
        "# set the training labels:\n",
        "train_labels = train_texts_df[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].values.tolist()\n",
        "train_comments = train_texts_df['comment_text'].tolist()\n",
        "\n",
        "# Load the pre-trained DistilBERT model and tokenizer\n",
        "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=6)\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "train_dataset = ToxicCommentsDataset(train_comments, train_labels, tokenizer, max_length=512)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Then you loads the test data from two CSV files located at /content/drive/MyDrive/data/test_labels.csv and /content/drive/MyDrive/data/test.csv into pandas DataFrames test_labels and test_data, respectively.\n",
        "\n",
        "Next, the code filters out any rows in test_labels that contain -1 using the any method and creates a boolean mask for those rows using the ~ operator. The filtered test_labels DataFrame is created by applying the mask using the loc method.\n",
        "\n",
        "The code then modifies test_data to only include rows where the id column exists in test_labels_filtered. This is done using the isin method on the id column of test_data.\n",
        "\n",
        "After that, the code randomly samples 50% of the rows from test_data_filtered and test_labels_filtered using the sample method with frac=0.5 and random_state=33.\n",
        "\n",
        "Finally, the toxic, severe_toxic, obscene, threat, insult, and identity_hate columns are extracted from test_labels_filtered and converted to a list of lists using the values.tolist() method. The comments are extracted from the comment_text column of test_data_filtered and stored in test_comments."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iqnX6265NGW2",
        "outputId": "dcbda026-1793-4323-e000-6122a8aa615b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                      id  toxic  severe_toxic  obscene  threat  insult  \\\n",
            "128700  d718f29ed43fa5e7      1             0        0       0       0   \n",
            "23627   27661a70fa723a71      0             0        0       0       0   \n",
            "7664    0cd773ed62c92549      0             0        0       0       0   \n",
            "110519  b854eec6e725eb7b      0             0        0       0       0   \n",
            "66792   6f3502e118fb6d0e      0             0        0       0       0   \n",
            "\n",
            "        identity_hate  \n",
            "128700              0  \n",
            "23627               0  \n",
            "7664                0  \n",
            "110519              0  \n",
            "66792               0  \n",
            "                      id                                       comment_text\n",
            "128700  d718f29ed43fa5e7                       == I Hope You Die == \\n\\n :)\n",
            "23627   27661a70fa723a71  *Support as long as Cheyenne (Jason Derulo son...\n",
            "7664    0cd773ed62c92549         :::Consensus has not yet been established.\n",
            "110519  b854eec6e725eb7b  \" \\n :Heh, this is one of those weird things w...\n",
            "66792   6f3502e118fb6d0e  ::I'm concerned about some of the above.   For...\n"
          ]
        }
      ],
      "source": [
        "test_labels = pd.read_csv('/content/drive/MyDrive/data/test_labels.csv')\n",
        "test_data = pd.read_csv('/content/drive/MyDrive/data/test.csv')\n",
        "\n",
        "# Filter out rows with -1 in test_labels\n",
        "mask = ~(test_labels == -1).any(axis=1)\n",
        "test_labels_filtered = test_labels.loc[mask]\n",
        "\n",
        "# modify test_data to only include data in which id also exists in test_labels_filtered\n",
        "test_data_filtered = test_data[test_data['id'].isin(test_labels_filtered['id'])]\n",
        "\n",
        "\n",
        "# randomly sample 10% of the data\n",
        "test_data_filtered = test_data_filtered.sample(frac=0.5, random_state=33)\n",
        "test_labels_filtered = test_labels_filtered.sample(frac=0.5, random_state=33)\n",
        "\n",
        "print(test_labels_filtered.head())\n",
        "print(test_data_filtered.head())\n",
        "\n",
        "# set the test labels:\n",
        "test_labels = test_labels_filtered[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].values.tolist()\n",
        "test_comments = test_data_filtered['comment_text'].tolist()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7YzsITMNXG4i"
      },
      "source": [
        "### Setting the model up"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CzPtlnoiLmWo"
      },
      "outputs": [],
      "source": [
        "# Define the optimizer and the loss function\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
        "loss_fn = torch.nn.BCEWithLogitsLoss()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JYhXkBWjpSvu"
      },
      "outputs": [],
      "source": [
        "from torch.cuda.amp import autocast\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "This code trains a toxicity classification model using a custom dataset class called ToxicCommentsDataset and the pre-trained DistilBERT model and tokenizer. The model is trained on a training set and the training process is displayed using a plot of the loss function. The trained model is saved to disk. The code also prepares a filtered test set for evaluation of the trained model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Train the model\n",
        "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "# set device again just in case\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model.to(device)\n",
        "\n",
        "# losses to plot\n",
        "train_losses = []\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "    running_loss = 0.0\n",
        "    stop = int(0.4*len(train_loader))\n",
        "    for i, batch in enumerate(tqdm(train_loader)):\n",
        "        # early stoppping\n",
        "        if i == stop: break\n",
        "        # move tensors to gpu\n",
        "        input_ids = batch['input_ids'].to(device)\n",
        "        attention_mask = batch['attention_mask'].to(device)\n",
        "        labels = batch['labels'].to(device)\n",
        "\n",
        "        # zero grads\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # use autocast for mixed precision\n",
        "        with autocast():\n",
        "            outputs = model(input_ids.to(device), attention_mask=attention_mask.to(device), labels=labels.to(device))\n",
        "            loss = loss_fn(outputs.logits, labels)\n",
        "\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        running_loss += loss.item()\n",
        "        train_losses.append((i, loss.item()))\n",
        "\n",
        "        if (i+1) % (stop//20) == 0:\n",
        "            print(f'batch {i+1}/{len(train_loader)}, loss: {loss.item():.4f}, running loss: {running_loss:.4f}')\n",
        "            plt.title(f\"epoch:{epoch}, iter:{i}\")\n",
        "            plt.plot(*zip(*train_losses))\n",
        "            plt.ylabel(\"Loss\")\n",
        "            plt.xlabel(\"iter\")\n",
        "            plt.show()\n",
        "            # plt.savefig(f\"/content/drive/MyDrive/data/training_loss_dinner_{epoch}.png\")\n",
        "\n",
        "    torch.save(model.state_dict(), f\"/content/drive/MyDrive/data/toxicity_classifier_dinner_epoch_{epoch+1}.pt\")\n",
        "    \n",
        "    print(f'Epoch {epoch+1}/{num_epochs}, Loss: {loss.item():.4f}, Running Loss {running_loss:.4f}')\n",
        "\n",
        "# Save the trained model\n",
        "model.save_pretrained('/content/drive/MyDrive/data/toxicity_classifier_dinner')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "grWMy6_zOBrC"
      },
      "source": [
        "## test Model eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lSYV8E76O1hY"
      },
      "outputs": [],
      "source": [
        "print(len(test_labels), len(test_comments))"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "In this code block, the saved pretrained model is loaded using the DistilBertForSequenceClassification class from the transformers library. The DistilBertTokenizerFast tokenizer is set up to tokenize the input data. A ToxicCommentsDataset is created using the test comments, test labels, tokenizer, and maximum sequence length. A DataLoader is created using the test dataset and batch size. The model is set to evaluation mode. Lists are created to store the predicted probabilities and true labels for each batch of data. The code iterates over the batches in the test data loader, moving the tensors to the device and disabling gradient computation. The forward pass is performed on the model, and the predicted probabilities are extracted using the sigmoid function. The probabilities and true labels are then appended to the respective lists. A progress update is printed every 20 batches."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BP_G1GB0y9ae"
      },
      "outputs": [],
      "source": [
        "# Load the saved pretrained model\n",
        "model = DistilBertForSequenceClassification.from_pretrained('/content/drive/MyDrive/data/toxicity_classifier')\n",
        "\n",
        "# Set the tokenizer\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "# Create test dataset\n",
        "test_dataset = ToxicCommentsDataset(test_comments, test_labels, tokenizer, max_length)\n",
        "\n",
        "# Create test data loader\n",
        "test_loader = DataLoader(test_dataset, batch_size=batch_size)\n",
        "\n",
        "# Set model to eval mode\n",
        "model.eval()\n",
        "\n",
        "# Create lists to store predictions and true labels\n",
        "preds = []\n",
        "true_labels = []\n",
        "\n",
        "model.to(device)\n",
        "\n",
        "# Iterate over batches in test data loader\n",
        "for i, batch in enumerate(tqdm(test_loader)):\n",
        "    if not (i % 2 == 0):\n",
        "      continue\n",
        "\n",
        "    # Move tensors to device\n",
        "    input_ids = batch['input_ids'].to(device)\n",
        "    attention_mask = batch['attention_mask'].to(device)\n",
        "    labels = batch['labels'].to(device)\n",
        "\n",
        "    # Disable gradient computation\n",
        "    with torch.no_grad():\n",
        "        # Forward pass\n",
        "        outputs = model(input_ids, attention_mask=attention_mask)\n",
        "\n",
        "        # Get predicted probabilities\n",
        "        probs = torch.sigmoid(outputs.logits)\n",
        "\n",
        "    # Append probabilities and true labels to lists\n",
        "    preds += probs.cpu().numpy().tolist()\n",
        "    true_labels += labels.cpu().numpy().tolist()\n",
        "\n",
        "    if i % 20 == 0:\n",
        "        print(f\"Processed {i}/{len(test_loader)} batches\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "This code snippet calculates several evaluation metrics for a toxicity classifier model trained on a dataset of toxic and non-toxic comments. The evaluation metrics calculated are accuracy, precision, recall, and F1 score.\n",
        "\n",
        "First, the predicted probabilities and true labels are flattened into 1D arrays using list comprehensions. Then, a binary label is assigned to each prediction based on a given threshold. If the predicted probability is greater than or equal to the threshold, the label is set to 1, otherwise, it is set to 0.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HPP5iHgd57ag"
      },
      "outputs": [],
      "source": [
        "# !pip install scikit-learn\n",
        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n",
        "\n",
        "print(preds)\n",
        "\n",
        "# Calculate metrics\n",
        "# Flatten the predictions and true labels to 1D arrays\n",
        "flat_preds = [p for sublist in preds for p in sublist]\n",
        "flat_true_labels = [l for sublist in true_labels for l in sublist]\n",
        "print(len(flat_true_labels), len(flat_preds))\n",
        "\n",
        "# Convert predicted probabilities to binary labels based on threshold\n",
        "threshold = 0.66666 # Thresholds for class 0 and class 1\n",
        "preds_binary = []\n",
        "for id in preds:\n",
        "    for prob in id:\n",
        "      if prob >= threshold: preds_binary.append(1)\n",
        "      else: preds_binary.append(0)\n",
        "\n",
        "print(preds_binary)\n",
        "\n",
        "# Calculate metrics for binary predictions\n",
        "accuracy = accuracy_score(flat_true_labels, preds_binary)\n",
        "precision = precision_score(flat_true_labels, preds_binary)\n",
        "recall = recall_score(flat_true_labels, preds_binary)\n",
        "f1 = f1_score(flat_true_labels, preds_binary)\n",
        "\n",
        "print('Accuracy: ', accuracy)\n",
        "print('Precision: ', precision)\n",
        "print('Recall: ', recall)\n",
        "print('F1: ', f1)\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}