initial commit

.gitignore

@@ -0,0 +1,6 @@
+.venv
+*.h5
+models
+dist
+*info
+data

notes/BC_model_base.ipynb

@@ -0,0 +1,332 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "# reading dataset files\n",
+    "muslim_file = \"../data/right/Haqiqatjou.csv\"\n",
+    "muslim_df = pd.read_csv(muslim_file)\n",
+    "\n",
+    "\n",
+    "sanghi_file = \"../data/wrong/MrSinha_.csv\"\n",
+    "sanghi_df = pd.read_csv(sanghi_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# dataset formatting\n",
+    "muslim_df['category'] = 'MUSLIM'\n",
+    "\n",
+    "sanghi_df['category'] = 'SANGHI'\n",
+    "\n",
+    "\n",
+    "frames = [muslim_df, sanghi_df]\n",
+    "final_dataset = pd.concat(frames)\n",
+    "final_dataset = final_dataset.drop(['index'], axis=1)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ref : https://stackoverflow.com/questions/43777243/how-to-split-a-dataframe-in-pandas-in-predefined-percentages\n",
+    "\n",
+    "# 70% train dataset\n",
+    "# train_frames = [muslim_df.sample(frac=0.7), sanghi_df.sample(frac=0.7).reset_index(drop=True)]\n",
+    "train_frames = [muslim_df, sanghi_df]\n",
+    "train_dataset = pd.concat(train_frames)\n",
+    "train_dataset['content'] = train_dataset['content'].astype(str)\n",
+    "\n",
+    "\n",
+    "# 30% test dataset\n",
+    "remaining_frames = [muslim_df.drop(muslim_df.sample(frac=0.7).index), sanghi_df.drop(sanghi_df.sample(frac=0.7).index).reset_index(drop=True)]\n",
+    "test_dataset = pd.concat(remaining_frames)\n",
+    "# test_dataset['content'] = train_dataset['content'].astype(str)\n",
+    "# test_dataset\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-03-22 14:25:54.140036: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+      "2024-03-22 14:25:58.302374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
+     ]
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import os\n",
+    "import re\n",
+    "import shutil\n",
+    "import string\n",
+    "import tensorflow as tf\n",
+    "from keras import layers\n",
+    "from keras import losses\n",
+    "import keras\n",
+    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
+    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Tokenization\n",
+    "tokenizer = Tokenizer()\n",
+    "tokenizer.fit_on_texts(train_dataset['content'])\n",
+    "\n",
+    "vocab_size = len(tokenizer.word_index) + 1\n",
+    "max_length = 100  # adjust as needed\n",
+    "embedding_dim = 50  # adjust as needed\n",
+    "\n",
+    "X = tokenizer.texts_to_sequences(train_dataset['content'])\n",
+    "X = pad_sequences(X, maxlen=max_length, padding='post')\n",
+    "\n",
+    "# Encode labels\n",
+    "labels = train_dataset['category'].map({'MUSLIM': '1', 'SANGHI': 0}).astype('float32').values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/xd/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/keras/src/layers/core/embedding.py:81: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
+      "  super().__init__(**kwargs)\n",
+      "2024-03-21 20:20:29.968292: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.617785: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.618176: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.620147: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.620655: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.620962: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.824877: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.825245: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.825539: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-21 20:20:33.841867: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1928] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 1754 MB memory:  -> device: 0, name: NVIDIA GeForce 920MX, pci bus id: 0000:01:00.0, compute capability: 5.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# build the model\n",
+    "model = keras.Sequential([\n",
+    "    keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_shape=(max_length,)),\n",
+    "    keras.layers.GlobalAveragePooling1D(),\n",
+    "    keras.layers.Dense(16, activation='relu'),\n",
+    "    keras.layers.Dense(1, activation='sigmoid')\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Compile the model\n",
+    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Split into train and test datasets\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.3, random_state=42)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/10\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
+      "I0000 00:00:1711032641.198169   31002 service.cc:145] XLA service 0x7fa54c006910 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
+      "I0000 00:00:1711032641.198244   31002 service.cc:153]   StreamExecutor device (0): NVIDIA GeForce 920MX, Compute Capability 5.0\n",
+      "2024-03-21 20:20:41.877803: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
+      "2024-03-21 20:20:42.791103: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8907\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m34/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━\u001b[0m\u001b[37m━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - accuracy: 0.5261 - loss: 0.6929"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "I0000 00:00:1711032645.847800   31002 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 71ms/step - accuracy: 0.5337 - loss: 0.6922 - val_accuracy: 0.5514 - val_loss: 0.6814\n",
+      "Epoch 2/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step - accuracy: 0.5740 - loss: 0.6781 - val_accuracy: 0.5563 - val_loss: 0.6636\n",
+      "Epoch 3/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step - accuracy: 0.6603 - loss: 0.6492 - val_accuracy: 0.7781 - val_loss: 0.6244\n",
+      "Epoch 4/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step - accuracy: 0.8210 - loss: 0.5806 - val_accuracy: 0.8328 - val_loss: 0.5476\n",
+      "Epoch 5/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 5ms/step - accuracy: 0.8506 - loss: 0.4952 - val_accuracy: 0.7653 - val_loss: 0.4860\n",
+      "Epoch 6/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step - accuracy: 0.8238 - loss: 0.4224 - val_accuracy: 0.8842 - val_loss: 0.4122\n",
+      "Epoch 7/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step - accuracy: 0.9288 - loss: 0.3257 - val_accuracy: 0.8408 - val_loss: 0.3694\n",
+      "Epoch 8/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step - accuracy: 0.9280 - loss: 0.2705 - val_accuracy: 0.9357 - val_loss: 0.3032\n",
+      "Epoch 9/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 6ms/step - accuracy: 0.9178 - loss: 0.2446 - val_accuracy: 0.9357 - val_loss: 0.2772\n",
+      "Epoch 10/10\n",
+      "\u001b[1m46/46\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 7ms/step - accuracy: 0.9724 - loss: 0.1830 - val_accuracy: 0.8826 - val_loss: 0.2798\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<keras.src.callbacks.history.History at 0x7fa5504b6080>"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Train the model\n",
+    "model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m20/20\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - accuracy: 0.8722 - loss: 0.2829\n",
+      "Test Accuracy: 0.8826366662979126\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test)\n",
+    "print(\"Test Accuracy:\", accuracy)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save the model\n",
+    "try:\n",
+    "  model.save(\"../models/muslim-sanghi-binary-classification.keras\")\n",
+    "except FileNotFoundError:\n",
+    "  os.mkdir(\"../models\")\n",
+    "  model.save(\"../models/muslim-sanghi-binary-classification.keras\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 250ms/step\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.3806811"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# use model\n",
+    "def preprocess_text(text):\n",
+    "  # Tokenize the text\n",
+    "  tokenized_text = tokenizer.texts_to_sequences([text])\n",
+    "  # Pad sequences to the same length as training data\n",
+    "  padded_text = pad_sequences(tokenized_text, maxlen=max_length, padding='post')\n",
+    "  return padded_text\n",
+    "\n",
+    "# load model\n",
+    "model = keras.models.load_model(\"../models/muslim-sanghi-binary-classification.keras\")  # Replace \"your_model.h5\" with the path to your trained model\n",
+    "\n",
+    "# Preprocess the custom input text\n",
+    "preprocessed_text = preprocess_text(\"'Kanwariyas are slaves, they're doing slavery' A team member of Pappu shamelessly says it & Pappu happily agrees\")\n",
+    "\n",
+    "# Make predictions\n",
+    "predictions = model.predict(preprocessed_text)\n",
+    "\n",
+    "predictions[0][0]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notes/custom/right_wrong_BC_model.ipynb

@@ -0,0 +1,476 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path as pp\n",
+    "from pandas.core.frame import DataFrame as df\n",
+    "import os\n",
+    "\n",
+    "# reading dataset files\n",
+    "right_dir: pp = pp(\"../data/right\")\n",
+    "wrong_dir: pp = pp(\"../data/wrong\")\n",
+    "\n",
+    "# dataframes\n",
+    "right_df : df = df()\n",
+    "wrong_df : df = df()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# merging right datafiles into a dataframe\n",
+    "rdf_list = []\n",
+    "for file in os.scandir(right_dir.absolute()):\n",
+    "  if os.path.exists(file):\n",
+    "    rdf_list.append(pd.read_csv(file))\n",
+    "right_df = pd.concat(rdf_list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>index</th>\n",
+       "      <th>content</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>Feeling annoyed? Follow this thread</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Thread, why you should vote for BJP. #PhirEKBa...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>The biggest festival of democracy is here! EC ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Slogan competition \\nShare your slogan idea ( ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>4 जून की करो तैयारी,\\n\\nआ रहे हैं भगवाधारी....</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>92</th>\n",
+       "      <td>92</td>\n",
+       "      <td>मैं बता रहा हूँ, बेंगलुरू मामले में भी बहुत ते...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>93</th>\n",
+       "      <td>93</td>\n",
+       "      <td>ईरान में करीब 1200 भारतीय फंसे हैं। 800 छात्र ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>94</th>\n",
+       "      <td>94</td>\n",
+       "      <td>वैसे तो TikTok ने PMCares फंड में LAC पर चीन स...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>95</th>\n",
+       "      <td>95</td>\n",
+       "      <td>बंगाल में भीड़ से खचाखच भरी रैलियां हो सकती है...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>96</th>\n",
+       "      <td>96</td>\n",
+       "      <td>Thank You PM \\n@narendramodi\\n ji for follow b...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1675 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    index                                            content\n",
+       "0       0               Feeling annoyed? Follow this thread \n",
+       "1       1  Thread, why you should vote for BJP. #PhirEKBa...\n",
+       "2       2  The biggest festival of democracy is here! EC ...\n",
+       "3       3  Slogan competition \\nShare your slogan idea ( ...\n",
+       "4       4     4 जून की करो तैयारी,\\n\\nआ रहे हैं भगवाधारी....\n",
+       "..    ...                                                ...\n",
+       "92     92  मैं बता रहा हूँ, बेंगलुरू मामले में भी बहुत ते...\n",
+       "93     93  ईरान में करीब 1200 भारतीय फंसे हैं। 800 छात्र ...\n",
+       "94     94  वैसे तो TikTok ने PMCares फंड मे��� LAC पर चीन स...\n",
+       "95     95  बंगाल में भीड़ से खचाखच भरी रैलियां हो सकती है...\n",
+       "96     96  Thank You PM \\n@narendramodi\\n ji for follow b...\n",
+       "\n",
+       "[1675 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# merging left datafiles into a dataframe\n",
+    "ldf_list = []\n",
+    "for file in os.scandir(wrong_dir.absolute()):\n",
+    "  if os.path.exists(file):\n",
+    "    ldf_list.append(pd.read_csv(file))\n",
+    "wrong_df = pd.concat(ldf_list)\n",
+    "wrong_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# add category to the dataframes\n",
+    "right_df['category'] = 'RIGHT'\n",
+    "wrong_df['category'] = 'WRONG'\n",
+    "wrong_df\n",
+    "\n",
+    "# cleaning up the columns and adding the binray labels to the dataframes\n",
+    "frame = [right_df, wrong_df]\n",
+    "final_dataset: df = pd.concat(frame)\n",
+    "\n",
+    "final_dataset.drop('index', axis=1)\n",
+    "final_dataset\n",
+    "final_dataset['content'] = final_dataset['content'].astype(str)   # all rows in 'content' column must be of type str\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-03-29 19:54:59.568730: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
+      "2024-03-29 19:55:07.003956: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
+     ]
+    }
+   ],
+   "source": [
+    ".to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Tokenization of dataset\n",
+    "tokenizer = Tokenizer()\n",
+    "tokenizer.fit_on_texts(final_dataset['content'])\n",
+    "\n",
+    "vocab_size = len(tokenizer.word_index) + 1    # our dataset vocab size (space split)\n",
+    "max_length = 200    # max words in a sentence\n",
+    "embedding_dim = 50    # TODO: need to adjust accordingly\n",
+    "\n",
+    "X = tokenizer.texts_to_sequences(final_dataset['content'])\n",
+    "X = pad_sequences(X, maxlen=max_length, padding='post')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Encode the lables\n",
+    "labels = final_dataset['category'].map({'RIGHT': '1', 'WRONG': '0'}).astype('float32').values\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/xd/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/keras/src/layers/core/embedding.py:81: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n",
+      "  super().__init__(**kwargs)\n",
+      "2024-03-29 19:55:16.544096: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.152411: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.152806: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.154795: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.155560: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.156123: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.347819: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.348262: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.348616: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:998] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
+      "2024-03-29 19:55:21.365839: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1928] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 1754 MB memory:  -> device: 0, name: NVIDIA GeForce 920MX, pci bus id: 0000:01:00.0, compute capability: 5.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Build the model\n",
+    "model = keras.Sequential([\n",
+    "  keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_shape=(max_length,)),\n",
+    "  keras.layers.GlobalAveragePooling1D(),\n",
+    "  keras.layers.Dense(16, activation='relu'),\n",
+    "  keras.layers.Dense(1, activation='sigmoid')\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Compile the model\n",
+    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(3140, 200)\n",
+      "(1347, 200)\n",
+      "(3140,)\n",
+      "(1347,)\n",
+      "(4487,)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# split the dataset into train and test\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.3, random_state=42, shuffle=True)\n",
+    "print(X_train.shape)\n",
+    "print(X_test.shape)\n",
+    "print(y_train.shape)\n",
+    "print(y_test.shape)\n",
+    "\n",
+    "print(labels.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/10\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
+      "I0000 00:00:1711722330.932635   76533 service.cc:145] XLA service 0x7bbdd40053b0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
+      "I0000 00:00:1711722330.932689   76533 service.cc:153]   StreamExecutor device (0): NVIDIA GeForce 920MX, Compute Capability 5.0\n",
+      "2024-03-29 19:55:31.370178: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n",
+      "2024-03-29 19:55:33.092199: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:465] Loaded cuDNN version 8907\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m17/99\u001b[0m \u001b[32m━━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 7ms/step - accuracy: 0.5972 - loss: 0.6806"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "I0000 00:00:1711722338.226902   76533 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 56ms/step - accuracy: 0.6186 - loss: 0.6631 - val_accuracy: 0.6511 - val_loss: 0.6453\n",
+      "Epoch 2/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 7ms/step - accuracy: 0.6242 - loss: 0.6554 - val_accuracy: 0.6511 - val_loss: 0.6393\n",
+      "Epoch 3/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 7ms/step - accuracy: 0.6134 - loss: 0.6635 - val_accuracy: 0.6511 - val_loss: 0.6375\n",
+      "Epoch 4/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 7ms/step - accuracy: 0.6270 - loss: 0.6466 - val_accuracy: 0.6511 - val_loss: 0.6305\n",
+      "Epoch 5/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 7ms/step - accuracy: 0.6063 - loss: 0.6464 - val_accuracy: 0.6548 - val_loss: 0.6168\n",
+      "Epoch 6/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 7ms/step - accuracy: 0.6640 - loss: 0.6361 - val_accuracy: 0.6600 - val_loss: 0.5972\n",
+      "Epoch 7/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 8ms/step - accuracy: 0.6912 - loss: 0.5938 - val_accuracy: 0.7053 - val_loss: 0.5723\n",
+      "Epoch 8/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 7ms/step - accuracy: 0.7263 - loss: 0.5345 - val_accuracy: 0.5880 - val_loss: 0.6528\n",
+      "Epoch 9/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 6ms/step - accuracy: 0.7473 - loss: 0.5100 - val_accuracy: 0.7669 - val_loss: 0.5012\n",
+      "Epoch 10/10\n",
+      "\u001b[1m99/99\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 6ms/step - accuracy: 0.8396 - loss: 0.4156 - val_accuracy: 0.7194 - val_loss: 0.5071\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<keras.src.callbacks.history.History at 0x7bbdc43be710>"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# train the model\n",
+    "model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m43/43\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 4ms/step - accuracy: 0.7254 - loss: 0.5047\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.7193763852119446"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test)\n",
+    "accuracy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save the model\n",
+    "try:\n",
+    "  model.save(\"../models/right-wrong-BC.keras\")\n",
+    "except FileNotFoundError:\n",
+    "  os.mkdir(\"../models\")\n",
+    "  model.save(\"../models/right-wrong-BC.keras\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 347ms/step\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.71704614"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# use model\n",
+    "def preprocess_text(text):\n",
+    "  # Tokenize the text\n",
+    "  tokenized_text = tokenizer.texts_to_sequences([text])\n",
+    "  # Pad sequences to the same length as training data\n",
+    "  padded_text = pad_sequences(tokenized_text, maxlen=max_length, padding='post')\n",
+    "  return padded_text\n",
+    "\n",
+    "# load model\n",
+    "model = keras.models.load_model(\"../models/right-wrong-BC.keras\")  # Replace \"your_model.h5\" with the path to your trained model\n",
+    "\n",
+    "# Preprocess the custom input text\n",
+    "preprocessed_text = preprocess_text(\"Modi ji is Moon Pappu on Bangkok honeymoon\")\n",
+    "\n",
+    "# Make predictions\n",
+    "predictions = model.predict(preprocessed_text)\n",
+    "\n",
+    "predictions[0][0]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notes/custom/sentimental_analysis_model.ipynb

@@ -0,0 +1,324 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdatasets\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_dataset\n\u001b[1;32m      3\u001b[0m dataset \u001b[38;5;241m=\u001b[39m load_dataset(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimsoumyaneel/sentiment-analysis-llama2\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[0;32m~/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/datasets/__init__.py:18\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# ruff: noqa\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;66;03m# See the License for the specific language governing permissions and\u001b[39;00m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;66;03m# limitations under the License.\u001b[39;00m\n\u001b[1;32m     16\u001b[0m __version__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.18.0\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01marrow_dataset\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Dataset\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01marrow_reader\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ReadInstruction\n\u001b[1;32m     20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbuilder\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder\n",
+      "File \u001b[0;32m~/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:59\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mfsspec\u001b[39;00m\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[1;32m     60\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpyarrow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpa\u001b[39;00m\n\u001b[1;32m     61\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpyarrow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompute\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpc\u001b[39;00m\n",
+      "File \u001b[0;32m~/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/pandas/__init__.py:26\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m _hard_dependencies, _dependency, _missing_dependencies\n\u001b[1;32m     24\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     25\u001b[0m     \u001b[38;5;66;03m# numpy compat\u001b[39;00m\n\u001b[0;32m---> 26\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     27\u001b[0m         is_numpy_dev \u001b[38;5;28;01mas\u001b[39;00m _is_numpy_dev,  \u001b[38;5;66;03m# pyright: ignore[reportUnusedImport] # noqa: F401\u001b[39;00m\n\u001b[1;32m     28\u001b[0m     )\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m _err:  \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m     30\u001b[0m     _module \u001b[38;5;241m=\u001b[39m _err\u001b[38;5;241m.\u001b[39mname\n",
+      "File \u001b[0;32m~/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/pandas/compat/__init__.py:27\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompressors\u001b[39;00m\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m is_numpy_dev\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpyarrow\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m     28\u001b[0m     pa_version_under10p1,\n\u001b[1;32m     29\u001b[0m     pa_version_under11p0,\n\u001b[1;32m     30\u001b[0m     pa_version_under13p0,\n\u001b[1;32m     31\u001b[0m     pa_version_under14p0,\n\u001b[1;32m     32\u001b[0m     pa_version_under14p1,\n\u001b[1;32m     33\u001b[0m )\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m TYPE_CHECKING:\n\u001b[1;32m     36\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_typing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m F\n",
+      "File \u001b[0;32m~/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/pandas/compat/pyarrow.py:8\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutil\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mversion\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Version\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m----> 8\u001b[0m     \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpyarrow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpa\u001b[39;00m\n\u001b[1;32m     10\u001b[0m     _palv \u001b[38;5;241m=\u001b[39m Version(Version(pa\u001b[38;5;241m.\u001b[39m__version__)\u001b[38;5;241m.\u001b[39mbase_version)\n\u001b[1;32m     11\u001b[0m     pa_version_under10p1 \u001b[38;5;241m=\u001b[39m _palv \u001b[38;5;241m<\u001b[39m Version(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m10.0.1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[0;32m~/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/pyarrow/__init__.py:65\u001b[0m\n\u001b[1;32m     63\u001b[0m _gc_enabled \u001b[38;5;241m=\u001b[39m _gc\u001b[38;5;241m.\u001b[39misenabled()\n\u001b[1;32m     64\u001b[0m _gc\u001b[38;5;241m.\u001b[39mdisable()\n\u001b[0;32m---> 65\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpyarrow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlib\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_lib\u001b[39;00m\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _gc_enabled:\n\u001b[1;32m     67\u001b[0m     _gc\u001b[38;5;241m.\u001b[39menable()\n",
+      "File \u001b[0;32m<frozen importlib._bootstrap>:404\u001b[0m, in \u001b[0;36mparent\u001b[0;34m(self)\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\"imsoumyaneel/sentiment-analysis-llama2\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pandas.core.frame import DataFrame as df\n",
+    "\n",
+    "train_dataset = df(dataset['train'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sentence</th>\n",
+       "      <th>label</th>\n",
+       "      <th>text</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>I'll throw out the garbage .</td>\n",
+       "      <td>neutral</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>So Dick , how about getting some coffee for to...</td>\n",
+       "      <td>joy</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Come on , you can at least try a little , besi...</td>\n",
+       "      <td>neutral</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>What ’ s wrong with that ? Cigarette is the th...</td>\n",
+       "      <td>anger</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Not for me , Dick .</td>\n",
+       "      <td>neutral</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>598293</th>\n",
+       "      <td>You got banned for participating in a brigade.</td>\n",
+       "      <td>sadness</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>598294</th>\n",
+       "      <td>A joke is subjective pal, second of all you ne...</td>\n",
+       "      <td>joy</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>598295</th>\n",
+       "      <td>Well, I'm glad you're out of all that now. How...</td>\n",
+       "      <td>joy</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>598296</th>\n",
+       "      <td>Everyone likes [NAME].</td>\n",
+       "      <td>love</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>598297</th>\n",
+       "      <td>The FDA has plenty to criticize. But like here...</td>\n",
+       "      <td>anger</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>598298 rows × 3 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                 sentence    label  \\\n",
+       "0                            I'll throw out the garbage .  neutral   \n",
+       "1       So Dick , how about getting some coffee for to...      joy   \n",
+       "2       Come on , you can at least try a little , besi...  neutral   \n",
+       "3       What ’ s wrong with that ? Cigarette is the th...    anger   \n",
+       "4                                     Not for me , Dick .  neutral   \n",
+       "...                                                   ...      ...   \n",
+       "598293     You got banned for participating in a brigade.  sadness   \n",
+       "598294  A joke is subjective pal, second of all you ne...      joy   \n",
+       "598295  Well, I'm glad you're out of all that now. How...      joy   \n",
+       "598296                             Everyone likes [NAME].     love   \n",
+       "598297  The FDA has plenty to criticize. But like here...    anger   \n",
+       "\n",
+       "                                                     text  \n",
+       "0       ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "1       ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "2       ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "3       ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "4       ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "...                                                   ...  \n",
+       "598293  ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "598294  ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "598295  ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "598296  ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "598297  ###Human:\\nyou are a sentiment analist. guess ...  \n",
+       "\n",
+       "[598298 rows x 3 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# imports for model creation\n",
+    "import tensorflow as tf\n",
+    "from keras import layers\n",
+    "from keras import losses\n",
+    "import keras\n",
+    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
+    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Tokenization of dataset\n",
+    "tokenizer = Tokenizer()\n",
+    "tokenizer.fit_on_texts(train_dataset['sentence'])\n",
+    "\n",
+    "vocab_size = len(tokenizer.word_index) + 1    # our dataset vocab size (space split)\n",
+    "max_length = 200    # max words in a sentence\n",
+    "embedding_dim = 50    # TODO: need to adjust accordingly\n",
+    "\n",
+    "X = tokenizer.texts_to_sequences(train_dataset['sentence'])\n",
+    "X = pad_sequences(X, maxlen=max_length, padding='post')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Encode the lables\n",
+    "labels = train_dataset['label'].map({'neutral': '1', 'joy': '2', 'sadness': '3', 'anger': '4', 'fear': '5', 'love': '6', 'surprise': '7'}).astype('float32').values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Build the model\n",
+    "model = keras.Sequential([\n",
+    "  keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_shape=(max_length,)),\n",
+    "  keras.layers.GlobalAveragePooling1D(),\n",
+    "  keras.layers.Dense(16, activation='relu'),\n",
+    "  keras.layers.Dense(1, activation='sigmoid')\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Compile the model\n",
+    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# split the dataset into train and test\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.3, random_state=42, shuffle=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# train the model\n",
+    "model.fit(X_train, y_train, epochs=100, batch_size=32, validation_data=(X_test, y_test))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test)\n",
+    "accuracy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save the model\n",
+    "try:\n",
+    "  model.save(\"../models/sentimental-analysis-llama2.keras\")\n",
+    "except FileNotFoundError:\n",
+    "  os.mkdir(\"../models\")\n",
+    "  model.save(\"../models/sentimental-analysis-llama2.keras\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notes/dair_ai-emotions_classification.ipynb

@@ -0,0 +1,450 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2b46cd0f-ae6d-4781-8dab-89df1f880ada",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:16.074098Z",
+     "iopub.status.busy": "2024-03-31T14:00:16.073728Z",
+     "iopub.status.idle": "2024-03-31T14:00:16.461502Z",
+     "shell.execute_reply": "2024-03-31T14:00:16.460565Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:16.074074Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No config specified, defaulting to: emotion/split\n",
+      "Reusing dataset emotion (/root/.cache/huggingface/datasets/dair-ai___emotion/split/1.0.0/cca5efe2dfeb58c1d098e0f9eeb200e9927d889b5a03c67097275dfb5fe463bd)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c3b3103a90124dfea0bbec49829f8746",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/3 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['text', 'label'],\n",
+       "        num_rows: 16000\n",
+       "    })\n",
+       "    validation: Dataset({\n",
+       "        features: ['text', 'label'],\n",
+       "        num_rows: 2000\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['text', 'label'],\n",
+       "        num_rows: 2000\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "dataset = load_dataset(\"dair-ai/emotion\")\n",
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "cbdbcb02-bcc5-4928-8eff-e238437f004b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:16.463534Z",
+     "iopub.status.busy": "2024-03-31T14:00:16.463202Z",
+     "iopub.status.idle": "2024-03-31T14:00:16.498438Z",
+     "shell.execute_reply": "2024-03-31T14:00:16.497492Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:16.463508Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "Int64Index: 18000 entries, 0 to 1999\n",
+      "Data columns (total 2 columns):\n",
+      " #   Column  Non-Null Count  Dtype \n",
+      "---  ------  --------------  ----- \n",
+      " 0   text    18000 non-null  object\n",
+      " 1   label   18000 non-null  int64 \n",
+      "dtypes: int64(1), object(1)\n",
+      "memory usage: 421.9+ KB\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text</th>\n",
+       "      <th>label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>i didnt feel humiliated</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>i can go from feeling so hopeless to so damned...</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>im grabbing a minute to post i feel greedy wrong</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>i am ever feeling nostalgic about the fireplac...</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>i am feeling grouchy</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                text  label\n",
+       "0                            i didnt feel humiliated      0\n",
+       "1  i can go from feeling so hopeless to so damned...      0\n",
+       "2   im grabbing a minute to post i feel greedy wrong      3\n",
+       "3  i am ever feeling nostalgic about the fireplac...      2\n",
+       "4                               i am feeling grouchy      3"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "dataset_train = dataset['train'].to_pandas()\n",
+    "dataset_test = dataset['test'].to_pandas()\n",
+    "\n",
+    "complete_dataset = pd.concat([dataset_train, dataset_test])\n",
+    "\n",
+    "complete_dataset.info()\n",
+    "complete_dataset.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "0cfc926c-70df-4089-9b2e-f201faa223df",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:16.500097Z",
+     "iopub.status.busy": "2024-03-31T14:00:16.499802Z",
+     "iopub.status.idle": "2024-03-31T14:00:16.505424Z",
+     "shell.execute_reply": "2024-03-31T14:00:16.504005Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:16.500071Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# imports for model creation\n",
+    "import tensorflow as tf\n",
+    "from keras import layers\n",
+    "from keras import losses\n",
+    "import keras\n",
+    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
+    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "eb4bab6b-ae99-4fae-bf0b-ca91be630db3",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:16.508818Z",
+     "iopub.status.busy": "2024-03-31T14:00:16.508418Z",
+     "iopub.status.idle": "2024-03-31T14:00:17.259616Z",
+     "shell.execute_reply": "2024-03-31T14:00:17.258634Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:16.508782Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "tokenizer = Tokenizer()\n",
+    "tokenizer.fit_on_texts(complete_dataset['text'])\n",
+    "\n",
+    "vocab_size = len(tokenizer.word_index) + 1\n",
+    "max_length = 200    # max words in a sentence\n",
+    "embedding_dim = 50    # TODO: need to adjust accordingly\n",
+    "\n",
+    "X_train = tokenizer.texts_to_sequences(dataset_train['text'])\n",
+    "X_train = pad_sequences(X_train, maxlen=max_length, padding='post')\n",
+    "\n",
+    "X_test = tokenizer.texts_to_sequences(dataset_test['text'])\n",
+    "X_test = pad_sequences(X_test, maxlen=max_length, padding='post')\n",
+    "\n",
+    "y_train = dataset_train['label']\n",
+    "y_test = dataset_test['label']\n",
+    "\n",
+    "from keras.utils import to_categorical\n",
+    "\n",
+    "num_classes = 6  # Assuming you have 3 classes\n",
+    "y_train_encoded = to_categorical(y_train, num_classes=num_classes)\n",
+    "y_test_encoded = to_categorical(y_test, num_classes=num_classes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d7202d74-95c7-4bb2-aea5-54481dfcafd6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:17.261577Z",
+     "iopub.status.busy": "2024-03-31T14:00:17.261236Z",
+     "iopub.status.idle": "2024-03-31T14:00:17.267983Z",
+     "shell.execute_reply": "2024-03-31T14:00:17.267053Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:17.261539Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(16000, 200)\n",
+      "(2000, 200)\n",
+      "(16000,)\n",
+      "(2000,)\n"
+     ]
+    }
+   ],
+   "source": [
+    "labels = complete_dataset['label']\n",
+    "\n",
+    "print(X_train.shape)\n",
+    "print(X_test.shape)\n",
+    "print(y_train.shape)\n",
+    "print(y_test.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "738e3137-7ea4-4e71-9395-773e537083cf",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:17.269524Z",
+     "iopub.status.busy": "2024-03-31T14:00:17.269276Z",
+     "iopub.status.idle": "2024-03-31T14:00:17.328258Z",
+     "shell.execute_reply": "2024-03-31T14:00:17.327093Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:17.269498Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Build the model\n",
+    "model = keras.Sequential([\n",
+    "  keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_shape=(max_length,)),\n",
+    "  keras.layers.GlobalAveragePooling1D(),\n",
+    "  keras.layers.Dense(32, activation='relu'),\n",
+    "  keras.layers.Dense(6, activation='sigmoid')\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "cd072f39-99e6-44f0-8c7f-106a0055c43b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:17.330501Z",
+     "iopub.status.busy": "2024-03-31T14:00:17.330148Z",
+     "iopub.status.idle": "2024-03-31T14:00:17.340754Z",
+     "shell.execute_reply": "2024-03-31T14:00:17.339909Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:17.330466Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Compile the model\n",
+    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "72ad6548-5d1c-4221-88c7-014dcbaea0ee",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:17.342873Z",
+     "iopub.status.busy": "2024-03-31T14:00:17.342494Z",
+     "iopub.status.idle": "2024-03-31T14:00:17.347797Z",
+     "shell.execute_reply": "2024-03-31T14:00:17.346354Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:17.342836Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# split the dataset into train and test\n",
+    "# from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "# X_train, X_test, y_train, y_test = train_test_split(, labels, test_size=0.3, random_state=42, shuffle=True)\n",
+    "# X_train"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "9267da90-7a84-49d1-94d0-04a2cd3062e0",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:00:43.055635Z",
+     "iopub.status.busy": "2024-03-31T14:00:43.055255Z",
+     "iopub.status.idle": "2024-03-31T14:01:14.962372Z",
+     "shell.execute_reply": "2024-03-31T14:01:14.961429Z",
+     "shell.execute_reply.started": "2024-03-31T14:00:43.055606Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/15\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-03-31 14:12:18.969565: E tensorflow/stream_executor/cuda/cuda_blas.cc:232] failed to create cublas handle: CUBLAS_STATUS_NOT_INITIALIZED\n",
+      "2024-03-31 14:12:18.969636: E tensorflow/stream_executor/cuda/cuda_blas.cc:234] Failure to initialize cublas may be due to OOM (cublas needs some free memory when you initialize it, and your deep-learning framework may have preallocated more than its fair share), or may be because this binary was not built with support for the GPU in your machine.\n",
+      "2024-03-31 14:12:18.969674: W tensorflow/core/framework/op_kernel.cc:1745] OP_REQUIRES failed at matmul_op_impl.h:438 : INTERNAL: Attempting to perform BLAS operation using StreamExecutor without BLAS support\n"
+     ]
+    },
+    {
+     "ename": "InternalError",
+     "evalue": "Graph execution error:\n\nDetected at node 'sequential/dense/MatMul' defined at (most recent call last):\n    File \"/usr/lib/python3.9/runpy.py\", line 197, in _run_module_as_main\n      return _run_code(code, main_globals, None,\n    File \"/usr/lib/python3.9/runpy.py\", line 87, in _run_code\n      exec(code, run_globals)\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/usr/local/lib/python3.9/dist-packages/traitlets/config/application.py\", line 1041, in launch_instance\n      app.start()\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"/usr/local/lib/python3.9/dist-packages/tornado/platform/asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"/usr/lib/python3.9/asyncio/base_events.py\", line 601, in run_forever\n      self._run_once()\n    File \"/usr/lib/python3.9/asyncio/base_events.py\", line 1905, in _run_once\n      handle._run()\n    File \"/usr/lib/python3.9/asyncio/events.py\", line 80, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n      await self.process_one()\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 499, in process_one\n      await dispatch(*args)\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n      await result\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n      reply_content = await reply_content\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/ipkernel.py\", line 383, in do_execute\n      res = shell.run_cell(\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 2885, in run_cell\n      result = self._run_cell(\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 2940, in _run_cell\n      return runner(coro)\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 3139, in run_cell_async\n      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 3318, in run_ast_nodes\n      if await self.run_code(code, result, async_=asy):\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 3378, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_517/2691274767.py\", line 2, in <module>\n      model.fit(X_train, y_train_encoded, epochs=15, batch_size=32, validation_data=(X_test, y_test_encoded))\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1409, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1051, in train_function\n      return step_function(self, iterator)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1040, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1030, in run_step\n      outputs = model.train_step(data)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 889, in train_step\n      y_pred = self(x, training=True)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 490, in __call__\n      return super().__call__(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/base_layer.py\", line 1014, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 92, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/sequential.py\", line 374, in call\n      return super(Sequential, self).call(inputs, training=training, mask=mask)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/functional.py\", line 458, in call\n      return self._run_internal_graph(\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/functional.py\", line 596, in _run_internal_graph\n      outputs = node.layer(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/base_layer.py\", line 1014, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 92, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/layers/core/dense.py\", line 221, in call\n      outputs = tf.matmul(a=inputs, b=self.kernel)\nNode: 'sequential/dense/MatMul'\nAttempting to perform BLAS operation using StreamExecutor without BLAS support\n\t [[{{node sequential/dense/MatMul}}]] [Op:__inference_train_function_727]",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mInternalError\u001b[0m                             Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn [12], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# train the model\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_train_encoded\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m15\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m32\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mX_test\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_test_encoded\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py:67\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:  \u001b[38;5;66;03m# pylint: disable=broad-except\u001b[39;00m\n\u001b[1;32m     66\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m---> 67\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[1;32m     68\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     69\u001b[0m   \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
+      "File \u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/execute.py:54\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     53\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 54\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m pywrap_tfe\u001b[38;5;241m.\u001b[39mTFE_Py_Execute(ctx\u001b[38;5;241m.\u001b[39m_handle, device_name, op_name,\n\u001b[1;32m     55\u001b[0m                                       inputs, attrs, num_outputs)\n\u001b[1;32m     56\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     57\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
+      "\u001b[0;31mInternalError\u001b[0m: Graph execution error:\n\nDetected at node 'sequential/dense/MatMul' defined at (most recent call last):\n    File \"/usr/lib/python3.9/runpy.py\", line 197, in _run_module_as_main\n      return _run_code(code, main_globals, None,\n    File \"/usr/lib/python3.9/runpy.py\", line 87, in _run_code\n      exec(code, run_globals)\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel_launcher.py\", line 17, in <module>\n      app.launch_new_instance()\n    File \"/usr/local/lib/python3.9/dist-packages/traitlets/config/application.py\", line 1041, in launch_instance\n      app.start()\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelapp.py\", line 712, in start\n      self.io_loop.start()\n    File \"/usr/local/lib/python3.9/dist-packages/tornado/platform/asyncio.py\", line 199, in start\n      self.asyncio_loop.run_forever()\n    File \"/usr/lib/python3.9/asyncio/base_events.py\", line 601, in run_forever\n      self._run_once()\n    File \"/usr/lib/python3.9/asyncio/base_events.py\", line 1905, in _run_once\n      handle._run()\n    File \"/usr/lib/python3.9/asyncio/events.py\", line 80, in _run\n      self._context.run(self._callback, *self._args)\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n      await self.process_one()\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 499, in process_one\n      await dispatch(*args)\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n      await result\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n      reply_content = await reply_content\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/ipkernel.py\", line 383, in do_execute\n      res = shell.run_cell(\n    File \"/usr/local/lib/python3.9/dist-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n      return super().run_cell(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 2885, in run_cell\n      result = self._run_cell(\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 2940, in _run_cell\n      return runner(coro)\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n      coro.send(None)\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 3139, in run_cell_async\n      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 3318, in run_ast_nodes\n      if await self.run_code(code, result, async_=asy):\n    File \"/usr/local/lib/python3.9/dist-packages/IPython/core/interactiveshell.py\", line 3378, in run_code\n      exec(code_obj, self.user_global_ns, self.user_ns)\n    File \"/tmp/ipykernel_517/2691274767.py\", line 2, in <module>\n      model.fit(X_train, y_train_encoded, epochs=15, batch_size=32, validation_data=(X_test, y_test_encoded))\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1409, in fit\n      tmp_logs = self.train_function(iterator)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1051, in train_function\n      return step_function(self, iterator)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1040, in step_function\n      outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 1030, in run_step\n      outputs = model.train_step(data)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 889, in train_step\n      y_pred = self(x, training=True)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\", line 490, in __call__\n      return super().__call__(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/base_layer.py\", line 1014, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 92, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/sequential.py\", line 374, in call\n      return super(Sequential, self).call(inputs, training=training, mask=mask)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/functional.py\", line 458, in call\n      return self._run_internal_graph(\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/functional.py\", line 596, in _run_internal_graph\n      outputs = node.layer(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 64, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/engine/base_layer.py\", line 1014, in __call__\n      outputs = call_fn(inputs, *args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\", line 92, in error_handler\n      return fn(*args, **kwargs)\n    File \"/usr/local/lib/python3.9/dist-packages/keras/layers/core/dense.py\", line 221, in call\n      outputs = tf.matmul(a=inputs, b=self.kernel)\nNode: 'sequential/dense/MatMul'\nAttempting to perform BLAS operation using StreamExecutor without BLAS support\n\t [[{{node sequential/dense/MatMul}}]] [Op:__inference_train_function_727]"
+     ]
+    }
+   ],
+   "source": [
+    "# train the model\n",
+    "model.fit(X_train, y_train_encoded, epochs=15, batch_size=32, validation_data=(X_test, y_test_encoded))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "id": "24e17bec-2fbe-400f-9273-a5abe823f193",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T14:01:20.843282Z",
+     "iopub.status.busy": "2024-03-31T14:01:20.842873Z",
+     "iopub.status.idle": "2024-03-31T14:01:21.062240Z",
+     "shell.execute_reply": "2024-03-31T14:01:21.061522Z",
+     "shell.execute_reply.started": "2024-03-31T14:01:20.843253Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "63/63 [==============================] - 0s 2ms/step - loss: 0.5242 - accuracy: 0.8580\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.8579999804496765"
+      ]
+     },
+     "execution_count": 48,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test_encoded)\n",
+    "accuracy"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notes/imsoumyaneel-sentiment_analysis_llama2.ipynb

@@ -0,0 +1,648 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "2b46cd0f-ae6d-4781-8dab-89df1f880ada",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:38:33.012523Z",
+     "iopub.status.busy": "2024-03-31T16:38:33.012195Z",
+     "iopub.status.idle": "2024-03-31T16:38:41.339903Z",
+     "shell.execute_reply": "2024-03-31T16:38:41.339283Z",
+     "shell.execute_reply.started": "2024-03-31T16:38:33.012515Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using custom data configuration imsoumyaneel--sentiment-analysis-llama2-406c8d12ee6e98f7\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Downloading and preparing dataset csv/imsoumyaneel--sentiment-analysis-llama2 to /root/.cache/huggingface/datasets/imsoumyaneel___csv/imsoumyaneel--sentiment-analysis-llama2-406c8d12ee6e98f7/0.0.0/652c3096f041ee27b04d2232d41f10547a8fecda3e284a79a0ec4053c916ef7a...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1b57dcefac4c41bd8afdc69d582d21ae",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "73fbc9718d8540d3a795a552e943aa7f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading data:   0%|          | 0.00/173M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5b85187c04ff42baa213ed0026e2fdd6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "176a9bfaff1b4ff0ae8e6a701054e277",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "0 tables [00:00, ? tables/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/lib/python3.9/dist-packages/datasets/download/streaming_download_manager.py:695: FutureWarning: the 'mangle_dupe_cols' keyword is deprecated and will be removed in a future version. Please take steps to stop the use of 'mangle_dupe_cols'\n",
+      "  return pd.read_csv(xopen(filepath_or_buffer, \"rb\", use_auth_token=use_auth_token), **kwargs)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/imsoumyaneel___csv/imsoumyaneel--sentiment-analysis-llama2-406c8d12ee6e98f7/0.0.0/652c3096f041ee27b04d2232d41f10547a8fecda3e284a79a0ec4053c916ef7a. Subsequent calls will reuse this data.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a74c347ebc614136b0a2ad549ef01e3d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['sentence', 'label', 'text'],\n",
+       "        num_rows: 598298\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "# dataset = load_dataset(\"dair-ai/emotion\")\n",
+    "dataset = load_dataset(\"imsoumyaneel/sentiment-analysis-llama2\")\n",
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "cbdbcb02-bcc5-4928-8eff-e238437f004b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:38:41.341147Z",
+     "iopub.status.busy": "2024-03-31T16:38:41.340979Z",
+     "iopub.status.idle": "2024-03-31T16:38:42.665731Z",
+     "shell.execute_reply": "2024-03-31T16:38:42.665184Z",
+     "shell.execute_reply.started": "2024-03-31T16:38:41.341132Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 598298 entries, 0 to 598297\n",
+      "Data columns (total 4 columns):\n",
+      " #   Column     Non-Null Count   Dtype \n",
+      "---  ------     --------------   ----- \n",
+      " 0   sentence   598298 non-null  object\n",
+      " 1   label      598298 non-null  object\n",
+      " 2   text       598298 non-null  object\n",
+      " 3   new_label  598298 non-null  object\n",
+      "dtypes: object(4)\n",
+      "memory usage: 18.3+ MB\n",
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "Int64Index: 478638 entries, 352227 to 559736\n",
+      "Data columns (total 4 columns):\n",
+      " #   Column     Non-Null Count   Dtype \n",
+      "---  ------     --------------   ----- \n",
+      " 0   sentence   478638 non-null  object\n",
+      " 1   label      478638 non-null  object\n",
+      " 2   text       478638 non-null  object\n",
+      " 3   new_label  478638 non-null  object\n",
+      "dtypes: object(4)\n",
+      "memory usage: 18.3+ MB\n",
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "Int64Index: 119660 entries, 0 to 598297\n",
+      "Data columns (total 4 columns):\n",
+      " #   Column     Non-Null Count   Dtype \n",
+      "---  ------     --------------   ----- \n",
+      " 0   sentence   119660 non-null  object\n",
+      " 1   label      119660 non-null  object\n",
+      " 2   text       119660 non-null  object\n",
+      " 3   new_label  119660 non-null  object\n",
+      "dtypes: object(4)\n",
+      "memory usage: 4.6+ MB\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sentence</th>\n",
+       "      <th>label</th>\n",
+       "      <th>text</th>\n",
+       "      <th>new_label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>I'll throw out the garbage .</td>\n",
+       "      <td>neutral</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>So Dick , how about getting some coffee for to...</td>\n",
+       "      <td>joy</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Come on , you can at least try a little , besi...</td>\n",
+       "      <td>neutral</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>What ’ s wrong with that ? Cigarette is the th...</td>\n",
+       "      <td>anger</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Not for me , Dick .</td>\n",
+       "      <td>neutral</td>\n",
+       "      <td>###Human:\\nyou are a sentiment analist. guess ...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                            sentence    label  \\\n",
+       "0                       I'll throw out the garbage .  neutral   \n",
+       "1  So Dick , how about getting some coffee for to...      joy   \n",
+       "2  Come on , you can at least try a little , besi...  neutral   \n",
+       "3  What ’ s wrong with that ? Cigarette is the th...    anger   \n",
+       "4                                Not for me , Dick .  neutral   \n",
+       "\n",
+       "                                                text new_label  \n",
+       "0  ###Human:\\nyou are a sentiment analist. guess ...         1  \n",
+       "1  ###Human:\\nyou are a sentiment analist. guess ...         0  \n",
+       "2  ###Human:\\nyou are a sentiment analist. guess ...         1  \n",
+       "3  ###Human:\\nyou are a sentiment analist. guess ...         3  \n",
+       "4  ###Human:\\nyou are a sentiment analist. guess ...         1  "
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "complete_dataset = dataset['train'].to_pandas()\n",
+    "complete_dataset['new_label'] = complete_dataset['label'].map({'joy': '0', 'neutral': '1', 'sadness': '2', 'anger': '3', 'fear': '4', 'love': '5', 'surprise': '6'}).values\n",
+    "\n",
+    "train_dataset = complete_dataset.sample(frac=0.8,random_state=200)\n",
+    "test_dataset = complete_dataset.drop(train_dataset.index)\n",
+    "\n",
+    "complete_dataset.info()\n",
+    "train_dataset.info()\n",
+    "test_dataset.info()\n",
+    "\n",
+    "complete_dataset.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "0cfc926c-70df-4089-9b2e-f201faa223df",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:38:42.666938Z",
+     "iopub.status.busy": "2024-03-31T16:38:42.666761Z",
+     "iopub.status.idle": "2024-03-31T16:38:46.106948Z",
+     "shell.execute_reply": "2024-03-31T16:38:46.106267Z",
+     "shell.execute_reply.started": "2024-03-31T16:38:42.666926Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# imports for model creation\n",
+    "import tensorflow as tf\n",
+    "from keras import layers\n",
+    "from keras import losses\n",
+    "import keras\n",
+    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
+    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "eb4bab6b-ae99-4fae-bf0b-ca91be630db3",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:38:46.108849Z",
+     "iopub.status.busy": "2024-03-31T16:38:46.108483Z",
+     "iopub.status.idle": "2024-03-31T16:39:01.033885Z",
+     "shell.execute_reply": "2024-03-31T16:39:01.033311Z",
+     "shell.execute_reply.started": "2024-03-31T16:38:46.108831Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "tokenizer = Tokenizer()\n",
+    "tokenizer.fit_on_texts(complete_dataset['sentence'])\n",
+    "\n",
+    "vocab_size = len(tokenizer.word_index) + 1\n",
+    "max_length = 200    # max words in a sentence\n",
+    "embedding_dim = 50    # TODO: need to adjust accordinglys\n",
+    "\n",
+    "X_train = tokenizer.texts_to_sequences(train_dataset['sentence'])\n",
+    "X_train = pad_sequences(X_train, maxlen=max_length, padding='post')\n",
+    "\n",
+    "X_test = tokenizer.texts_to_sequences(test_dataset['sentence'])\n",
+    "X_test = pad_sequences(X_test, maxlen=max_length, padding='post')\n",
+    "\n",
+    "y_train = train_dataset['new_label']\n",
+    "y_test = test_dataset['new_label']\n",
+    "\n",
+    "from keras.utils import to_categorical\n",
+    "\n",
+    "num_classes = 7  # Assuming you have 3 classes\n",
+    "y_train_encoded = to_categorical(y_train, num_classes=num_classes)\n",
+    "y_test_encoded = to_categorical(y_test, num_classes=num_classes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "d7202d74-95c7-4bb2-aea5-54481dfcafd6",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:39:01.035022Z",
+     "iopub.status.busy": "2024-03-31T16:39:01.034846Z",
+     "iopub.status.idle": "2024-03-31T16:39:01.038541Z",
+     "shell.execute_reply": "2024-03-31T16:39:01.038020Z",
+     "shell.execute_reply.started": "2024-03-31T16:39:01.035006Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(478638, 200)\n",
+      "(119660, 200)\n",
+      "(478638,)\n",
+      "(119660,)\n"
+     ]
+    }
+   ],
+   "source": [
+    "labels = complete_dataset['label']\n",
+    "\n",
+    "print(X_train.shape)\n",
+    "print(X_test.shape)\n",
+    "print(y_train.shape)\n",
+    "print(y_test.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "738e3137-7ea4-4e71-9395-773e537083cf",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:39:01.039687Z",
+     "iopub.status.busy": "2024-03-31T16:39:01.039206Z",
+     "iopub.status.idle": "2024-03-31T16:39:02.148732Z",
+     "shell.execute_reply": "2024-03-31T16:39:02.148025Z",
+     "shell.execute_reply.started": "2024-03-31T16:39:01.039671Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Build the model\n",
+    "model = keras.Sequential([\n",
+    "  keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_shape=(max_length,)),\n",
+    "  keras.layers.GlobalAveragePooling1D(),\n",
+    "  keras.layers.Dense(32, activation='relu'),\n",
+    "  keras.layers.Dense(7, activation='sigmoid')\n",
+    "])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "cd072f39-99e6-44f0-8c7f-106a0055c43b",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:39:02.150108Z",
+     "iopub.status.busy": "2024-03-31T16:39:02.149634Z",
+     "iopub.status.idle": "2024-03-31T16:39:02.159085Z",
+     "shell.execute_reply": "2024-03-31T16:39:02.158478Z",
+     "shell.execute_reply.started": "2024-03-31T16:39:02.150090Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Compile the model\n",
+    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "72ad6548-5d1c-4221-88c7-014dcbaea0ee",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:39:02.160402Z",
+     "iopub.status.busy": "2024-03-31T16:39:02.159930Z",
+     "iopub.status.idle": "2024-03-31T16:39:02.162845Z",
+     "shell.execute_reply": "2024-03-31T16:39:02.162259Z",
+     "shell.execute_reply.started": "2024-03-31T16:39:02.160382Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# split the dataset into train and test\n",
+    "# from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "# X_train, X_test, y_train, y_test = train_test_split(, labels, test_size=0.3, random_state=42, shuffle=True)\n",
+    "# X_train"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "9267da90-7a84-49d1-94d0-04a2cd3062e0",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:39:34.346301Z",
+     "iopub.status.busy": "2024-03-31T16:39:34.345554Z",
+     "iopub.status.idle": "2024-03-31T16:48:40.470989Z",
+     "shell.execute_reply": "2024-03-31T16:48:40.470195Z",
+     "shell.execute_reply.started": "2024-03-31T16:39:34.346268Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/10\n",
+      "14958/14958 [==============================] - 55s 4ms/step - loss: 0.4894 - accuracy: 0.8447 - val_loss: 0.4174 - val_accuracy: 0.8586\n",
+      "Epoch 2/10\n",
+      "14958/14958 [==============================] - 54s 4ms/step - loss: 0.3798 - accuracy: 0.8692 - val_loss: 0.3835 - val_accuracy: 0.8651\n",
+      "Epoch 3/10\n",
+      "14958/14958 [==============================] - 54s 4ms/step - loss: 0.3453 - accuracy: 0.8761 - val_loss: 0.3638 - val_accuracy: 0.8655\n",
+      "Epoch 4/10\n",
+      "14958/14958 [==============================] - 54s 4ms/step - loss: 0.3166 - accuracy: 0.8810 - val_loss: 0.3513 - val_accuracy: 0.8645\n",
+      "Epoch 5/10\n",
+      "14958/14958 [==============================] - 55s 4ms/step - loss: 0.2941 - accuracy: 0.8848 - val_loss: 0.3548 - val_accuracy: 0.8669\n",
+      "Epoch 6/10\n",
+      "14958/14958 [==============================] - 54s 4ms/step - loss: 0.2789 - accuracy: 0.8881 - val_loss: 0.3423 - val_accuracy: 0.8654\n",
+      "Epoch 7/10\n",
+      "14958/14958 [==============================] - 55s 4ms/step - loss: 0.2675 - accuracy: 0.8909 - val_loss: 0.3447 - val_accuracy: 0.8646\n",
+      "Epoch 8/10\n",
+      "14958/14958 [==============================] - 55s 4ms/step - loss: 0.2590 - accuracy: 0.8937 - val_loss: 0.3418 - val_accuracy: 0.8658\n",
+      "Epoch 9/10\n",
+      "14958/14958 [==============================] - 55s 4ms/step - loss: 0.2511 - accuracy: 0.8963 - val_loss: 0.3417 - val_accuracy: 0.8636\n",
+      "Epoch 10/10\n",
+      "14958/14958 [==============================] - 54s 4ms/step - loss: 0.2446 - accuracy: 0.8981 - val_loss: 0.3639 - val_accuracy: 0.8604\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<keras.callbacks.History at 0x7f8c901c6280>"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# train the model\n",
+    "model.fit(X_train, y_train_encoded, epochs=10, batch_size=32, validation_data=(X_test, y_test_encoded))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "24e17bec-2fbe-400f-9273-a5abe823f193",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T16:57:03.280928Z",
+     "iopub.status.busy": "2024-03-31T16:57:03.280193Z",
+     "iopub.status.idle": "2024-03-31T16:57:09.194519Z",
+     "shell.execute_reply": "2024-03-31T16:57:09.193928Z",
+     "shell.execute_reply.started": "2024-03-31T16:57:03.280897Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "3740/3740 [==============================] - 6s 2ms/step - loss: 0.3639 - accuracy: 0.8604\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.8604295253753662"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test_encoded)\n",
+    "accuracy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "15fbbb09-ffdf-41d3-ba11-8877aa2c078e",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T17:01:34.387114Z",
+     "iopub.status.busy": "2024-03-31T17:01:34.386216Z",
+     "iopub.status.idle": "2024-03-31T17:01:34.528910Z",
+     "shell.execute_reply": "2024-03-31T17:01:34.528157Z",
+     "shell.execute_reply.started": "2024-03-31T17:01:34.387078Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# save the model\n",
+    "import os\n",
+    "try:\n",
+    "    model.save(\"../models/sentimental-analysis-llama2.keras\")\n",
+    "except FileNotFoundError:\n",
+    "    os.mkdir(\"../models\")\n",
+    "    model.save(\"../models/sentimental-analysis-llama2.keras\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "792f0680-5a32-4c46-b5b4-eb6795b51aeb",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2024-03-31T17:04:41.901658Z",
+     "iopub.status.busy": "2024-03-31T17:04:41.901124Z",
+     "iopub.status.idle": "2024-03-31T17:04:41.948670Z",
+     "shell.execute_reply": "2024-03-31T17:04:41.948177Z",
+     "shell.execute_reply.started": "2024-03-31T17:04:41.901637Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1/1 [==============================] - 0s 17ms/step\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "array([[0.809063  , 0.78246254, 0.02547726, 0.03657908, 0.00648503,\n",
+       "        0.02069169, 0.07264358]], dtype=float32)"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def preprocess_text(text):\n",
+    "    # Tokenize the text\n",
+    "    tokenized_text = tokenizer.texts_to_sequences([text])\n",
+    "    # Pad sequences to the same length as training data\n",
+    "    padded_text = pad_sequences(tokenized_text, maxlen=max_length, padding='post')\n",
+    "    return padded_text\n",
+    "\n",
+    "# Preprocess the custom input text\n",
+    "preprocessed_text = preprocess_text(\"this is good\")\n",
+    "\n",
+    "# Make predictions\n",
+    "predictions = model.predict(preprocessed_text)\n",
+    "\n",
+    "predictions"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notes/model_utilizer.ipynb

@@ -0,0 +1,84 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
+    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
+    "from tensorflow import keras\n",
+    "from pathlib import Path as pp\n",
+    "\n",
+    "import os\n",
+    "notebook_path = os.getcwd()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ValueError",
+     "evalue": "File not found: filepath=../imsoumyaneel-sentiment_analysis_llama2.keras. Please ensure the file is an accessible `.keras` zip file.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[19], line 15\u001b[0m\n\u001b[1;32m     12\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m padded_text\n\u001b[1;32m     14\u001b[0m \u001b[38;5;66;03m# load model\u001b[39;00m\n\u001b[0;32m---> 15\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mkeras\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m../imsoumyaneel-sentiment_analysis_llama2.keras\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# Replace \"your_model.h5\" with the path to your trained model\u001b[39;00m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;66;03m# Preprocess the custom input text\u001b[39;00m\n\u001b[1;32m     18\u001b[0m preprocessed_text \u001b[38;5;241m=\u001b[39m preprocess_text(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mi love this thing!\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[0;32m~/Documents/models/twitter_model/.venv/lib/python3.10/site-packages/keras/src/saving/saving_api.py:185\u001b[0m, in \u001b[0;36mload_model\u001b[0;34m(filepath, custom_objects, compile, safe_mode)\u001b[0m\n\u001b[1;32m    183\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m legacy_h5_format\u001b[38;5;241m.\u001b[39mload_model_from_hdf5(filepath)\n\u001b[1;32m    184\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mstr\u001b[39m(filepath)\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.keras\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m--> 185\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    186\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFile not found: filepath=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfilepath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    187\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease ensure the file is an accessible `.keras` \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    188\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzip file.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    189\u001b[0m     )\n\u001b[1;32m    190\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    191\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    192\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFile format not supported: filepath=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfilepath\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    193\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mKeras 3 only supports V3 `.keras` files and \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    202\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmight have a different name).\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    203\u001b[0m     )\n",
+      "\u001b[0;31mValueError\u001b[0m: File not found: filepath=../imsoumyaneel-sentiment_analysis_llama2.keras. Please ensure the file is an accessible `.keras` zip file."
+     ]
+    }
+   ],
+   "source": [
+    "tokenizer = Tokenizer()\n",
+    "max_length = 200\n",
+    "models_dir = pp(pp(notebook_path).parent, 'models')\n",
+    "\n",
+    "\n",
+    "# use model\n",
+    "def preprocess_text(text):\n",
+    "  # Tokenize the text\n",
+    "  tokenized_text = tokenizer.texts_to_sequences([text])\n",
+    "  # Pad sequences to the same length as training data\n",
+    "  padded_text = pad_sequences(tokenized_text, maxlen=max_length, padding='post')\n",
+    "  return padded_text\n",
+    "\n",
+    "# load model\n",
+    "model = keras.models.load_model('../imsoumyaneel-sentiment_analysis_llama2.keras')  # Replace \"your_model.h5\" with the path to your trained model\n",
+    "\n",
+    "# Preprocess the custom input text\n",
+    "preprocessed_text = preprocess_text(\"i love this thing!\")\n",
+    "\n",
+    "# Make predictions\n",
+    "predictions = model.predict(preprocessed_text)\n",
+    "\n",
+    "predictions[0][0]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notes/references/reference_for_model_creation.ipynb

@@ -0,0 +1,326 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "\n",
+    "Person: \n",
+    "- my dataset csv has two columns content and category, content is just comments as string and ccategory is the label like positive or nagative\n",
+    "\n",
+    "i want to create a binary classification sentimental analysis model using tensorflow, \n",
+    "i have seperated my dataset into 70-3o train and test dataset\n",
+    "how do i tokenize and create/train my model and test it\n",
+    "\n",
+    "ChatGPT: \n",
+    "- To create a binary classification sentiment analysis model using TensorFlow, you can follow these steps: \n",
+    "1. **Import Necessary Libraries** : Import TensorFlow, NumPy, and any other required libraries. \n",
+    "2. **Load the Dataset** : Load your CSV dataset into memory using libraries like Pandas. \n",
+    "3. **Preprocess the Data** : Tokenize the text data, convert it to sequences, pad sequences to make them uniform in length, and encode the labels. \n",
+    "4. **Build the Model** : Create a neural network model using TensorFlow's Keras API. \n",
+    "5. **Compile the Model** : Specify the loss function, optimizer, and metrics for training. \n",
+    "6. **Train the Model** : Fit the model to your training data. \n",
+    "7. **Evaluate the Model** : Test the trained model on your test dataset to evaluate its performance.\n",
+    "\n",
+    "Here's a Python code example to guide you through these steps:\n",
+    "\n",
+    "```python\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
+    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
+    "\n",
+    "# Step 2: Load the dataset\n",
+    "df = pd.read_csv(\"your_dataset.csv\")\n",
+    "\n",
+    "# Step 3: Preprocess the data\n",
+    "tokenizer = Tokenizer()\n",
+    "tokenizer.fit_on_texts(df['content'])\n",
+    "\n",
+    "vocab_size = len(tokenizer.word_index) + 1\n",
+    "max_length = 100  # adjust as needed\n",
+    "embedding_dim = 50  # adjust as needed\n",
+    "\n",
+    "X = tokenizer.texts_to_sequences(df['content'])\n",
+    "X = pad_sequences(X, maxlen=max_length, padding='post')\n",
+    "\n",
+    "# Encode labels\n",
+    "labels = pd.get_dummies(df['category'])\n",
+    "\n",
+    "# Split into train and test datasets\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.3, random_state=42)\n",
+    "\n",
+    "# Step 4: Build the model\n",
+    "model = tf.keras.Sequential([\n",
+    "    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length),\n",
+    "    tf.keras.layers.GlobalAveragePooling1D(),\n",
+    "    tf.keras.layers.Dense(16, activation='relu'),\n",
+    "    tf.keras.layers.Dense(1, activation='sigmoid')\n",
+    "])\n",
+    "\n",
+    "# Step 5: Compile the model\n",
+    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
+    "\n",
+    "# Step 6: Train the model\n",
+    "model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))\n",
+    "\n",
+    "# Step 7: Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test)\n",
+    "print(\"Test Accuracy:\", accuracy)\n",
+    "```\n",
+    "\n",
+    "\n",
+    "\n",
+    "This code assumes that your labels are already one-hot encoded. Adjust the architecture, hyperparameters, and preprocessing steps as needed for your specific dataset and requirements.\n",
+    "\n",
+    "Person: \n",
+    "- ValueError: Arguments `target` and `output` must have the same shape. Received: target.shape=(None, 2), output.shape=(None, 1)\n",
+    "\n",
+    "ChatGPT: \n",
+    "- The error you're encountering suggests that there's a mismatch between the shape of the target labels and the output of your model. Since you're performing binary classification, the output layer of your model should have one neuron with a sigmoid activation function. However, your target labels seem to have two columns, which might be causing the issue.\n",
+    "\n",
+    "To fix this, you need to ensure that your target labels match the shape expected by the model. Since you're performing binary classification, you only need one output neuron, and your target labels should be encoded accordingly.\n",
+    "\n",
+    "Here's how you can adjust your code to fix the issue:\n",
+    "1. Ensure your target labels are encoded as a single column for binary classification.\n",
+    "2. Modify the output layer of your model to have one neuron with a sigmoid activation function.\n",
+    "3. Use binary cross-entropy as the loss function.\n",
+    "\n",
+    "Here's the modified code:\n",
+    "\n",
+    "```python\n",
+    "# Step 2: Load the dataset\n",
+    "df = pd.read_csv(\"your_dataset.csv\")\n",
+    "\n",
+    "# Step 3: Preprocess the data\n",
+    "tokenizer = Tokenizer()\n",
+    "tokenizer.fit_on_texts(df['content'])\n",
+    "\n",
+    "vocab_size = len(tokenizer.word_index) + 1\n",
+    "max_length = 100  # adjust as needed\n",
+    "embedding_dim = 50  # adjust as needed\n",
+    "\n",
+    "X = tokenizer.texts_to_sequences(df['content'])\n",
+    "X = pad_sequences(X, maxlen=max_length, padding='post')\n",
+    "\n",
+    "# Encode labels\n",
+    "labels = df['category'].map({'positive': 1, 'negative': 0})  # assuming 'positive' is labeled as 1 and 'negative' as 0\n",
+    "\n",
+    "# Split into train and test datasets\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.3, random_state=42)\n",
+    "\n",
+    "# Step 4: Build the model\n",
+    "model = tf.keras.Sequential([\n",
+    "    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length),\n",
+    "    tf.keras.layers.GlobalAveragePooling1D(),\n",
+    "    tf.keras.layers.Dense(16, activation='relu'),\n",
+    "    tf.keras.layers.Dense(1, activation='sigmoid')  # output layer with one neuron for binary classification\n",
+    "])\n",
+    "\n",
+    "# Step 5: Compile the model\n",
+    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])  # binary cross-entropy for binary classification\n",
+    "\n",
+    "# Step 6: Train the model\n",
+    "model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))\n",
+    "\n",
+    "# Step 7: Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test)\n",
+    "print(\"Test Accuracy:\", accuracy)\n",
+    "```\n",
+    "\n",
+    "\n",
+    "\n",
+    "This should resolve the shape mismatch issue and allow you to train and evaluate your binary classification sentiment analysis model.\n",
+    "\n",
+    "Person: \n",
+    "- what do you mean by Ensure your target labels are encoded as a single column for binary classification.\n",
+    "\n",
+    "ChatGPT: \n",
+    "- In binary classification tasks, where you're trying to classify data into two categories (e.g., positive and negative sentiment), it's common practice to encode the target labels using a single column. Each label is represented as either 0 or 1, where 0 typically represents one class (e.g., negative sentiment) and 1 represents the other class (e.g., positive sentiment).\n",
+    "\n",
+    "For example, if your original dataset looks like this:contentcategory\"great product\"positive\"terrible service\"negative\n",
+    "\n",
+    "You need to encode the 'category' column to convert the labels into numeric values. You might choose to encode 'positive' as 1 and 'negative' as 0, resulting in a single column for the target labels:contentcategory (encoded)\"great product\"1\"terrible service\"0\n",
+    "\n",
+    "This ensures that each target label is represented as a single number, making it compatible with the output of your neural network model, which typically consists of one neuron for binary classification.\n",
+    "\n",
+    "In the provided code, the line `labels = df['category'].map({'positive': 1, 'negative': 0})` accomplishes this encoding. It maps 'positive' to 1 and 'negative' to 0, creating a single column of numeric labels suitable for binary classification.\n",
+    "\n",
+    "Person: \n",
+    "- ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type int).\n",
+    "\n",
+    "ChatGPT: \n",
+    "- The error you're encountering suggests that there's an issue with the data type of your target labels when passing them to TensorFlow. TensorFlow expects the target labels to be in a compatible format, typically as NumPy arrays or TensorFlow tensors.\n",
+    "\n",
+    "In the provided code, the target labels are converted to a Pandas Series with integer values (0s and 1s). However, TensorFlow expects the target labels to be either NumPy arrays or TensorFlow tensors.\n",
+    "\n",
+    "To resolve this issue, you can explicitly convert the Pandas Series to a NumPy array before passing it to TensorFlow. Here's how you can do it:\n",
+    "\n",
+    "```python\n",
+    "# Encode labels and convert to NumPy array\n",
+    "labels = df['category'].map({'positive': 1, 'negative': 0}).values  # Convert to NumPy array\n",
+    "\n",
+    "# Split into train and test datasets\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.3, random_state=42)\n",
+    "\n",
+    "# Step 4: Build the model\n",
+    "model = tf.keras.Sequential([\n",
+    "    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length),\n",
+    "    tf.keras.layers.GlobalAveragePooling1D(),\n",
+    "    tf.keras.layers.Dense(16, activation='relu'),\n",
+    "    tf.keras.layers.Dense(1, activation='sigmoid')  # output layer with one neuron for binary classification\n",
+    "])\n",
+    "\n",
+    "# Step 5: Compile the model\n",
+    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])  # binary cross-entropy for binary classification\n",
+    "\n",
+    "# Step 6: Train the model\n",
+    "model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))\n",
+    "\n",
+    "# Step 7: Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test)\n",
+    "print(\"Test Accuracy:\", accuracy)\n",
+    "```\n",
+    "\n",
+    "\n",
+    "\n",
+    "By converting the Pandas Series to a NumPy array with `.values`, you ensure that the target labels are in a compatible format for TensorFlow, which should resolve the ValueError.\n",
+    "\n",
+    "Person: \n",
+    "- ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type int). \n",
+    "\n",
+    "still getting\n",
+    "\n",
+    "ChatGPT: \n",
+    "- The error message \"Failed to convert a NumPy array to a Tensor (Unsupported object type int)\" suggests that TensorFlow is encountering an issue with the data type of your target labels during training or evaluation. This could occur if TensorFlow expects a certain data type for the target labels but receives a different one.\n",
+    "\n",
+    "To resolve this issue, you can explicitly convert the target labels to a compatible data type for TensorFlow. Typically, TensorFlow expects target labels to be of type `float32` or `int32`.\n",
+    "\n",
+    "Let's convert the target labels to the appropriate data type before passing them to TensorFlow:\n",
+    "\n",
+    "```python\n",
+    "# Convert target labels to float32\n",
+    "labels = df['category'].map({'positive': 1, 'negative': 0}).astype('float32').values\n",
+    "\n",
+    "# Split into train and test datasets\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "X_train, X_test, y_train, y_test = train_test_split(X, labels, test_size=0.3, random_state=42)\n",
+    "\n",
+    "# Step 4: Build the model\n",
+    "model = tf.keras.Sequential([\n",
+    "    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=max_length),\n",
+    "    tf.keras.layers.GlobalAveragePooling1D(),\n",
+    "    tf.keras.layers.Dense(16, activation='relu'),\n",
+    "    tf.keras.layers.Dense(1, activation='sigmoid')  # output layer with one neuron for binary classification\n",
+    "])\n",
+    "\n",
+    "# Step 5: Compile the model\n",
+    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])  # binary cross-entropy for binary classification\n",
+    "\n",
+    "# Step 6: Train the model\n",
+    "model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))\n",
+    "\n",
+    "# Step 7: Evaluate the model\n",
+    "loss, accuracy = model.evaluate(X_test, y_test)\n",
+    "print(\"Test Accuracy:\", accuracy)\n",
+    "```\n",
+    "\n",
+    "\n",
+    "\n",
+    "By explicitly converting the target labels to `float32` using `.astype('float32')`, you ensure that they are in a compatible data type for TensorFlow. This should resolve the ValueError you encountered.\n",
+    "\n",
+    "Person: \n",
+    "- how to use the model with custom input text\n",
+    "\n",
+    "ChatGPT: \n",
+    "- Once you've trained your sentiment analysis model, you can use it to predict the sentiment of custom input text. Here's how you can do it:\n",
+    "1. Preprocess the custom input text to make it compatible with the model.\n",
+    "2. Use the trained model to predict the sentiment of the preprocessed input text.\n",
+    "\n",
+    "Here's a step-by-step guide on how to achieve this:\n",
+    "\n",
+    "```python\n",
+    "import tensorflow as tf\n",
+    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
+    "import numpy as np\n",
+    "\n",
+    "# Assuming you've already trained and saved your model, load the model\n",
+    "model = tf.keras.models.load_model(\"your_model.h5\")  # Replace \"your_model.h5\" with the path to your trained model\n",
+    "\n",
+    "# Function to preprocess input text\n",
+    "def preprocess_text(text):\n",
+    "    # Tokenize the text\n",
+    "    tokenized_text = tokenizer.texts_to_sequences([text])\n",
+    "    # Pad sequences to the same length as training data\n",
+    "    padded_text = pad_sequences(tokenized_text, maxlen=max_length, padding='post')\n",
+    "    return padded_text\n",
+    "\n",
+    "# Example custom input text\n",
+    "custom_text = \"I really enjoyed the product. It exceeded my expectations!\"\n",
+    "\n",
+    "# Preprocess the custom input text\n",
+    "preprocessed_text = preprocess_text(custom_text)\n",
+    "\n",
+    "# Make predictions\n",
+    "predictions = model.predict(preprocessed_text)\n",
+    "\n",
+    "# Convert prediction to sentiment label\n",
+    "sentiment_label = \"positive\" if predictions[0][0] > 0.5 else \"negative\"\n",
+    "\n",
+    "print(\"Predicted Sentiment:\", sentiment_label)\n",
+    "```\n",
+    "\n",
+    "\n",
+    "\n",
+    "In this example: \n",
+    "- `preprocess_text()` function preprocesses the custom input text by tokenizing and padding it to make it compatible with the model's input format. \n",
+    "- `model.predict()` is used to make predictions on the preprocessed input text.\n",
+    "- The threshold of 0.5 is used to classify the sentiment. If the predicted probability is greater than 0.5, the sentiment is classified as positive; otherwise, it's classified as negative.\n",
+    "\n",
+    "Replace `\"your_model.h5\"` with the path to your trained model file. You also need to have the `tokenizer`, `max_length`, and any other preprocessing steps used during training available in order to preprocess the custom input text correctly.\n",
+    "\n",
+    "Person: \n",
+    "- how to save model\n",
+    "\n",
+    "ChatGPT: \n",
+    "- To save your TensorFlow/Keras model after training, you can use the `save()` method. This method allows you to save the entire model including its architecture, weights, and training configuration in the Hierarchical Data Format (HDF5) file format. Here's how you can save your model:\n",
+    "\n",
+    "```python\n",
+    "# Assuming you've already trained your model\n",
+    "# model = ...  # your trained model\n",
+    "\n",
+    "# Save the model\n",
+    "model.save(\"your_model.h5\")\n",
+    "```\n",
+    "\n",
+    "\n",
+    "\n",
+    "This will save your model to a file named `\"your_model.h5\"` in the current directory. You can specify a different directory or path if needed.\n",
+    "\n",
+    "You can then load your saved model later using `tf.keras.models.load_model()` as shown in the previous example:\n",
+    "\n",
+    "```python\n",
+    "import tensorflow as tf\n",
+    "\n",
+    "# Load the saved model\n",
+    "model = tf.keras.models.load_model(\"your_model.h5\")\n",
+    "```\n",
+    "\n",
+    "\n",
+    "\n",
+    "This will load the entire model, including its architecture, weights, and training configuration, allowing you to make predictions or further train the model if needed.\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -0,0 +1,9 @@
+datasets
+keras
+pandas
+tf_keras
+tensorflow
+transformers
+tf-keras
+tensorrt
+tensorflow[and-cuda]