updated LFS

.env.example

@@ -0,0 +1 @@
+OPENAI_API_KEY=

.gitattributes

@@ -37,3 +37,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.docx filter=lfs diff=lfs merge=lfs -text
 *.pptx filter=lfs diff=lfs merge=lfs -text
 *.xlsx filter=lfs diff=lfs merge=lfs -text
+data/label_list.xlsx filter=lfs diff=lfs merge=lfs -text
+data/new_labels.xlsx filter=lfs diff=lfs merge=lfs -text
+data/processed_data.parquet filter=lfs diff=lfs merge=lfs -text
+data/scrapped_data1.parquet filter=lfs diff=lfs merge=lfs -text
+data/scrapped_data2.csv filter=lfs diff=lfs merge=lfs -text
+data/all_port_labelled.csv filter=lfs diff=lfs merge=lfs -text
+data/cleaned_data.csv filter=lfs diff=lfs merge=lfs -text
+data/cleaned_data_with_categories.csv filter=lfs diff=lfs merge=lfs -text
+data/processed_data2.parquet filter=lfs diff=lfs merge=lfs -text
+data/topic_viz_benchmark_moderate.html filter=lfs diff=lfs merge=lfs -text
+data/topic_viz_benchmark_severe.html filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -148,3 +148,4 @@ qa_chain_test_results.csv
 data/logs/hftgi-2024-03-18.txt
 qa_chain_all_results.csv
 qa_batch_1_test_results.csv
+*.html

README.md

@@ -1,12 +1,12 @@
----
-title: Global Incidents
-emoji: 💬
-colorFrom: yellow
-colorTo: purple
-sdk: gradio
-sdk_version: 4.36.1
-app_file: app.py
-pinned: false
----
-
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+# DM Project
+
+The project consists of three parts: the classification, the GPT and its evaluation (GPT Evaluation) and the news scraper + LDA modelling.
+
+all the files can be run independently without interference of others and pre-setup, except for the LDA folder. 
+
+For LDA, 
+the processed_data.parquet, processed the original data and is generated by the file basic_text_preprocessing, is used in topic_modelling_benchmark_using_headline;
+
+the processed_data1.parquet, processed the scraped news content and is generated by the file basic_text_preprocessing_on_scraped_data, is used in files topic_modelling_minor, topic_modelling_severe and topic_modelling_moderate.
+
+However, be careful when running the GPT and news scraper file, as you may need your own API key for GPT script to run properly. Also, it takes very long for the news scraper scirpt to finish. 

data/all_port_labelled.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f2a6c487806e6307d2763ffef90378092cbdd9682ed2a952cb129d5ec06f7a3
+size 8767987

data/cleaned_data.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75438fa508739e03dcf9e9ad18d5fe2f01ffef0c9fd5b6b8caf9e0aff65aa18d
+size 7998227

data/cleaned_data_with_categories.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be908d68560d66d238287fad1c3bacfb6e2d5a205b77822cc75ba9df57e55fa9
+size 78688

data/label_list.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a47566ba13a2b4f97f0105bb0aba8f1abeecd5a1bf7f24af47cac44d4d579e5
+size 6462

data/new_labels.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d2c27675a2408cfaa948f4030b19f67deb9fe72bfa3eb80f721ebb4386ea77a
+size 10338

data/processed_data.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa419c0b692c27050df6059204dd9dc16570bcd2344b9be061ac30b06363112e
+size 5081543

data/processed_data2.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41d62483447ae9377eb9c13ed383a09b6b0fa82b7f896c2d1033f6e9109b69d6
+size 29146326

data/scrapped_data1.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57abf4a381c4d2501e154cd23bd1ca3200e08c446bee2eee14abde93204fcdb6
+size 18114548

data/scrapped_data2.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bd8a6b3ada152b3ebc8aae8b6036045b993f668015ccd73d675ecaa43954ce6
+size 37502335

notebooks/02_Classification models on maritime incident.ipynb

@@ -0,0 +1,2545 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e550a89c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n",
+      "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1fecbc87",
+   "metadata": {},
+   "source": [
+    "## Import Statement"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5169e3ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "76905f72",
+   "metadata": {},
+   "source": [
+    "### read the data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "b1043895",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_csv(\"data/all_port_labelled.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "2e40d90a",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Unnamed: 0</th>\n",
+       "      <th>Index</th>\n",
+       "      <th>Unnamed: 0.1</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>...</th>\n",
+       "      <th>IT</th>\n",
+       "      <th>EP</th>\n",
+       "      <th>NEW</th>\n",
+       "      <th>CSD</th>\n",
+       "      <th>RPE</th>\n",
+       "      <th>MN</th>\n",
+       "      <th>NM</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>8.0</td>\n",
+       "      <td>34.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>63.0</td>\n",
+       "      <td>Indonesia: Undersea internet cables damaged by...</td>\n",
+       "      <td>News sources are stating that recent typhoons ...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Travel Warning</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>4/9/17 14:30</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>14.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2 rows × 46 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   Unnamed: 0  Index  Unnamed: 0.1  \\\n",
+       "0         0.0    8.0          34.0   \n",
+       "1         1.0   10.0          63.0   \n",
+       "\n",
+       "                                            Headline  \\\n",
+       "0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1  Indonesia: Undersea internet cables damaged by...   \n",
+       "\n",
+       "                                             Details  Severity  \\\n",
+       "0  Media sources indicate that workers at the Gra...  Moderate   \n",
+       "1  News sources are stating that recent typhoons ...     Minor   \n",
+       "\n",
+       "              Category     Region       Datetime    Year  ...   IT   EP  NEW  \\\n",
+       "0  Mine Workers Strike  Indonesia  28/5/17 17:08  2017.0  ...  0.0  0.0  0.0   \n",
+       "1       Travel Warning  Indonesia   4/9/17 14:30  2017.0  ...  0.0  0.0  0.0   \n",
+       "\n",
+       "   CSD  RPE   MN   NM if_labeled  Month  Week  \n",
+       "0  0.0  0.0  0.0  1.0      False    5.0  21.0  \n",
+       "1  0.0  0.0  1.0  0.0      False    4.0  14.0  \n",
+       "\n",
+       "[2 rows x 46 columns]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head(2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "643a7e40",
+   "metadata": {},
+   "source": [
+    "### Clean empty data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "d6ee1fd7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "from nltk.stem import WordNetLemmatizer\n",
+    "import string\n",
+    "\n",
+    "# nltk.download('punkt')\n",
+    "# nltk.download('stopwords')\n",
+    "# nltk.download('wordnet')\n",
+    "\n",
+    "\n",
+    "def clean_text(text):\n",
+    "    # Lowercase\n",
+    "    text = text.lower()\n",
+    "    # Tokenization\n",
+    "    tokens = word_tokenize(text)\n",
+    "    # Removing punctuation\n",
+    "    tokens = [word for word in tokens if word not in string.punctuation]\n",
+    "    # Removing stop words\n",
+    "    stop_words = set(stopwords.words(\"english\"))\n",
+    "    tokens = [word for word in tokens if word not in stop_words]\n",
+    "    # Lemmatization\n",
+    "    lemmatizer = WordNetLemmatizer()\n",
+    "    tokens = [lemmatizer.lemmatize(word) for word in tokens]\n",
+    "\n",
+    "    return \" \".join(tokens)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "9e35b49a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[nltk_data] Downloading package omw-1.4 to\n",
+      "[nltk_data]     /Users/inflaton/nltk_data...\n",
+      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import nltk\n",
+    "\n",
+    "nltk.download(\"omw-1.4\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ca331c4b",
+   "metadata": {},
+   "source": [
+    "### The Details column has an issue\n",
+    "\n",
+    "some of the data are of the type float and none of the text processing functions can be applied to it therefore we have to process it"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "2438c58f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 5782 entries, 0 to 5781\n",
+      "Data columns (total 2 columns):\n",
+      " #   Column          Non-Null Count  Dtype \n",
+      "---  ------          --------------  ----- \n",
+      " 0   Details         5781 non-null   object\n",
+      " 1   maritime_label  5781 non-null   object\n",
+      "dtypes: object(2)\n",
+      "memory usage: 90.5+ KB\n",
+      "<class 'pandas.core.frame.DataFrame'>\n",
+      "RangeIndex: 5782 entries, 0 to 5781\n",
+      "Data columns (total 3 columns):\n",
+      " #   Column           Non-Null Count  Dtype \n",
+      "---  ------           --------------  ----- \n",
+      " 0   Details          5781 non-null   object\n",
+      " 1   maritime_label   5781 non-null   object\n",
+      " 2   Details_cleaned  5781 non-null   object\n",
+      "dtypes: object(3)\n",
+      "memory usage: 135.6+ KB\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/7x/56svhln929zdh2xhr3mwqg4r0000gn/T/ipykernel_16237/2443564520.py:3: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  text_df['Details_cleaned'] = text_df['Details'].apply(lambda x: clean_text(x) if not isinstance(x, float) else None)\n"
+     ]
+    }
+   ],
+   "source": [
+    "text_df = df[[\"Details\", \"maritime_label\"]]\n",
+    "text_df.info()\n",
+    "text_df[\"Details_cleaned\"] = text_df[\"Details\"].apply(\n",
+    "    lambda x: clean_text(x) if not isinstance(x, float) else None\n",
+    ")\n",
+    "# no_nan_df[no_nan_df[\"Details\"].apply(lambda x: print(type(x)))]\n",
+    "# cleaned_df = text_df[text_df[\"Details\"].apply(lambda x: clean_text(x))]\n",
+    "# cleaned_df = df['Details'][1:2]\n",
+    "# type(no_nan_df[\"Details\"][0])\n",
+    "# print(clean_text(no_nan_df[\"Details\"][0]))\n",
+    "text_df.info()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "4d3b0011",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Details</th>\n",
+       "      <th>maritime_label</th>\n",
+       "      <th>Details_cleaned</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>FALSE</td>\n",
+       "      <td>medium source indicate worker grasberg mine ex...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>News sources are stating that recent typhoons ...</td>\n",
+       "      <td>FALSE</td>\n",
+       "      <td>news source stating recent typhoon impact hong...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>TRUE</td>\n",
+       "      <td>persisting port congestion shanghai ’ yangshan...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Updated local media sources from Jakarta indic...</td>\n",
+       "      <td>TRUE</td>\n",
+       "      <td>updated local medium source jakarta indicate e...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>TRUE</td>\n",
+       "      <td>according local police jakarta two explosion c...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                             Details maritime_label  \\\n",
+       "0  Media sources indicate that workers at the Gra...          FALSE   \n",
+       "1  News sources are stating that recent typhoons ...          FALSE   \n",
+       "2  The persisting port congestion at Shanghai’s Y...           TRUE   \n",
+       "3  Updated local media sources from Jakarta indic...           TRUE   \n",
+       "4  According to local police in Jakarta, two expl...           TRUE   \n",
+       "\n",
+       "                                     Details_cleaned  \n",
+       "0  medium source indicate worker grasberg mine ex...  \n",
+       "1  news source stating recent typhoon impact hong...  \n",
+       "2  persisting port congestion shanghai ’ yangshan...  \n",
+       "3  updated local medium source jakarta indicate e...  \n",
+       "4  according local police jakarta two explosion c...  "
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "processed_data = text_df.dropna()\n",
+    "processed_data.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3c4be609",
+   "metadata": {},
+   "source": [
+    "## Naive Bayes Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "5c660011",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "\n",
+    "# from sklearn.feature_extraction.text import CountVectorizer\n",
+    "from sklearn.naive_bayes import MultinomialNB\n",
+    "from sklearn.metrics import accuracy_score, classification_report"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "8f009a65",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X = processed_data[\"Details_cleaned\"]\n",
+    "y = processed_data[\"maritime_label\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "0185a967",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_train, X_test, y_train, y_test = train_test_split(\n",
+    "    X, y, test_size=0.2, random_state=42\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "d3c2de6b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# vectorizer = CountVectorizer()\n",
+    "# X_train_vec = vectorizer.fit_transform(X_train)\n",
+    "# X_test_vec = vectorizer.transform(X_test)\n",
+    "\n",
+    "tfidf_vectorizer = TfidfVectorizer(max_features=1000)\n",
+    "X_train_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
+    "X_test_tfidf = tfidf_vectorizer.transform(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "ead2fc7a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style>#sk-container-id-1 {\n",
+       "  /* Definition of color scheme common for light and dark mode */\n",
+       "  --sklearn-color-text: black;\n",
+       "  --sklearn-color-line: gray;\n",
+       "  /* Definition of color scheme for unfitted estimators */\n",
+       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
+       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
+       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
+       "  --sklearn-color-unfitted-level-3: chocolate;\n",
+       "  /* Definition of color scheme for fitted estimators */\n",
+       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
+       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
+       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
+       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
+       "\n",
+       "  /* Specific color for light theme */\n",
+       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
+       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-icon: #696969;\n",
+       "\n",
+       "  @media (prefers-color-scheme: dark) {\n",
+       "    /* Redefinition of color scheme for dark theme */\n",
+       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
+       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-icon: #878787;\n",
+       "  }\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 pre {\n",
+       "  padding: 0;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 input.sk-hidden--visually {\n",
+       "  border: 0;\n",
+       "  clip: rect(1px 1px 1px 1px);\n",
+       "  clip: rect(1px, 1px, 1px, 1px);\n",
+       "  height: 1px;\n",
+       "  margin: -1px;\n",
+       "  overflow: hidden;\n",
+       "  padding: 0;\n",
+       "  position: absolute;\n",
+       "  width: 1px;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
+       "  border: 1px dashed var(--sklearn-color-line);\n",
+       "  margin: 0 0.4em 0.5em 0.4em;\n",
+       "  box-sizing: border-box;\n",
+       "  padding-bottom: 0.4em;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-container {\n",
+       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
+       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
+       "     so we also need the `!important` here to be able to override the\n",
+       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
+       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
+       "  display: inline-block !important;\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
+       "  display: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-parallel-item,\n",
+       "div.sk-serial,\n",
+       "div.sk-item {\n",
+       "  /* draw centered vertical line to link estimators */\n",
+       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
+       "  background-size: 2px 100%;\n",
+       "  background-repeat: no-repeat;\n",
+       "  background-position: center center;\n",
+       "}\n",
+       "\n",
+       "/* Parallel-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-1 div.sk-parallel-item::after {\n",
+       "  content: \"\";\n",
+       "  width: 100%;\n",
+       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
+       "  flex-grow: 1;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-parallel {\n",
+       "  display: flex;\n",
+       "  align-items: stretch;\n",
+       "  justify-content: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-parallel-item {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
+       "  align-self: flex-end;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
+       "  align-self: flex-start;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
+       "  width: 0;\n",
+       "}\n",
+       "\n",
+       "/* Serial-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-1 div.sk-serial {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "  align-items: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  padding-right: 1em;\n",
+       "  padding-left: 1em;\n",
+       "}\n",
+       "\n",
+       "\n",
+       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
+       "clickable and can be expanded/collapsed.\n",
+       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
+       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
+       "*/\n",
+       "\n",
+       "/* Pipeline and ColumnTransformer style (default) */\n",
+       "\n",
+       "#sk-container-id-1 div.sk-toggleable {\n",
+       "  /* Default theme specific background. It is overwritten whether we have a\n",
+       "  specific estimator or a Pipeline/ColumnTransformer */\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable label */\n",
+       "#sk-container-id-1 label.sk-toggleable__label {\n",
+       "  cursor: pointer;\n",
+       "  display: block;\n",
+       "  width: 100%;\n",
+       "  margin-bottom: 0;\n",
+       "  padding: 0.5em;\n",
+       "  box-sizing: border-box;\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
+       "  /* Arrow on the left of the label */\n",
+       "  content: \"▸\";\n",
+       "  float: left;\n",
+       "  margin-right: 0.25em;\n",
+       "  color: var(--sklearn-color-icon);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable content - dropdown */\n",
+       "\n",
+       "#sk-container-id-1 div.sk-toggleable__content {\n",
+       "  max-height: 0;\n",
+       "  max-width: 0;\n",
+       "  overflow: hidden;\n",
+       "  text-align: left;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
+       "  margin: 0.2em;\n",
+       "  border-radius: 0.25em;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
+       "  /* Expand drop-down */\n",
+       "  max-height: 200px;\n",
+       "  max-width: 100%;\n",
+       "  overflow: auto;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
+       "  content: \"▾\";\n",
+       "}\n",
+       "\n",
+       "/* Pipeline/ColumnTransformer-specific style */\n",
+       "\n",
+       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific style */\n",
+       "\n",
+       "/* Colorize estimator box */\n",
+       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
+       "#sk-container-id-1 div.sk-label label {\n",
+       "  /* The background is the default theme color */\n",
+       "  color: var(--sklearn-color-text-on-default-background);\n",
+       "}\n",
+       "\n",
+       "/* On hover, darken the color of the background */\n",
+       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Label box, darken color on hover, fitted */\n",
+       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator label */\n",
+       "\n",
+       "#sk-container-id-1 div.sk-label label {\n",
+       "  font-family: monospace;\n",
+       "  font-weight: bold;\n",
+       "  display: inline-block;\n",
+       "  line-height: 1.2em;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-label-container {\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific */\n",
+       "#sk-container-id-1 div.sk-estimator {\n",
+       "  font-family: monospace;\n",
+       "  border: 1px dotted var(--sklearn-color-border-box);\n",
+       "  border-radius: 0.25em;\n",
+       "  box-sizing: border-box;\n",
+       "  margin-bottom: 0.5em;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-estimator.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "/* on hover */\n",
+       "#sk-container-id-1 div.sk-estimator:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
+       "\n",
+       "/* Common style for \"i\" and \"?\" */\n",
+       "\n",
+       ".sk-estimator-doc-link,\n",
+       "a:link.sk-estimator-doc-link,\n",
+       "a:visited.sk-estimator-doc-link {\n",
+       "  float: right;\n",
+       "  font-size: smaller;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1em;\n",
+       "  height: 1em;\n",
+       "  width: 1em;\n",
+       "  text-decoration: none !important;\n",
+       "  margin-left: 1ex;\n",
+       "  /* unfitted */\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted,\n",
+       "a:link.sk-estimator-doc-link.fitted,\n",
+       "a:visited.sk-estimator-doc-link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "/* Span, style for the box shown on hovering the info icon */\n",
+       ".sk-estimator-doc-link span {\n",
+       "  display: none;\n",
+       "  z-index: 9999;\n",
+       "  position: relative;\n",
+       "  font-weight: normal;\n",
+       "  right: .2ex;\n",
+       "  padding: .5ex;\n",
+       "  margin: .5ex;\n",
+       "  width: min-content;\n",
+       "  min-width: 20ex;\n",
+       "  max-width: 50ex;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  box-shadow: 2pt 2pt 4pt #999;\n",
+       "  /* unfitted */\n",
+       "  background: var(--sklearn-color-unfitted-level-0);\n",
+       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted span {\n",
+       "  /* fitted */\n",
+       "  background: var(--sklearn-color-fitted-level-0);\n",
+       "  border: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link:hover span {\n",
+       "  display: block;\n",
+       "}\n",
+       "\n",
+       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
+       "\n",
+       "#sk-container-id-1 a.estimator_doc_link {\n",
+       "  float: right;\n",
+       "  font-size: 1rem;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1rem;\n",
+       "  height: 1rem;\n",
+       "  width: 1rem;\n",
+       "  text-decoration: none;\n",
+       "  /* unfitted */\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultinomialNB()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;MultinomialNB<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.naive_bayes.MultinomialNB.html\">?<span>Documentation for MultinomialNB</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>MultinomialNB()</pre></div> </div></div></div></div>"
+      ],
+      "text/plain": [
+       "MultinomialNB()"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "naive_bayes = MultinomialNB()\n",
+    "naive_bayes.fit(X_train_tfidf, y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "74c5df68",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "predictions = naive_bayes.predict(X_test_tfidf)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "109e9456",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy of Naive Bayes model: 0.8582541054451167\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "       FALSE       0.88      0.94      0.91       847\n",
+      "        TRUE       0.79      0.65      0.71       310\n",
+      "\n",
+      "    accuracy                           0.86      1157\n",
+      "   macro avg       0.83      0.79      0.81      1157\n",
+      "weighted avg       0.85      0.86      0.85      1157\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "accuracy = accuracy_score(y_test, predictions)\n",
+    "print(\"Accuracy of Naive Bayes model:\", accuracy)\n",
+    "print(classification_report(y_test, predictions))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9518614a",
+   "metadata": {},
+   "source": [
+    "## Logistic Regression model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "912ad7a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.linear_model import LogisticRegression\n",
+    "from sklearn.metrics import accuracy_score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "03eac734",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_train, X_test, y_train, y_test = train_test_split(\n",
+    "    X, y, test_size=0.2, random_state=42\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "e84ff87c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tfidf_vectorizer = TfidfVectorizer(max_features=1000)\n",
+    "X_train_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
+    "X_test_tfidf = tfidf_vectorizer.transform(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "cedb263c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style>#sk-container-id-2 {\n",
+       "  /* Definition of color scheme common for light and dark mode */\n",
+       "  --sklearn-color-text: black;\n",
+       "  --sklearn-color-line: gray;\n",
+       "  /* Definition of color scheme for unfitted estimators */\n",
+       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
+       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
+       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
+       "  --sklearn-color-unfitted-level-3: chocolate;\n",
+       "  /* Definition of color scheme for fitted estimators */\n",
+       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
+       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
+       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
+       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
+       "\n",
+       "  /* Specific color for light theme */\n",
+       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
+       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-icon: #696969;\n",
+       "\n",
+       "  @media (prefers-color-scheme: dark) {\n",
+       "    /* Redefinition of color scheme for dark theme */\n",
+       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
+       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-icon: #878787;\n",
+       "  }\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 pre {\n",
+       "  padding: 0;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 input.sk-hidden--visually {\n",
+       "  border: 0;\n",
+       "  clip: rect(1px 1px 1px 1px);\n",
+       "  clip: rect(1px, 1px, 1px, 1px);\n",
+       "  height: 1px;\n",
+       "  margin: -1px;\n",
+       "  overflow: hidden;\n",
+       "  padding: 0;\n",
+       "  position: absolute;\n",
+       "  width: 1px;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-dashed-wrapped {\n",
+       "  border: 1px dashed var(--sklearn-color-line);\n",
+       "  margin: 0 0.4em 0.5em 0.4em;\n",
+       "  box-sizing: border-box;\n",
+       "  padding-bottom: 0.4em;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-container {\n",
+       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
+       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
+       "     so we also need the `!important` here to be able to override the\n",
+       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
+       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
+       "  display: inline-block !important;\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-text-repr-fallback {\n",
+       "  display: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-parallel-item,\n",
+       "div.sk-serial,\n",
+       "div.sk-item {\n",
+       "  /* draw centered vertical line to link estimators */\n",
+       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
+       "  background-size: 2px 100%;\n",
+       "  background-repeat: no-repeat;\n",
+       "  background-position: center center;\n",
+       "}\n",
+       "\n",
+       "/* Parallel-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-2 div.sk-parallel-item::after {\n",
+       "  content: \"\";\n",
+       "  width: 100%;\n",
+       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
+       "  flex-grow: 1;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-parallel {\n",
+       "  display: flex;\n",
+       "  align-items: stretch;\n",
+       "  justify-content: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-parallel-item {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-parallel-item:first-child::after {\n",
+       "  align-self: flex-end;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-parallel-item:last-child::after {\n",
+       "  align-self: flex-start;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-parallel-item:only-child::after {\n",
+       "  width: 0;\n",
+       "}\n",
+       "\n",
+       "/* Serial-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-2 div.sk-serial {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "  align-items: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  padding-right: 1em;\n",
+       "  padding-left: 1em;\n",
+       "}\n",
+       "\n",
+       "\n",
+       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
+       "clickable and can be expanded/collapsed.\n",
+       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
+       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
+       "*/\n",
+       "\n",
+       "/* Pipeline and ColumnTransformer style (default) */\n",
+       "\n",
+       "#sk-container-id-2 div.sk-toggleable {\n",
+       "  /* Default theme specific background. It is overwritten whether we have a\n",
+       "  specific estimator or a Pipeline/ColumnTransformer */\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable label */\n",
+       "#sk-container-id-2 label.sk-toggleable__label {\n",
+       "  cursor: pointer;\n",
+       "  display: block;\n",
+       "  width: 100%;\n",
+       "  margin-bottom: 0;\n",
+       "  padding: 0.5em;\n",
+       "  box-sizing: border-box;\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 label.sk-toggleable__label-arrow:before {\n",
+       "  /* Arrow on the left of the label */\n",
+       "  content: \"▸\";\n",
+       "  float: left;\n",
+       "  margin-right: 0.25em;\n",
+       "  color: var(--sklearn-color-icon);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable content - dropdown */\n",
+       "\n",
+       "#sk-container-id-2 div.sk-toggleable__content {\n",
+       "  max-height: 0;\n",
+       "  max-width: 0;\n",
+       "  overflow: hidden;\n",
+       "  text-align: left;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-toggleable__content.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-toggleable__content pre {\n",
+       "  margin: 0.2em;\n",
+       "  border-radius: 0.25em;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-toggleable__content.fitted pre {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
+       "  /* Expand drop-down */\n",
+       "  max-height: 200px;\n",
+       "  max-width: 100%;\n",
+       "  overflow: auto;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
+       "  content: \"▾\";\n",
+       "}\n",
+       "\n",
+       "/* Pipeline/ColumnTransformer-specific style */\n",
+       "\n",
+       "#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific style */\n",
+       "\n",
+       "/* Colorize estimator box */\n",
+       "#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-label label.sk-toggleable__label,\n",
+       "#sk-container-id-2 div.sk-label label {\n",
+       "  /* The background is the default theme color */\n",
+       "  color: var(--sklearn-color-text-on-default-background);\n",
+       "}\n",
+       "\n",
+       "/* On hover, darken the color of the background */\n",
+       "#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Label box, darken color on hover, fitted */\n",
+       "#sk-container-id-2 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator label */\n",
+       "\n",
+       "#sk-container-id-2 div.sk-label label {\n",
+       "  font-family: monospace;\n",
+       "  font-weight: bold;\n",
+       "  display: inline-block;\n",
+       "  line-height: 1.2em;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-label-container {\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific */\n",
+       "#sk-container-id-2 div.sk-estimator {\n",
+       "  font-family: monospace;\n",
+       "  border: 1px dotted var(--sklearn-color-border-box);\n",
+       "  border-radius: 0.25em;\n",
+       "  box-sizing: border-box;\n",
+       "  margin-bottom: 0.5em;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-estimator.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "/* on hover */\n",
+       "#sk-container-id-2 div.sk-estimator:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 div.sk-estimator.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
+       "\n",
+       "/* Common style for \"i\" and \"?\" */\n",
+       "\n",
+       ".sk-estimator-doc-link,\n",
+       "a:link.sk-estimator-doc-link,\n",
+       "a:visited.sk-estimator-doc-link {\n",
+       "  float: right;\n",
+       "  font-size: smaller;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1em;\n",
+       "  height: 1em;\n",
+       "  width: 1em;\n",
+       "  text-decoration: none !important;\n",
+       "  margin-left: 1ex;\n",
+       "  /* unfitted */\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted,\n",
+       "a:link.sk-estimator-doc-link.fitted,\n",
+       "a:visited.sk-estimator-doc-link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "/* Span, style for the box shown on hovering the info icon */\n",
+       ".sk-estimator-doc-link span {\n",
+       "  display: none;\n",
+       "  z-index: 9999;\n",
+       "  position: relative;\n",
+       "  font-weight: normal;\n",
+       "  right: .2ex;\n",
+       "  padding: .5ex;\n",
+       "  margin: .5ex;\n",
+       "  width: min-content;\n",
+       "  min-width: 20ex;\n",
+       "  max-width: 50ex;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  box-shadow: 2pt 2pt 4pt #999;\n",
+       "  /* unfitted */\n",
+       "  background: var(--sklearn-color-unfitted-level-0);\n",
+       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted span {\n",
+       "  /* fitted */\n",
+       "  background: var(--sklearn-color-fitted-level-0);\n",
+       "  border: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link:hover span {\n",
+       "  display: block;\n",
+       "}\n",
+       "\n",
+       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
+       "\n",
+       "#sk-container-id-2 a.estimator_doc_link {\n",
+       "  float: right;\n",
+       "  font-size: 1rem;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1rem;\n",
+       "  height: 1rem;\n",
+       "  width: 1rem;\n",
+       "  text-decoration: none;\n",
+       "  /* unfitted */\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 a.estimator_doc_link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "#sk-container-id-2 a.estimator_doc_link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-2 a.estimator_doc_link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;LogisticRegression<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.linear_model.LogisticRegression.html\">?<span>Documentation for LogisticRegression</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>LogisticRegression()</pre></div> </div></div></div></div>"
+      ],
+      "text/plain": [
+       "LogisticRegression()"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = LogisticRegression()\n",
+    "model.fit(X_train_tfidf, y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "6f49fddb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy of Logistic Regression Model: 0.9308556611927399\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "       FALSE       0.92      0.99      0.95       847\n",
+      "        TRUE       0.98      0.76      0.86       310\n",
+      "\n",
+      "    accuracy                           0.93      1157\n",
+      "   macro avg       0.95      0.88      0.90      1157\n",
+      "weighted avg       0.93      0.93      0.93      1157\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "y_pred = model.predict(X_test_tfidf)\n",
+    "\n",
+    "accuracy = accuracy_score(y_test, y_pred)\n",
+    "print(\"Accuracy of Logistic Regression Model:\", accuracy)\n",
+    "print(classification_report(y_test, y_pred))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "613c0cdf",
+   "metadata": {},
+   "source": [
+    "## Support Vector Machine (SVM) model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "706302c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.svm import SVC\n",
+    "from sklearn.metrics import accuracy_score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "b0988ca4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_train, X_test, y_train, y_test = train_test_split(\n",
+    "    X, y, test_size=0.2, random_state=42\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "4f682c60",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tfidf_vectorizer = TfidfVectorizer(max_features=1000)\n",
+    "X_train_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
+    "X_test_tfidf = tfidf_vectorizer.transform(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "71ae91d9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style>#sk-container-id-3 {\n",
+       "  /* Definition of color scheme common for light and dark mode */\n",
+       "  --sklearn-color-text: black;\n",
+       "  --sklearn-color-line: gray;\n",
+       "  /* Definition of color scheme for unfitted estimators */\n",
+       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
+       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
+       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
+       "  --sklearn-color-unfitted-level-3: chocolate;\n",
+       "  /* Definition of color scheme for fitted estimators */\n",
+       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
+       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
+       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
+       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
+       "\n",
+       "  /* Specific color for light theme */\n",
+       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
+       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-icon: #696969;\n",
+       "\n",
+       "  @media (prefers-color-scheme: dark) {\n",
+       "    /* Redefinition of color scheme for dark theme */\n",
+       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
+       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-icon: #878787;\n",
+       "  }\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 pre {\n",
+       "  padding: 0;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 input.sk-hidden--visually {\n",
+       "  border: 0;\n",
+       "  clip: rect(1px 1px 1px 1px);\n",
+       "  clip: rect(1px, 1px, 1px, 1px);\n",
+       "  height: 1px;\n",
+       "  margin: -1px;\n",
+       "  overflow: hidden;\n",
+       "  padding: 0;\n",
+       "  position: absolute;\n",
+       "  width: 1px;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-dashed-wrapped {\n",
+       "  border: 1px dashed var(--sklearn-color-line);\n",
+       "  margin: 0 0.4em 0.5em 0.4em;\n",
+       "  box-sizing: border-box;\n",
+       "  padding-bottom: 0.4em;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-container {\n",
+       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
+       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
+       "     so we also need the `!important` here to be able to override the\n",
+       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
+       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
+       "  display: inline-block !important;\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-text-repr-fallback {\n",
+       "  display: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-parallel-item,\n",
+       "div.sk-serial,\n",
+       "div.sk-item {\n",
+       "  /* draw centered vertical line to link estimators */\n",
+       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
+       "  background-size: 2px 100%;\n",
+       "  background-repeat: no-repeat;\n",
+       "  background-position: center center;\n",
+       "}\n",
+       "\n",
+       "/* Parallel-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-3 div.sk-parallel-item::after {\n",
+       "  content: \"\";\n",
+       "  width: 100%;\n",
+       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
+       "  flex-grow: 1;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-parallel {\n",
+       "  display: flex;\n",
+       "  align-items: stretch;\n",
+       "  justify-content: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-parallel-item {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-parallel-item:first-child::after {\n",
+       "  align-self: flex-end;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-parallel-item:last-child::after {\n",
+       "  align-self: flex-start;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-parallel-item:only-child::after {\n",
+       "  width: 0;\n",
+       "}\n",
+       "\n",
+       "/* Serial-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-3 div.sk-serial {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "  align-items: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  padding-right: 1em;\n",
+       "  padding-left: 1em;\n",
+       "}\n",
+       "\n",
+       "\n",
+       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
+       "clickable and can be expanded/collapsed.\n",
+       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
+       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
+       "*/\n",
+       "\n",
+       "/* Pipeline and ColumnTransformer style (default) */\n",
+       "\n",
+       "#sk-container-id-3 div.sk-toggleable {\n",
+       "  /* Default theme specific background. It is overwritten whether we have a\n",
+       "  specific estimator or a Pipeline/ColumnTransformer */\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable label */\n",
+       "#sk-container-id-3 label.sk-toggleable__label {\n",
+       "  cursor: pointer;\n",
+       "  display: block;\n",
+       "  width: 100%;\n",
+       "  margin-bottom: 0;\n",
+       "  padding: 0.5em;\n",
+       "  box-sizing: border-box;\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 label.sk-toggleable__label-arrow:before {\n",
+       "  /* Arrow on the left of the label */\n",
+       "  content: \"▸\";\n",
+       "  float: left;\n",
+       "  margin-right: 0.25em;\n",
+       "  color: var(--sklearn-color-icon);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable content - dropdown */\n",
+       "\n",
+       "#sk-container-id-3 div.sk-toggleable__content {\n",
+       "  max-height: 0;\n",
+       "  max-width: 0;\n",
+       "  overflow: hidden;\n",
+       "  text-align: left;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-toggleable__content.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-toggleable__content pre {\n",
+       "  margin: 0.2em;\n",
+       "  border-radius: 0.25em;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-toggleable__content.fitted pre {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
+       "  /* Expand drop-down */\n",
+       "  max-height: 200px;\n",
+       "  max-width: 100%;\n",
+       "  overflow: auto;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
+       "  content: \"▾\";\n",
+       "}\n",
+       "\n",
+       "/* Pipeline/ColumnTransformer-specific style */\n",
+       "\n",
+       "#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific style */\n",
+       "\n",
+       "/* Colorize estimator box */\n",
+       "#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-label label.sk-toggleable__label,\n",
+       "#sk-container-id-3 div.sk-label label {\n",
+       "  /* The background is the default theme color */\n",
+       "  color: var(--sklearn-color-text-on-default-background);\n",
+       "}\n",
+       "\n",
+       "/* On hover, darken the color of the background */\n",
+       "#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Label box, darken color on hover, fitted */\n",
+       "#sk-container-id-3 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator label */\n",
+       "\n",
+       "#sk-container-id-3 div.sk-label label {\n",
+       "  font-family: monospace;\n",
+       "  font-weight: bold;\n",
+       "  display: inline-block;\n",
+       "  line-height: 1.2em;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-label-container {\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific */\n",
+       "#sk-container-id-3 div.sk-estimator {\n",
+       "  font-family: monospace;\n",
+       "  border: 1px dotted var(--sklearn-color-border-box);\n",
+       "  border-radius: 0.25em;\n",
+       "  box-sizing: border-box;\n",
+       "  margin-bottom: 0.5em;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-estimator.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "/* on hover */\n",
+       "#sk-container-id-3 div.sk-estimator:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 div.sk-estimator.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
+       "\n",
+       "/* Common style for \"i\" and \"?\" */\n",
+       "\n",
+       ".sk-estimator-doc-link,\n",
+       "a:link.sk-estimator-doc-link,\n",
+       "a:visited.sk-estimator-doc-link {\n",
+       "  float: right;\n",
+       "  font-size: smaller;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1em;\n",
+       "  height: 1em;\n",
+       "  width: 1em;\n",
+       "  text-decoration: none !important;\n",
+       "  margin-left: 1ex;\n",
+       "  /* unfitted */\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted,\n",
+       "a:link.sk-estimator-doc-link.fitted,\n",
+       "a:visited.sk-estimator-doc-link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "/* Span, style for the box shown on hovering the info icon */\n",
+       ".sk-estimator-doc-link span {\n",
+       "  display: none;\n",
+       "  z-index: 9999;\n",
+       "  position: relative;\n",
+       "  font-weight: normal;\n",
+       "  right: .2ex;\n",
+       "  padding: .5ex;\n",
+       "  margin: .5ex;\n",
+       "  width: min-content;\n",
+       "  min-width: 20ex;\n",
+       "  max-width: 50ex;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  box-shadow: 2pt 2pt 4pt #999;\n",
+       "  /* unfitted */\n",
+       "  background: var(--sklearn-color-unfitted-level-0);\n",
+       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted span {\n",
+       "  /* fitted */\n",
+       "  background: var(--sklearn-color-fitted-level-0);\n",
+       "  border: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link:hover span {\n",
+       "  display: block;\n",
+       "}\n",
+       "\n",
+       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
+       "\n",
+       "#sk-container-id-3 a.estimator_doc_link {\n",
+       "  float: right;\n",
+       "  font-size: 1rem;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1rem;\n",
+       "  height: 1rem;\n",
+       "  width: 1rem;\n",
+       "  text-decoration: none;\n",
+       "  /* unfitted */\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 a.estimator_doc_link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "#sk-container-id-3 a.estimator_doc_link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-3 a.estimator_doc_link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>SVC(kernel=&#x27;linear&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;SVC<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.svm.SVC.html\">?<span>Documentation for SVC</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>SVC(kernel=&#x27;linear&#x27;)</pre></div> </div></div></div></div>"
+      ],
+      "text/plain": [
+       "SVC(kernel='linear')"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "svm_model = SVC(kernel=\"linear\")\n",
+    "svm_model.fit(X_train_tfidf, y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "2dc1b193",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "y_pred = svm_model.predict(X_test_tfidf)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "92801e61",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy of SVM model: 0.9524632670700086\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "       FALSE       0.94      1.00      0.97       847\n",
+      "        TRUE       1.00      0.83      0.90       310\n",
+      "\n",
+      "    accuracy                           0.95      1157\n",
+      "   macro avg       0.97      0.91      0.94      1157\n",
+      "weighted avg       0.96      0.95      0.95      1157\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "accuracy = accuracy_score(y_test, y_pred)\n",
+    "print(\"Accuracy of SVM model:\", accuracy)\n",
+    "print(classification_report(y_test, y_pred))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1d1f6ebd",
+   "metadata": {},
+   "source": [
+    "## Random Forest Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "9170c174",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.ensemble import RandomForestClassifier\n",
+    "from sklearn.metrics import accuracy_score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "2092ca05",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "X_train, X_test, y_train, y_test = train_test_split(\n",
+    "    X, y, test_size=0.2, random_state=42\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "206296ce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tfidf_vectorizer = TfidfVectorizer(max_features=1000)\n",
+    "X_train_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
+    "X_test_tfidf = tfidf_vectorizer.transform(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "258bd78f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<style>#sk-container-id-4 {\n",
+       "  /* Definition of color scheme common for light and dark mode */\n",
+       "  --sklearn-color-text: black;\n",
+       "  --sklearn-color-line: gray;\n",
+       "  /* Definition of color scheme for unfitted estimators */\n",
+       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
+       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
+       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
+       "  --sklearn-color-unfitted-level-3: chocolate;\n",
+       "  /* Definition of color scheme for fitted estimators */\n",
+       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
+       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
+       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
+       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
+       "\n",
+       "  /* Specific color for light theme */\n",
+       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
+       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
+       "  --sklearn-color-icon: #696969;\n",
+       "\n",
+       "  @media (prefers-color-scheme: dark) {\n",
+       "    /* Redefinition of color scheme for dark theme */\n",
+       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
+       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
+       "    --sklearn-color-icon: #878787;\n",
+       "  }\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 pre {\n",
+       "  padding: 0;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 input.sk-hidden--visually {\n",
+       "  border: 0;\n",
+       "  clip: rect(1px 1px 1px 1px);\n",
+       "  clip: rect(1px, 1px, 1px, 1px);\n",
+       "  height: 1px;\n",
+       "  margin: -1px;\n",
+       "  overflow: hidden;\n",
+       "  padding: 0;\n",
+       "  position: absolute;\n",
+       "  width: 1px;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-dashed-wrapped {\n",
+       "  border: 1px dashed var(--sklearn-color-line);\n",
+       "  margin: 0 0.4em 0.5em 0.4em;\n",
+       "  box-sizing: border-box;\n",
+       "  padding-bottom: 0.4em;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-container {\n",
+       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
+       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
+       "     so we also need the `!important` here to be able to override the\n",
+       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
+       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
+       "  display: inline-block !important;\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-text-repr-fallback {\n",
+       "  display: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-parallel-item,\n",
+       "div.sk-serial,\n",
+       "div.sk-item {\n",
+       "  /* draw centered vertical line to link estimators */\n",
+       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
+       "  background-size: 2px 100%;\n",
+       "  background-repeat: no-repeat;\n",
+       "  background-position: center center;\n",
+       "}\n",
+       "\n",
+       "/* Parallel-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-4 div.sk-parallel-item::after {\n",
+       "  content: \"\";\n",
+       "  width: 100%;\n",
+       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
+       "  flex-grow: 1;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-parallel {\n",
+       "  display: flex;\n",
+       "  align-items: stretch;\n",
+       "  justify-content: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  position: relative;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-parallel-item {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-parallel-item:first-child::after {\n",
+       "  align-self: flex-end;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-parallel-item:last-child::after {\n",
+       "  align-self: flex-start;\n",
+       "  width: 50%;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-parallel-item:only-child::after {\n",
+       "  width: 0;\n",
+       "}\n",
+       "\n",
+       "/* Serial-specific style estimator block */\n",
+       "\n",
+       "#sk-container-id-4 div.sk-serial {\n",
+       "  display: flex;\n",
+       "  flex-direction: column;\n",
+       "  align-items: center;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  padding-right: 1em;\n",
+       "  padding-left: 1em;\n",
+       "}\n",
+       "\n",
+       "\n",
+       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
+       "clickable and can be expanded/collapsed.\n",
+       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
+       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
+       "*/\n",
+       "\n",
+       "/* Pipeline and ColumnTransformer style (default) */\n",
+       "\n",
+       "#sk-container-id-4 div.sk-toggleable {\n",
+       "  /* Default theme specific background. It is overwritten whether we have a\n",
+       "  specific estimator or a Pipeline/ColumnTransformer */\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable label */\n",
+       "#sk-container-id-4 label.sk-toggleable__label {\n",
+       "  cursor: pointer;\n",
+       "  display: block;\n",
+       "  width: 100%;\n",
+       "  margin-bottom: 0;\n",
+       "  padding: 0.5em;\n",
+       "  box-sizing: border-box;\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 label.sk-toggleable__label-arrow:before {\n",
+       "  /* Arrow on the left of the label */\n",
+       "  content: \"▸\";\n",
+       "  float: left;\n",
+       "  margin-right: 0.25em;\n",
+       "  color: var(--sklearn-color-icon);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "}\n",
+       "\n",
+       "/* Toggleable content - dropdown */\n",
+       "\n",
+       "#sk-container-id-4 div.sk-toggleable__content {\n",
+       "  max-height: 0;\n",
+       "  max-width: 0;\n",
+       "  overflow: hidden;\n",
+       "  text-align: left;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-toggleable__content.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-toggleable__content pre {\n",
+       "  margin: 0.2em;\n",
+       "  border-radius: 0.25em;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-toggleable__content.fitted pre {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
+       "  /* Expand drop-down */\n",
+       "  max-height: 200px;\n",
+       "  max-width: 100%;\n",
+       "  overflow: auto;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
+       "  content: \"▾\";\n",
+       "}\n",
+       "\n",
+       "/* Pipeline/ColumnTransformer-specific style */\n",
+       "\n",
+       "#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific style */\n",
+       "\n",
+       "/* Colorize estimator box */\n",
+       "#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-label label.sk-toggleable__label,\n",
+       "#sk-container-id-4 div.sk-label label {\n",
+       "  /* The background is the default theme color */\n",
+       "  color: var(--sklearn-color-text-on-default-background);\n",
+       "}\n",
+       "\n",
+       "/* On hover, darken the color of the background */\n",
+       "#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Label box, darken color on hover, fitted */\n",
+       "#sk-container-id-4 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Estimator label */\n",
+       "\n",
+       "#sk-container-id-4 div.sk-label label {\n",
+       "  font-family: monospace;\n",
+       "  font-weight: bold;\n",
+       "  display: inline-block;\n",
+       "  line-height: 1.2em;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-label-container {\n",
+       "  text-align: center;\n",
+       "}\n",
+       "\n",
+       "/* Estimator-specific */\n",
+       "#sk-container-id-4 div.sk-estimator {\n",
+       "  font-family: monospace;\n",
+       "  border: 1px dotted var(--sklearn-color-border-box);\n",
+       "  border-radius: 0.25em;\n",
+       "  box-sizing: border-box;\n",
+       "  margin-bottom: 0.5em;\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-estimator.fitted {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-0);\n",
+       "}\n",
+       "\n",
+       "/* on hover */\n",
+       "#sk-container-id-4 div.sk-estimator:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 div.sk-estimator.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-2);\n",
+       "}\n",
+       "\n",
+       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
+       "\n",
+       "/* Common style for \"i\" and \"?\" */\n",
+       "\n",
+       ".sk-estimator-doc-link,\n",
+       "a:link.sk-estimator-doc-link,\n",
+       "a:visited.sk-estimator-doc-link {\n",
+       "  float: right;\n",
+       "  font-size: smaller;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1em;\n",
+       "  height: 1em;\n",
+       "  width: 1em;\n",
+       "  text-decoration: none !important;\n",
+       "  margin-left: 1ex;\n",
+       "  /* unfitted */\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted,\n",
+       "a:link.sk-estimator-doc-link.fitted,\n",
+       "a:visited.sk-estimator-doc-link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
+       ".sk-estimator-doc-link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover,\n",
+       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
+       ".sk-estimator-doc-link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "/* Span, style for the box shown on hovering the info icon */\n",
+       ".sk-estimator-doc-link span {\n",
+       "  display: none;\n",
+       "  z-index: 9999;\n",
+       "  position: relative;\n",
+       "  font-weight: normal;\n",
+       "  right: .2ex;\n",
+       "  padding: .5ex;\n",
+       "  margin: .5ex;\n",
+       "  width: min-content;\n",
+       "  min-width: 20ex;\n",
+       "  max-width: 50ex;\n",
+       "  color: var(--sklearn-color-text);\n",
+       "  box-shadow: 2pt 2pt 4pt #999;\n",
+       "  /* unfitted */\n",
+       "  background: var(--sklearn-color-unfitted-level-0);\n",
+       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link.fitted span {\n",
+       "  /* fitted */\n",
+       "  background: var(--sklearn-color-fitted-level-0);\n",
+       "  border: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "\n",
+       ".sk-estimator-doc-link:hover span {\n",
+       "  display: block;\n",
+       "}\n",
+       "\n",
+       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
+       "\n",
+       "#sk-container-id-4 a.estimator_doc_link {\n",
+       "  float: right;\n",
+       "  font-size: 1rem;\n",
+       "  line-height: 1em;\n",
+       "  font-family: monospace;\n",
+       "  background-color: var(--sklearn-color-background);\n",
+       "  border-radius: 1rem;\n",
+       "  height: 1rem;\n",
+       "  width: 1rem;\n",
+       "  text-decoration: none;\n",
+       "  /* unfitted */\n",
+       "  color: var(--sklearn-color-unfitted-level-1);\n",
+       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 a.estimator_doc_link.fitted {\n",
+       "  /* fitted */\n",
+       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
+       "  color: var(--sklearn-color-fitted-level-1);\n",
+       "}\n",
+       "\n",
+       "/* On hover */\n",
+       "#sk-container-id-4 a.estimator_doc_link:hover {\n",
+       "  /* unfitted */\n",
+       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
+       "  color: var(--sklearn-color-background);\n",
+       "  text-decoration: none;\n",
+       "}\n",
+       "\n",
+       "#sk-container-id-4 a.estimator_doc_link.fitted:hover {\n",
+       "  /* fitted */\n",
+       "  background-color: var(--sklearn-color-fitted-level-3);\n",
+       "}\n",
+       "</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" checked><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;RandomForestClassifier<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.ensemble.RandomForestClassifier.html\">?<span>Documentation for RandomForestClassifier</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>RandomForestClassifier(random_state=42)</pre></div> </div></div></div></div>"
+      ],
+      "text/plain": [
+       "RandomForestClassifier(random_state=42)"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rf_model = RandomForestClassifier(n_estimators=100, random_state=42)\n",
+    "rf_model.fit(X_train_tfidf, y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "0e2910f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "y_pred = rf_model.predict(X_test_tfidf)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "f06900d3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy of Random Forest Model: 0.9628349178910977\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "       FALSE       0.96      1.00      0.98       847\n",
+      "        TRUE       0.99      0.87      0.93       310\n",
+      "\n",
+      "    accuracy                           0.96      1157\n",
+      "   macro avg       0.97      0.93      0.95      1157\n",
+      "weighted avg       0.96      0.96      0.96      1157\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "accuracy = accuracy_score(y_test, y_pred)\n",
+    "print(\"Accuracy of Random Forest Model:\", accuracy)\n",
+    "print(classification_report(y_test, y_pred))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/04_topic_modelling_benchmark_using_headline.ipynb

@@ -0,0 +1,913 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "46322fb5-5918-4b70-9689-9e0781439ac4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n",
+      "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "daf1e3d1-75ac-4299-8bed-2f413a49f9a6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import sent_tokenize\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "\n",
+    "import gensim\n",
+    "from gensim import corpora\n",
+    "from gensim import similarities\n",
+    "from gensim import models\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "# from wordcloud import WordCloud, ImageColorGenerator\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import os\n",
+    "import datetime\n",
+    "\n",
+    "import warnings\n",
+    "\n",
+    "warnings.filterwarnings(\"ignore\")\n",
+    "\n",
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "49e6de6b-71bd-4948-8827-52601406058f",
+   "metadata": {},
+   "source": [
+    "# Import Data with only the news headline and details"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "49222182-7811-4fa6-8c0a-21d3a546863e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_parquet(\"data/processed_data.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "3fb59a30",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>lat</th>\n",
+       "      <th>lon</th>\n",
+       "      <th>maritime_label</th>\n",
+       "      <th>found_ports</th>\n",
+       "      <th>contains_port_info</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "      <th>Headline_Details</th>\n",
+       "      <th>cleaned_Headline_Details</th>\n",
+       "      <th>binary_Headline_Details</th>\n",
+       "      <th>word_count</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-4.05608</td>\n",
+       "      <td>137.11302</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['freeport']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>[grasberg, grasberg, mine, worker, extend, str...</td>\n",
+       "      <td>[worker_grasberg_mine]</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2.0</td>\n",
+       "      <td>Indonesia: Undersea internet cables damaged by...</td>\n",
+       "      <td>News sources are stating that recent typhoons ...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Travel Warning</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>4/9/17 14:30</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['hong kong']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>14.0</td>\n",
+       "      <td>Indonesia: Undersea internet cables damaged by...</td>\n",
+       "      <td>[indonesia, undersea, internet, cable, damage,...</td>\n",
+       "      <td>[undersea_internet_cable]</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Congestion</td>\n",
+       "      <td>China</td>\n",
+       "      <td>27/4/17 9:16</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>29.52000</td>\n",
+       "      <td>121.33190</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['ningbo', 'qingdao', 'shanghai']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>[shanghai, port, congestion, impact, terminal,...</td>\n",
+       "      <td>[]</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>4.0</td>\n",
+       "      <td>UPDATE - Indonesia: Explosion at KP Terminal i...</td>\n",
+       "      <td>Updated local media sources from Jakarta indic...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 15:15</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-6.22465</td>\n",
+       "      <td>106.86700</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Explosion at KP Terminal i...</td>\n",
+       "      <td>[update, indonesia, explosion, at, kp, termina...</td>\n",
+       "      <td>[]</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>5.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 16:20</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>[update, indonesia, police, confirm, two, expl...</td>\n",
+       "      <td>[]</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    id                                           Headline  \\\n",
+       "0  1.0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1  2.0  Indonesia: Undersea internet cables damaged by...   \n",
+       "2  3.0  Shanghai port congestion impacts terminals in ...   \n",
+       "3  4.0  UPDATE - Indonesia: Explosion at KP Terminal i...   \n",
+       "4  5.0  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "\n",
+       "                                             Details  Severity  \\\n",
+       "0  Media sources indicate that workers at the Gra...  Moderate   \n",
+       "1  News sources are stating that recent typhoons ...     Minor   \n",
+       "2  The persisting port congestion at Shanghai’s Y...     Minor   \n",
+       "3  Updated local media sources from Jakarta indic...   Extreme   \n",
+       "4  According to local police in Jakarta, two expl...   Extreme   \n",
+       "\n",
+       "                     Category     Region       Datetime    Year       lat  \\\n",
+       "0         Mine Workers Strike  Indonesia  28/5/17 17:08  2017.0  -4.05608   \n",
+       "1              Travel Warning  Indonesia   4/9/17 14:30  2017.0       NaN   \n",
+       "2             Port Congestion      China   27/4/17 9:16  2017.0  29.52000   \n",
+       "3  Bombing, Police Operations  Indonesia  24/5/17 15:15  2017.0  -6.22465   \n",
+       "4  Bombing, Police Operations  Indonesia  24/5/17 16:20  2017.0       NaN   \n",
+       "\n",
+       "         lon  maritime_label                        found_ports  \\\n",
+       "0  137.11302           False                       ['freeport']   \n",
+       "1        NaN           False                      ['hong kong']   \n",
+       "2  121.33190            True  ['ningbo', 'qingdao', 'shanghai']   \n",
+       "3  106.86700            True                        ['jakarta']   \n",
+       "4        NaN            True                        ['jakarta']   \n",
+       "\n",
+       "   contains_port_info  if_labeled  Month  Week  \\\n",
+       "0                 1.0       False    5.0  21.0   \n",
+       "1                 1.0       False    4.0  14.0   \n",
+       "2                 1.0       False    4.0  17.0   \n",
+       "3                 1.0       False    5.0  21.0   \n",
+       "4                 1.0        True    5.0  21.0   \n",
+       "\n",
+       "                                    Headline_Details  \\\n",
+       "0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1  Indonesia: Undersea internet cables damaged by...   \n",
+       "2  Shanghai port congestion impacts terminals in ...   \n",
+       "3  UPDATE - Indonesia: Explosion at KP Terminal i...   \n",
+       "4  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "\n",
+       "                            cleaned_Headline_Details  \\\n",
+       "0  [grasberg, grasberg, mine, worker, extend, str...   \n",
+       "1  [indonesia, undersea, internet, cable, damage,...   \n",
+       "2  [shanghai, port, congestion, impact, terminal,...   \n",
+       "3  [update, indonesia, explosion, at, kp, termina...   \n",
+       "4  [update, indonesia, police, confirm, two, expl...   \n",
+       "\n",
+       "     binary_Headline_Details  word_count  \n",
+       "0     [worker_grasberg_mine]           1  \n",
+       "1  [undersea_internet_cable]           1  \n",
+       "2                         []           0  \n",
+       "3                         []           0  \n",
+       "4                         []           0  "
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "09113e88-66cc-414c-a953-da04db83c4ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(5778, 20)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "be6fc947",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DataFrames are identical: True\n"
+     ]
+    }
+   ],
+   "source": [
+    "df2 = pd.read_parquet(\"IS424_Data_Mining/code/LDA/processed_data.parquet\")\n",
+    "# Check if two DataFrames are exactly the same\n",
+    "are_identical = df.equals(df2)\n",
+    "print(f\"DataFrames are identical: {are_identical}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "037e74fc-bbcd-43e3-8346-799920cca8d8",
+   "metadata": {},
+   "source": [
+    "# Vectorisation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d67cef3a-59fb-4dd8-adc8-2cf288b90728",
+   "metadata": {},
+   "source": [
+    "NLP vectorization refers to the process of converting text data into numerical vectors that machine learning algorithms can understand and process. \n",
+    "\n",
+    "Bag-of-Words (BoW) is used here that represents text as a collection of unique words along with their frequencies. Each word is assigned an index, and the vector contains the count of each word present in the document."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "c95b7b8a-9767-469d-812d-c9a9d9fee0e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cleaned = df.copy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "de71c523-a59e-44b2-aa96-5f17d872c9c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "headline = cleaned.binary_Headline_Details"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "5b1e34e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array(['worker_grasberg_mine'], dtype=object)"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "headline[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "677055b4-978e-4253-90f4-3f903662e225",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# vectorise the words\n",
+    "doc_dict = gensim.corpora.Dictionary(headline)\n",
+    "docs_vecs = [doc_dict.doc2bow(doc) for doc in headline]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "a54d1768-b069-4936-a156-deaf0b506d93",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of unique tokens: 5319\n",
+      "Number of articles: 5778\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Number of unique tokens: %d\" % len(doc_dict))\n",
+    "print(\"Number of articles: %d\" % len(docs_vecs))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "9147fa86-1503-4252-bd9b-92fea1e6a926",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('due_dense_fog', 20),\n",
+      " ('strong_wind', 19),\n",
+      " ('indicate_average', 19),\n",
+      " ('day_port', 17),\n",
+      " ('san_antonio', 17),\n",
+      " ('vessel_port', 16),\n",
+      " ('low_visibility', 15),\n",
+      " ('average_wait', 13),\n",
+      " ('port_qingdao', 12),\n",
+      " ('east_coast_parkway', 12),\n",
+      " ('port_shanghai', 11),\n",
+      " ('port_hong_kong', 11),\n",
+      " ('severe_wind', 11),\n",
+      " ('blank_week_service', 11),\n",
+      " ('congestion_port_manila', 10),\n",
+      " ('wait_hour', 10),\n",
+      " ('day_situation', 9),\n",
+      " ('port_ningbo', 9),\n",
+      " ('coastal_area', 9),\n",
+      " ('strong_wind_forecast', 9),\n",
+      " ('vessel_arrival', 8),\n",
+      " ('high_wind', 8),\n",
+      " ('pomeranian_voivodeship', 8),\n",
+      " ('wait_day', 8),\n",
+      " ('previous_week', 8),\n",
+      " ('vessel_wait', 8),\n",
+      " ('average_wait_port', 8),\n",
+      " ('affect_operation_port', 7),\n",
+      " ('berth_manila_south', 7),\n",
+      " ('day_port_saigon', 7),\n",
+      " ('vessel_port_hong', 7),\n",
+      " ('vessel_port_shanghai', 7),\n",
+      " ('high_risk_port', 7),\n",
+      " ('wait_hour_port', 7),\n",
+      " ('qianwan_container', 7),\n",
+      " ('day_port_shanghai', 6),\n",
+      " ('international_container', 6),\n",
+      " ('tropical_storm', 6),\n",
+      " ('arrive_window', 6),\n",
+      " ('moderate_risk_port', 6),\n",
+      " ('vessel_port_qingdao', 6),\n",
+      " ('landside_rail_operation', 6),\n",
+      " ('dp_world_southampton', 6),\n",
+      " ('high_yard_density', 6),\n",
+      " ('vessel_port_ningbo', 6),\n",
+      " ('port_busan', 6),\n",
+      " ('strong_wind_area', 6),\n",
+      " ('new_york_city', 6),\n",
+      " ('duration_closure', 6),\n",
+      " ('vessel_berth_port', 5),\n",
+      " ('wait_decrease_day', 5),\n",
+      " ('disrupt_operation_port', 5),\n",
+      " ('waterside_landside_operation', 5),\n",
+      " ('congest_vessel', 5),\n",
+      " ('port_charleston', 5),\n",
+      " ('san_antonio_puerto', 5),\n",
+      " ('yantian_shipping', 5),\n",
+      " ('port_saigon', 5),\n",
+      " ('port_ho_chi', 5),\n",
+      " ('engine_failure', 5),\n",
+      " ('port_hong', 5),\n",
+      " ('dense_fog', 5),\n",
+      " ('disrupt_port_operation', 5),\n",
+      " ('high_wind_port', 5),\n",
+      " ('vessel_wait_day', 5),\n",
+      " ('vessel_port_tanjung', 5),\n",
+      " ('landside_waterside_operation', 5),\n",
+      " ('port_durban', 5),\n",
+      " ('port_manila', 5),\n",
+      " ('next_day_morning', 5),\n",
+      " ('strong_wind_halt', 5),\n",
+      " ('houston_ship_channel', 5),\n",
+      " ('low_visibility_forecast', 5),\n",
+      " ('full_port', 5),\n",
+      " ('tropical_storm_hanna', 5),\n",
+      " ('likely_port_closure', 5),\n",
+      " ('strong_wind_industry', 5),\n",
+      " ('hour_qingdao_qianwan', 5),\n",
+      " ('antonio_halt_operation', 5),\n",
+      " ('high_chance_port', 4),\n",
+      " ('pilot_service', 4),\n",
+      " ('low_productivity', 4),\n",
+      " ('heavy_rain', 4),\n",
+      " ('resume_operation', 4),\n",
+      " ('terminal_indicate', 4),\n",
+      " ('port_fuzhou', 4),\n",
+      " ('disrupt_port', 4),\n",
+      " ('nhava_sheva', 4),\n",
+      " ('berth_day', 4),\n",
+      " ('congestion_port', 4),\n",
+      " ('indicate_waterside_operation', 4),\n",
+      " ('injury_damage_incident', 4),\n",
+      " ('due_fog', 4),\n",
+      " ('wait_vessel_port', 4),\n",
+      " ('kong_industry', 4),\n",
+      " ('hong_kong', 4),\n",
+      " ('wait_vessel_berth', 4),\n",
+      " ('terminal_port', 4),\n",
+      " ('wait_vessel', 4),\n",
+      " ('affect_port_operation', 4)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Calculate word frequencies\n",
+    "word_frequencies = {doc_dict[word_id]: freq for word_id, freq in doc_dict.cfs.items()}\n",
+    "sorted_words = sorted(word_frequencies.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "pprint(sorted_words[:100])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5ed78239-2ce1-4784-a8f4-4c7438c8627b",
+   "metadata": {},
+   "source": [
+    "# LDA Modelling"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9db83273-461d-4f70-b23f-ec967579d94f",
+   "metadata": {},
+   "source": [
+    "## Benchmark Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "e6d577bd-9936-4d45-be90-345af2eb4827",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Build LDA benchmark model\n",
+    "lda_model = gensim.models.LdaMulticore(\n",
+    "    corpus=docs_vecs,\n",
+    "    id2word=doc_dict,\n",
+    "    num_topics=4,\n",
+    "    random_state=42,\n",
+    "    chunksize=100,\n",
+    "    passes=10,\n",
+    "    per_word_topics=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "c4f1521f-5f43-40d2-a3a3-a8ac2ca6fec2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[(0,\n",
+      "  '0.003*\"indicate_average\" + 0.002*\"coastal_area\" + 0.002*\"vessel_port_hong\" '\n",
+      "  '+ 0.002*\"east_coast_parkway\" + 0.002*\"port_charleston\" + '\n",
+      "  '0.002*\"average_wait_port\" + 0.002*\"severe_wind\" + '\n",
+      "  '0.002*\"pomeranian_voivodeship\" + 0.002*\"engine_failure\" + '\n",
+      "  '0.002*\"due_dense_fog\"'),\n",
+      " (1,\n",
+      "  '0.005*\"san_antonio\" + 0.003*\"blank_week_service\" + 0.003*\"wait_hour\" + '\n",
+      "  '0.003*\"day_port_saigon\" + 0.002*\"low_visibility\" + 0.002*\"high_wind\" + '\n",
+      "  '0.002*\"waterside_landside_operation\" + 0.002*\"tropical_storm\" + '\n",
+      "  '0.002*\"qianwan_container\" + 0.002*\"port_ningbo\"'),\n",
+      " (2,\n",
+      "  '0.004*\"strong_wind\" + 0.002*\"port_shanghai\" + 0.002*\"port_qingdao\" + '\n",
+      "  '0.002*\"vessel_port_qingdao\" + 0.002*\"day_port_shanghai\" + '\n",
+      "  '0.002*\"indicate_average\" + 0.002*\"disrupt_operation_port\" + '\n",
+      "  '0.002*\"strong_wind_area\" + 0.002*\"port_ho_chi\" + '\n",
+      "  '0.001*\"operation_pier_port\"'),\n",
+      " (3,\n",
+      "  '0.004*\"port_hong_kong\" + 0.003*\"vessel_port\" + 0.003*\"day_port\" + '\n",
+      "  '0.003*\"congestion_port_manila\" + 0.002*\"berth_manila_south\" + '\n",
+      "  '0.002*\"arrive_window\" + 0.002*\"landside_rail_operation\" + '\n",
+      "  '0.002*\"international_container\" + 0.002*\"day_situation\" + '\n",
+      "  '0.002*\"congestion_port\"')]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Print the Keyword in the 10 topics\n",
+    "pprint(lda_model.print_topics())\n",
+    "doc_lda = lda_model[docs_vecs]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "fd57b1f4-a6cd-41e8-964f-d8a1d30aa3c9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Coherence Score LDAModel:  0.7011993291597081\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Compute Benchmark Coherence Score\n",
+    "coherence_model_lda = CoherenceModel(\n",
+    "    model=lda_model, texts=headline, dictionary=doc_dict, coherence=\"c_v\"\n",
+    ")\n",
+    "coherence_lda = coherence_model_lda.get_coherence()\n",
+    "print(\"\\nCoherence Score LDAModel: \", coherence_lda)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "152e5a3a-7afe-4fb8-a02f-d7492ad80936",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Perplexity for LDAModel:  -9.594271136114548\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Compute Benchmark Perplexity\n",
+    "perplex = lda_model.log_perplexity(docs_vecs, total_docs=None)  # For LDAModel\n",
+    "# a measure of how good the model is. lower the better.\n",
+    "\n",
+    "print(\"\\nPerplexity for LDAModel: \", perplex)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "7dd3a60a-5c6f-4249-9868-30528a5b0ac8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=30092) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis\n",
+    "\n",
+    "# feed the LDA model into the pyLDAvis instance\n",
+    "pyLDAvis.enable_notebook()\n",
+    "visual = gensimvis.prepare(lda_model, docs_vecs, doc_dict)\n",
+    "\n",
+    "# Save the output to the html file\n",
+    "pyLDAvis.save_html(visual, \"topic_viz_benchmark.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "48bd84a0-ce9b-4117-bf6e-d9afb9936147",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Topic Keywords</th>\n",
+       "      <th>Topic ID</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.003*\"indicate_average\" + 0.002*\"coastal_area\" + 0.002*\"vessel_port_hong\" + 0.002*\"east_coast_parkway\" + 0.002*\"port_charleston\" + 0.002*\"average_wait_port\"</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.005*\"san_antonio\" + 0.003*\"blank_week_service\" + 0.003*\"wait_hour\" + 0.003*\"day_port_saigon\" + 0.002*\"low_visibility\" + 0.002*\"high_wind\"</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0.004*\"strong_wind\" + 0.002*\"port_shanghai\" + 0.002*\"port_qingdao\" + 0.002*\"vessel_port_qingdao\" + 0.002*\"day_port_shanghai\" + 0.002*\"indicate_average\"</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0.004*\"port_hong_kong\" + 0.003*\"vessel_port\" + 0.003*\"day_port\" + 0.003*\"congestion_port_manila\" + 0.002*\"berth_manila_south\" + 0.002*\"arrive_window\"</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                  Topic Keywords  \\\n",
+       "0  0.003*\"indicate_average\" + 0.002*\"coastal_area\" + 0.002*\"vessel_port_hong\" + 0.002*\"east_coast_parkway\" + 0.002*\"port_charleston\" + 0.002*\"average_wait_port\"   \n",
+       "1                    0.005*\"san_antonio\" + 0.003*\"blank_week_service\" + 0.003*\"wait_hour\" + 0.003*\"day_port_saigon\" + 0.002*\"low_visibility\" + 0.002*\"high_wind\"   \n",
+       "2        0.004*\"strong_wind\" + 0.002*\"port_shanghai\" + 0.002*\"port_qingdao\" + 0.002*\"vessel_port_qingdao\" + 0.002*\"day_port_shanghai\" + 0.002*\"indicate_average\"   \n",
+       "3          0.004*\"port_hong_kong\" + 0.003*\"vessel_port\" + 0.003*\"day_port\" + 0.003*\"congestion_port_manila\" + 0.002*\"berth_manila_south\" + 0.002*\"arrive_window\"   \n",
+       "\n",
+       "   Topic ID  \n",
+       "0         0  \n",
+       "1         1  \n",
+       "2         2  \n",
+       "3         3  "
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.set_option(\"max_colwidth\", 200)\n",
+    "# Get the topics and their top keywords into a dataframe\n",
+    "topics = lda_model.show_topics(num_words=6)\n",
+    "\n",
+    "topic_keywords = pd.DataFrame()\n",
+    "for topic_id, topic in topics:\n",
+    "    topic_keywords.at[topic_id, \"Topic Keywords\"] = topic\n",
+    "\n",
+    "topic_keywords[\"Topic ID\"] = topic_keywords.index\n",
+    "# topic_keywords['Topic Name'] = topic_mapping\n",
+    "topic_keywords"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3247fe12",
+   "metadata": {},
+   "source": [
+    "## Conclusion\n",
+    "\n",
+    "No significant insights gained from the model result as we cannot identify any topics, maybe more data is needed, will deploy web scraping for full news content after mid term."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/05b_newsScraper_run_2.ipynb

@@ -0,0 +1,1074 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "42801c6a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n",
+      "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ddf1e32e-7751-43db-9b5a-22cb08e35c6c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import nltk\n",
+    "from nltk.corpus import PlaintextCorpusReader\n",
+    "from nltk.corpus import stopwords\n",
+    "from nltk.stem.porter import *\n",
+    "from nltk import pos_tag, word_tokenize\n",
+    "from nltk.stem import WordNetLemmatizer\n",
+    "from nltk.probability import FreqDist\n",
+    "from nltk.tokenize import sent_tokenize\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "import contractions\n",
+    "\n",
+    "import gensim\n",
+    "from gensim import corpora\n",
+    "from gensim import similarities\n",
+    "from gensim import models\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "# from wordcloud import WordCloud, ImageColorGenerator\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import os\n",
+    "import glob\n",
+    "import json\n",
+    "\n",
+    "import pickle\n",
+    "from datetime import datetime\n",
+    "import datetime\n",
+    "from newspaper import Article\n",
+    "from gnews import GNews"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a9c6ed07-a7d0-4aaa-b571-038919c75e05",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Function to get full news article by headline\n",
+    "def get_news_article(headline):\n",
+    "    gnews = GNews()\n",
+    "    articles = gnews.get_news(headline)\n",
+    "\n",
+    "    if articles:\n",
+    "        for article in articles:\n",
+    "            # Assuming the headline contains the URL of the news article\n",
+    "            try:\n",
+    "                article = Article(article[\"url\"])\n",
+    "                article.download()\n",
+    "                article.parse()\n",
+    "                return article.url, article.title, article.text\n",
+    "            except Exception as e:\n",
+    "                # link cannot be scraped\n",
+    "                # print(\"get_news_article - exception:\", e)\n",
+    "                return [\n",
+    "                    f\"cannot scrape the url: {e}\",\n",
+    "                    \"cannot scrape the title\",\n",
+    "                    \"cannot scrape the content\",\n",
+    "                ]\n",
+    "    # No articles found for the given title.\n",
+    "    else:\n",
+    "        return [\"no url found\", \"no title found\", \"no content found\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "0acaf899-cd96-4424-b825-c71bf042355a",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "# Read the CSV file with news titles\n",
+    "csv_file_path = \"data/cleaned_data.csv\"\n",
+    "df = pd.read_csv(csv_file_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "465acb47-5f57-4d90-a272-7972d56e002b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(5782, 17)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "801d892d-3b87-4ecf-805e-3441815d00f1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# drop empty news\n",
+    "df.dropna(subset=[\"Headline\"], inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "a65a8eaf",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(5780, 17)"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "1267bf68-4726-49bf-ae21-9fed09682945",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "70"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df[[\"Year\", \"Headline\", \"Region\"]].duplicated().sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "51e0f672",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(5710, 17)"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_uni = df.drop_duplicates(subset=[\"Year\", \"Headline\", \"Region\"], keep=\"first\")\n",
+    "df_uni.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "f3c35a3c-d852-4285-86bf-3f7db7df78e7",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 13%|█▎        | 757/5710 [27:31<3:29:09,  2.53s/it] /Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/dateutil/parser/_parser.py:1207: UnknownTimezoneWarning: tzname PST identified but not understood.  Pass `tzinfos` argument in order to correctly return a timezone-aware datetime.  In a future version, this will raise an exception.\n",
+      "  warnings.warn(\"tzname {tzname} identified but not understood.  \"\n",
+      " 28%|██▊       | 1588/5710 [57:29<3:22:58,  2.95s/it] /Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/dateutil/parser/_parser.py:1207: UnknownTimezoneWarning: tzname PDT identified but not understood.  Pass `tzinfos` argument in order to correctly return a timezone-aware datetime.  In a future version, this will raise an exception.\n",
+      "  warnings.warn(\"tzname {tzname} identified but not understood.  \"\n",
+      " 68%|██████▊   | 3862/5710 [2:24:11<1:04:55,  2.11s/it] 06/29/2024 07:38:31 PM - newspaper ignoring lxml node error: Input object is not an XML element: HtmlProcessingInstruction\n",
+      " 70%|██████▉   | 3978/5710 [2:27:39<1:44:23,  3.62s/it]Building prefix dict from /Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/jieba/dict.txt ...\n",
+      "06/29/2024 07:41:56 PM - Building prefix dict from /Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/jieba/dict.txt ...\n",
+      "Loading model from cache /var/folders/7x/56svhln929zdh2xhr3mwqg4r0000gn/T/jieba.cache\n",
+      "06/29/2024 07:41:56 PM - Loading model from cache /var/folders/7x/56svhln929zdh2xhr3mwqg4r0000gn/T/jieba.cache\n",
+      "Loading model cost 0.28935670852661133 seconds.\n",
+      "06/29/2024 07:41:56 PM - Loading model cost 0.28935670852661133 seconds.\n",
+      "Prefix dict has been built succesfully.\n",
+      "06/29/2024 07:41:56 PM - Prefix dict has been built succesfully.\n",
+      "100%|██████████| 5710/5710 [3:30:27<00:00,  2.21s/it]  "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 16min 31s, sys: 3min 20s, total: 19min 51s\n",
+      "Wall time: 3h 30min 27s\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "<timed exec>:20: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "<timed exec>:21: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "<timed exec>:22: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "# Iterate through each row and get the full news article\n",
+    "tes = df_uni\n",
+    "\n",
+    "urls = []\n",
+    "titles = []\n",
+    "contents = []\n",
+    "\n",
+    "for index in tqdm(range(len(tes))):\n",
+    "    row = tes.iloc[index]\n",
+    "    headline = row[\"Headline\"]\n",
+    "\n",
+    "    results = get_news_article(headline)\n",
+    "    urls.append(results[0])\n",
+    "    titles.append(results[1])\n",
+    "    contents.append(results[2])\n",
+    "\n",
+    "# Update the DataFrame with the fetched data\n",
+    "tes[\"url\"] = urls\n",
+    "tes[\"title\"] = titles\n",
+    "tes[\"content\"] = contents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "66d98c3b-c1f2-42e6-ae5f-8192c5a91f44",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>lat</th>\n",
+       "      <th>lon</th>\n",
+       "      <th>maritime_label</th>\n",
+       "      <th>found_ports</th>\n",
+       "      <th>contains_port_info</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "      <th>Headline_Details</th>\n",
+       "      <th>url</th>\n",
+       "      <th>title</th>\n",
+       "      <th>content</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-4.05608</td>\n",
+       "      <td>137.11302</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['freeport']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZ2h0d...</td>\n",
+       "      <td>Freeport Indonesia mine workers extend strike ...</td>\n",
+       "      <td>By Wilda Asmarini and Susan Taylor\\n\\nJAKARTA/...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2.0</td>\n",
+       "      <td>Indonesia: Undersea internet cables damaged by...</td>\n",
+       "      <td>News sources are stating that recent typhoons ...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Travel Warning</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>4/9/17 14:30</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['hong kong']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>14.0</td>\n",
+       "      <td>Indonesia: Undersea internet cables damaged by...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Congestion</td>\n",
+       "      <td>China</td>\n",
+       "      <td>27/4/17 9:16</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>29.52000</td>\n",
+       "      <td>121.33190</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['ningbo', 'qingdao', 'shanghai']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZmh0d...</td>\n",
+       "      <td>Bad weather causing delays, congestion at majo...</td>\n",
+       "      <td>LA-LB ports investing $25 million in charging ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>4.0</td>\n",
+       "      <td>UPDATE - Indonesia: Explosion at KP Terminal i...</td>\n",
+       "      <td>Updated local media sources from Jakarta indic...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 15:15</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-6.22465</td>\n",
+       "      <td>106.86700</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Explosion at KP Terminal i...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>5.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 16:20</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiV2h0d...</td>\n",
+       "      <td>Three dead confirmed in Jakarta double bombing</td>\n",
+       "      <td>Two explosions strike bus station in Indonesia...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5777</th>\n",
+       "      <td>5776.0</td>\n",
+       "      <td>Winter storm may bring inclement weather condi...</td>\n",
+       "      <td>Intelligence received by Everstream Analytics ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Ice Storm</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>16/12/20 2:47</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>38.90072</td>\n",
+       "      <td>-77.05440</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['new york']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>51.0</td>\n",
+       "      <td>Winter storm may bring inclement weather condi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiWWh0d...</td>\n",
+       "      <td>Winter Storm Warnings posted for parts of Nort...</td>\n",
+       "      <td>A coastal storm is set to deliver another slop...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5778</th>\n",
+       "      <td>5777.0</td>\n",
+       "      <td>Winter weather expected to continue to impact ...</td>\n",
+       "      <td>Meteorological sources indicate that a series ...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Roadway Closure / Disruption, Ground Transport...</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>31/12/20 18:15</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>41.30357</td>\n",
+       "      <td>-72.90561</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['new york', 'virginia']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>53.0</td>\n",
+       "      <td>Winter weather expected to continue to impact ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiT2h0d...</td>\n",
+       "      <td>Deadly nor'easter wreaks havoc in New England ...</td>\n",
+       "      <td>BOSTON – A deadly late-season nor’easter that ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5779</th>\n",
+       "      <td>5778.0</td>\n",
+       "      <td>Workers of Svitzer Australia plan to strike on...</td>\n",
+       "      <td>Industry sources report on December 7 that Svi...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Industrial Action</td>\n",
+       "      <td>Australia</td>\n",
+       "      <td>7/12/20 6:16</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>-38.35169</td>\n",
+       "      <td>145.25050</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['geelong', 'melbourne']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>7.0</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>Workers of Svitzer Australia plan to strike on...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5780</th>\n",
+       "      <td>5779.0</td>\n",
+       "      <td>Workers stage 24-hour stoppage at DP World Ter...</td>\n",
+       "      <td>Industry sources indicate on December 14 that ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Port Strike</td>\n",
+       "      <td>Australia</td>\n",
+       "      <td>14/12/20 16:52</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['fremantle']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>51.0</td>\n",
+       "      <td>Workers stage 24-hour stoppage at DP World Ter...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiOWh0d...</td>\n",
+       "      <td>Union shuts down strikes as DP World threatens...</td>\n",
+       "      <td>The Maritime Union of Australia (MUA) has curt...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5781</th>\n",
+       "      <td>5780.0</td>\n",
+       "      <td>Workers with Ertsoverslagbedrijf Europoort C.V...</td>\n",
+       "      <td>On November 17, Dutch media sources reported t...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Strike</td>\n",
+       "      <td>Netherlands</td>\n",
+       "      <td>18/11/20 17:55</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>51.96121</td>\n",
+       "      <td>4.10550</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['rotterdam']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>11.0</td>\n",
+       "      <td>47.0</td>\n",
+       "      <td>Workers with Ertsoverslagbedrijf Europoort C.V...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5710 rows × 20 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          id                                           Headline  \\\n",
+       "0        1.0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1        2.0  Indonesia: Undersea internet cables damaged by...   \n",
+       "2        3.0  Shanghai port congestion impacts terminals in ...   \n",
+       "3        4.0  UPDATE - Indonesia: Explosion at KP Terminal i...   \n",
+       "4        5.0  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "...      ...                                                ...   \n",
+       "5777  5776.0  Winter storm may bring inclement weather condi...   \n",
+       "5778  5777.0  Winter weather expected to continue to impact ...   \n",
+       "5779  5778.0  Workers of Svitzer Australia plan to strike on...   \n",
+       "5780  5779.0  Workers stage 24-hour stoppage at DP World Ter...   \n",
+       "5781  5780.0  Workers with Ertsoverslagbedrijf Europoort C.V...   \n",
+       "\n",
+       "                                                Details  Severity  \\\n",
+       "0     Media sources indicate that workers at the Gra...  Moderate   \n",
+       "1     News sources are stating that recent typhoons ...     Minor   \n",
+       "2     The persisting port congestion at Shanghai’s Y...     Minor   \n",
+       "3     Updated local media sources from Jakarta indic...   Extreme   \n",
+       "4     According to local police in Jakarta, two expl...   Extreme   \n",
+       "...                                                 ...       ...   \n",
+       "5777  Intelligence received by Everstream Analytics ...  Moderate   \n",
+       "5778  Meteorological sources indicate that a series ...     Minor   \n",
+       "5779  Industry sources report on December 7 that Svi...  Moderate   \n",
+       "5780  Industry sources indicate on December 14 that ...  Moderate   \n",
+       "5781  On November 17, Dutch media sources reported t...     Minor   \n",
+       "\n",
+       "                                               Category         Region  \\\n",
+       "0                                   Mine Workers Strike      Indonesia   \n",
+       "1                                        Travel Warning      Indonesia   \n",
+       "2                                       Port Congestion          China   \n",
+       "3                            Bombing, Police Operations      Indonesia   \n",
+       "4                            Bombing, Police Operations      Indonesia   \n",
+       "...                                                 ...            ...   \n",
+       "5777                                          Ice Storm  United States   \n",
+       "5778  Roadway Closure / Disruption, Ground Transport...  United States   \n",
+       "5779                                  Industrial Action      Australia   \n",
+       "5780                                        Port Strike      Australia   \n",
+       "5781                                        Port Strike    Netherlands   \n",
+       "\n",
+       "            Datetime    Year       lat        lon maritime_label  \\\n",
+       "0      28/5/17 17:08  2017.0  -4.05608  137.11302          False   \n",
+       "1       4/9/17 14:30  2017.0       NaN        NaN          False   \n",
+       "2       27/4/17 9:16  2017.0  29.52000  121.33190           True   \n",
+       "3      24/5/17 15:15  2017.0  -6.22465  106.86700           True   \n",
+       "4      24/5/17 16:20  2017.0       NaN        NaN           True   \n",
+       "...              ...     ...       ...        ...            ...   \n",
+       "5777   16/12/20 2:47  2020.0  38.90072  -77.05440          False   \n",
+       "5778  31/12/20 18:15  2020.0  41.30357  -72.90561          False   \n",
+       "5779    7/12/20 6:16  2020.0 -38.35169  145.25050          False   \n",
+       "5780  14/12/20 16:52  2020.0       NaN        NaN           True   \n",
+       "5781  18/11/20 17:55  2020.0  51.96121    4.10550          False   \n",
+       "\n",
+       "                            found_ports  contains_port_info if_labeled  Month  \\\n",
+       "0                          ['freeport']                 1.0      False    5.0   \n",
+       "1                         ['hong kong']                 1.0      False    4.0   \n",
+       "2     ['ningbo', 'qingdao', 'shanghai']                 1.0      False    4.0   \n",
+       "3                           ['jakarta']                 1.0      False    5.0   \n",
+       "4                           ['jakarta']                 1.0       True    5.0   \n",
+       "...                                 ...                 ...        ...    ...   \n",
+       "5777                       ['new york']                 1.0      False   12.0   \n",
+       "5778           ['new york', 'virginia']                 1.0       True   12.0   \n",
+       "5779           ['geelong', 'melbourne']                 1.0      False    7.0   \n",
+       "5780                      ['fremantle']                 1.0       True   12.0   \n",
+       "5781                      ['rotterdam']                 1.0       True   11.0   \n",
+       "\n",
+       "      Week                                   Headline_Details  \\\n",
+       "0     21.0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1     14.0  Indonesia: Undersea internet cables damaged by...   \n",
+       "2     17.0  Shanghai port congestion impacts terminals in ...   \n",
+       "3     21.0  UPDATE - Indonesia: Explosion at KP Terminal i...   \n",
+       "4     21.0  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "...    ...                                                ...   \n",
+       "5777  51.0  Winter storm may bring inclement weather condi...   \n",
+       "5778  53.0  Winter weather expected to continue to impact ...   \n",
+       "5779  28.0  Workers of Svitzer Australia plan to strike on...   \n",
+       "5780  51.0  Workers stage 24-hour stoppage at DP World Ter...   \n",
+       "5781  47.0  Workers with Ertsoverslagbedrijf Europoort C.V...   \n",
+       "\n",
+       "                                                    url  \\\n",
+       "0     https://news.google.com/rss/articles/CBMiZ2h0d...   \n",
+       "1                                          no url found   \n",
+       "2     https://news.google.com/rss/articles/CBMiZmh0d...   \n",
+       "3                                          no url found   \n",
+       "4     https://news.google.com/rss/articles/CBMiV2h0d...   \n",
+       "...                                                 ...   \n",
+       "5777  https://news.google.com/rss/articles/CBMiWWh0d...   \n",
+       "5778  https://news.google.com/rss/articles/CBMiT2h0d...   \n",
+       "5779                                       no url found   \n",
+       "5780  https://news.google.com/rss/articles/CBMiOWh0d...   \n",
+       "5781                                       no url found   \n",
+       "\n",
+       "                                                  title  \\\n",
+       "0     Freeport Indonesia mine workers extend strike ...   \n",
+       "1                                        no title found   \n",
+       "2     Bad weather causing delays, congestion at majo...   \n",
+       "3                                        no title found   \n",
+       "4        Three dead confirmed in Jakarta double bombing   \n",
+       "...                                                 ...   \n",
+       "5777  Winter Storm Warnings posted for parts of Nort...   \n",
+       "5778  Deadly nor'easter wreaks havoc in New England ...   \n",
+       "5779                                     no title found   \n",
+       "5780  Union shuts down strikes as DP World threatens...   \n",
+       "5781                                     no title found   \n",
+       "\n",
+       "                                                content  \n",
+       "0     By Wilda Asmarini and Susan Taylor\\n\\nJAKARTA/...  \n",
+       "1                                      no content found  \n",
+       "2     LA-LB ports investing $25 million in charging ...  \n",
+       "3                                      no content found  \n",
+       "4     Two explosions strike bus station in Indonesia...  \n",
+       "...                                                 ...  \n",
+       "5777  A coastal storm is set to deliver another slop...  \n",
+       "5778  BOSTON – A deadly late-season nor’easter that ...  \n",
+       "5779                                   no content found  \n",
+       "5780  The Maritime Union of Australia (MUA) has curt...  \n",
+       "5781                                   no content found  \n",
+       "\n",
+       "[5710 rows x 20 columns]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "d6fe260c-1bbb-4556-b9ac-2b9b62c5ba5f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3218"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "count_rows = tes[\n",
+    "    ~tes[\"content\"].isin([\"cannot scrape the content\", \"no content found\"])\n",
+    "].shape[0]\n",
+    "count_rows"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "6818fe7b-6e7d-4d30-8ee8-866b29fd40c8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "2132"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "count_rows = tes[tes[\"content\"].isin([\"no content found\"])].shape[0]\n",
+    "count_rows"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "f59e2ce4-072a-43e2-a247-fd399c388c2d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "360"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "count_rows = tes[tes[\"content\"].isin([\"cannot scrape the content\"])].shape[0]\n",
+    "count_rows"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "8063b964",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "scrapped_df1 = pd.read_parquet(\"data/scrapped_data1.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "78896dd3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(5710, 20)"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "scrapped_df1.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "316d6be5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 5710/5710 [00:00<00:00, 35312.10it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 162 ms, sys: 3.81 ms, total: 166 ms\n",
+      "Wall time: 165 ms\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "<timed exec>:23: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "<timed exec>:24: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "<timed exec>:25: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "urls = []\n",
+    "titles = []\n",
+    "contents = []\n",
+    "\n",
+    "not_found = [\"cannot scrape the content\", \"no content found\", None]\n",
+    "\n",
+    "for index in tqdm(range(len(tes))):\n",
+    "    row1 = scrapped_df1.iloc[index]\n",
+    "    row2 = tes.iloc[index]\n",
+    "    \n",
+    "    if row1[\"content\"] in not_found and not row2[\"content\"] in not_found and row2[\"title\"] is not None:\n",
+    "        row = row2\n",
+    "    else:\n",
+    "        row = row1\n",
+    "\n",
+    "    urls.append(row[\"url\"])\n",
+    "    titles.append(row[\"title\"])\n",
+    "    contents.append(row[\"content\"])\n",
+    "\n",
+    "# Update the DataFrame with the fetched data\n",
+    "tes[\"url\"] = urls\n",
+    "tes[\"title\"] = titles\n",
+    "tes[\"content\"] = contents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "4926f410",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3694"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "count_rows = tes[~tes[\"content\"].isin(not_found)].shape[0]\n",
+    "count_rows"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "44bff271",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.6469352014010508"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "count_rows / len(tes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "e573b48a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1945"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "count_rows = tes[tes[\"content\"].isin([\"no content found\"])].shape[0]\n",
+    "count_rows"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "d4a527b3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "71"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "count_rows = tes[tes[\"content\"].isin([\"cannot scrape the content\"])].shape[0]\n",
+    "count_rows"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "158acaec",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/7x/56svhln929zdh2xhr3mwqg4r0000gn/T/ipykernel_64033/1183224327.py:1: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
+      "  tes[\"id\"] = tes[\"id\"].astype(int)\n"
+     ]
+    }
+   ],
+   "source": [
+    "tes[\"id\"] = tes[\"id\"].astype(int)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "154aada9-8e4a-479a-8288-a63b72ae86ad",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tes.to_csv(\"data/scrapped_data2.csv\", index=False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/07_topic_modelling_minor.ipynb

@@ -0,0 +1,1564 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "46322fb5-5918-4b70-9689-9e0781439ac4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n",
+      "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "daf1e3d1-75ac-4299-8bed-2f413a49f9a6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import sent_tokenize\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "\n",
+    "import gensim\n",
+    "from gensim import corpora\n",
+    "from gensim import similarities\n",
+    "from gensim import models\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "# from wordcloud import WordCloud, ImageColorGenerator\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import os\n",
+    "import datetime\n",
+    "\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "49e6de6b-71bd-4948-8827-52601406058f",
+   "metadata": {},
+   "source": [
+    "# Import Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "49222182-7811-4fa6-8c0a-21d3a546863e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_parquet('data/processed_data2.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3fb59a30",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>lat</th>\n",
+       "      <th>lon</th>\n",
+       "      <th>...</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "      <th>Headline_Details</th>\n",
+       "      <th>url</th>\n",
+       "      <th>title</th>\n",
+       "      <th>content</th>\n",
+       "      <th>cleaned_content</th>\n",
+       "      <th>binary_content</th>\n",
+       "      <th>word_count</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-4.05608</td>\n",
+       "      <td>137.11302</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZ2h0d...</td>\n",
+       "      <td>Freeport Indonesia mine workers extend strike ...</td>\n",
+       "      <td>Trucks are seen on a road in the Grasberg copp...</td>\n",
+       "      <td>[truck, be, see, on, road, in, grasberg, coppe...</td>\n",
+       "      <td>[adkerson_jakarta_try, agreement_freeport_indo...</td>\n",
+       "      <td>53</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Congestion</td>\n",
+       "      <td>China</td>\n",
+       "      <td>27/4/17 9:16</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>29.52000</td>\n",
+       "      <td>121.33190</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiVWh0d...</td>\n",
+       "      <td>Typhoon Muifa to shut China ports for second t...</td>\n",
+       "      <td>By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...</td>\n",
+       "      <td>[by, sam, whelan, typhoon, have, prompt, port,...</td>\n",
+       "      <td>[additional_ripple_effect, avoid_path_typhoon,...</td>\n",
+       "      <td>44</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>5</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 16:20</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZWh0d...</td>\n",
+       "      <td>Jakarta Police Receive 2 More Reports on Coldp...</td>\n",
+       "      <td>TEMPO.CO, Jakarta - South Jakarta Metro Police...</td>\n",
+       "      <td>[jakarta, south, jakarta, metro, police, recei...</td>\n",
+       "      <td>[actress_accord, available_day_concert, click_...</td>\n",
+       "      <td>24</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>6</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>Severe winds have downed billboards and trees ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Roadway Closure / Disruption, Flooding, Severe...</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>19/4/17 9:10</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-6.91264</td>\n",
+       "      <td>107.65700</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>16.0</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiSWh0d...</td>\n",
+       "      <td>Indonesia hit by some of strongest winds recorded</td>\n",
+       "      <td>A man stands near damaged houses following a t...</td>\n",
+       "      <td>[man, stand, near, damage, house, follow, torn...</td>\n",
+       "      <td>[bbc_indonesia, climatologist_government_resea...</td>\n",
+       "      <td>28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>14</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>Government sources are reporting a tornado has...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Tornado</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>17/9/18 19:55</td>\n",
+       "      <td>2018.0</td>\n",
+       "      <td>37.51000</td>\n",
+       "      <td>-77.61000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>9.0</td>\n",
+       "      <td>38.0</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMigAFod...</td>\n",
+       "      <td>UPDATE: Number of homes without power down to ...</td>\n",
+       "      <td>More than 90,000 homes and businesses across t...</td>\n",
+       "      <td>[more, than, home, business, across, richmond,...</td>\n",
+       "      <td>[advise_seek_alternate, affect_richmond, alter...</td>\n",
+       "      <td>134</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 23 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   id                                           Headline  \\\n",
+       "0   1  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1   3  Shanghai port congestion impacts terminals in ...   \n",
+       "2   5  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "3   6  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "4  14  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "\n",
+       "                                             Details  Severity  \\\n",
+       "0  Media sources indicate that workers at the Gra...  Moderate   \n",
+       "1  The persisting port congestion at Shanghai’s Y...     Minor   \n",
+       "2  According to local police in Jakarta, two expl...   Extreme   \n",
+       "3  Severe winds have downed billboards and trees ...  Moderate   \n",
+       "4  Government sources are reporting a tornado has...     Minor   \n",
+       "\n",
+       "                                            Category         Region  \\\n",
+       "0                                Mine Workers Strike      Indonesia   \n",
+       "1                                    Port Congestion          China   \n",
+       "2                         Bombing, Police Operations      Indonesia   \n",
+       "3  Roadway Closure / Disruption, Flooding, Severe...      Indonesia   \n",
+       "4                                            Tornado  United States   \n",
+       "\n",
+       "        Datetime    Year       lat        lon  ...  if_labeled Month  Week  \\\n",
+       "0  28/5/17 17:08  2017.0  -4.05608  137.11302  ...       False   5.0  21.0   \n",
+       "1   27/4/17 9:16  2017.0  29.52000  121.33190  ...       False   4.0  17.0   \n",
+       "2  24/5/17 16:20  2017.0       NaN        NaN  ...        True   5.0  21.0   \n",
+       "3   19/4/17 9:10  2017.0  -6.91264  107.65700  ...        True   4.0  16.0   \n",
+       "4  17/9/18 19:55  2018.0  37.51000  -77.61000  ...        True   9.0  38.0   \n",
+       "\n",
+       "                                    Headline_Details  \\\n",
+       "0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1  Shanghai port congestion impacts terminals in ...   \n",
+       "2  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "3  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "4  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "\n",
+       "                                                 url  \\\n",
+       "0  https://news.google.com/rss/articles/CBMiZ2h0d...   \n",
+       "1  https://news.google.com/rss/articles/CBMiVWh0d...   \n",
+       "2  https://news.google.com/rss/articles/CBMiZWh0d...   \n",
+       "3  https://news.google.com/rss/articles/CBMiSWh0d...   \n",
+       "4  https://news.google.com/rss/articles/CBMigAFod...   \n",
+       "\n",
+       "                                               title  \\\n",
+       "0  Freeport Indonesia mine workers extend strike ...   \n",
+       "1  Typhoon Muifa to shut China ports for second t...   \n",
+       "2  Jakarta Police Receive 2 More Reports on Coldp...   \n",
+       "3  Indonesia hit by some of strongest winds recorded   \n",
+       "4  UPDATE: Number of homes without power down to ...   \n",
+       "\n",
+       "                                             content  \\\n",
+       "0  Trucks are seen on a road in the Grasberg copp...   \n",
+       "1  By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...   \n",
+       "2  TEMPO.CO, Jakarta - South Jakarta Metro Police...   \n",
+       "3  A man stands near damaged houses following a t...   \n",
+       "4  More than 90,000 homes and businesses across t...   \n",
+       "\n",
+       "                                     cleaned_content  \\\n",
+       "0  [truck, be, see, on, road, in, grasberg, coppe...   \n",
+       "1  [by, sam, whelan, typhoon, have, prompt, port,...   \n",
+       "2  [jakarta, south, jakarta, metro, police, recei...   \n",
+       "3  [man, stand, near, damage, house, follow, torn...   \n",
+       "4  [more, than, home, business, across, richmond,...   \n",
+       "\n",
+       "                                      binary_content word_count  \n",
+       "0  [adkerson_jakarta_try, agreement_freeport_indo...         53  \n",
+       "1  [additional_ripple_effect, avoid_path_typhoon,...         44  \n",
+       "2  [actress_accord, available_day_concert, click_...         24  \n",
+       "3  [bbc_indonesia, climatologist_government_resea...         28  \n",
+       "4  [advise_seek_alternate, affect_richmond, alter...        134  \n",
+       "\n",
+       "[5 rows x 23 columns]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "09113e88-66cc-414c-a953-da04db83c4ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(3681, 23)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "037e74fc-bbcd-43e3-8346-799920cca8d8",
+   "metadata": {},
+   "source": [
+    "# Vectorisation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d67cef3a-59fb-4dd8-adc8-2cf288b90728",
+   "metadata": {},
+   "source": [
+    "NLP vectorization refers to the process of converting text data into numerical vectors that machine learning algorithms can understand and process. \n",
+    "\n",
+    "Bag-of-Words (BoW) is used here that represents text as a collection of unique words along with their frequencies. Each word is assigned an index, and the vector contains the count of each word present in the document."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "c95b7b8a-9767-469d-812d-c9a9d9fee0e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_copy = df.copy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "dfb2001e-04c1-49dc-b423-a64ea47af5a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# choose only the extreme and severe cases for modelling\n",
+    "cleaned = df_copy[df_copy['Severity'].isin(['Minor'])]\n",
+    "cleaned.reset_index(drop=True, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "3da09b6a-65c6-4f40-9a21-e0b798318ca5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1620, 23)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cleaned.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "de71c523-a59e-44b2-aa96-5f17d872c9c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "headline = cleaned.cleaned_content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "5b1e34e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array(['man', 'be', 'seriously', 'injure', 'after', 'boat', 'catch',\n",
+       "       'fire', 'in', 'sydney', 'marina', 'today', 'emergency', 'service',\n",
+       "       'be', 'call', 'birkenhead', 'point', 'marina', 'in', 'drummoyne',\n",
+       "       'shortly', 'after', 'follow', 'report', 'of', 'explosion', 'on',\n",
+       "       'boat', 'careflight', 's', 'rapid', 'response', 'helicopter', 'be',\n",
+       "       'task', 'at', 'crew', 'include', 'doctor', 'intensive', 'care',\n",
+       "       'paramedic', 'fly', 'drummoyne', 'land', 'in', 'nearby', 'brett',\n",
+       "       'park', 'just', 'minute', 'late', 'careflight', 'boat', 'catch',\n",
+       "       'fire', 'near', 'birkenhead', 'point', 'shopping', 'outlet',\n",
+       "       'simon', 'r', 'supply', 'reader', 'image', 'of', 'boat', 'on',\n",
+       "       'fire', 'at', 'marina', 'in', 'drummoyne', 'simon', 'r', 'supply',\n",
+       "       'three', 'nsw', 'ambulance', 'crew', 'careflight', 'chopper',\n",
+       "       'attend', 'scene', 'find', 'man', 'suffer', 'serious', 'burn',\n",
+       "       'legs', 'arm', 'shoulder', 'man', 'suffer', 'burn', 'percent',\n",
+       "       'of', 'body', 'boat', 'reportedly', 'explode', 'into', 'flame',\n",
+       "       'shortly', 'before', 'morning', 'paramedic', 'treat', 'man', 'at',\n",
+       "       'scene', 'with', 'burn', 'percent', 'of', 'body', 'firefighting',\n",
+       "       'boat', 'be', 'deploy', 'help', 'extinguish', 'blaze', 'man', 'be',\n",
+       "       'treat', 'on', 'scene', 'be', 'surround', 'by', 'family', 'be',\n",
+       "       'not', 'onboard', 'boat', 'at', 'time'], dtype=object)"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "headline[5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "677055b4-978e-4253-90f4-3f903662e225",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# vectorise the words\n",
+    "doc_dict = gensim.corpora.Dictionary(headline)\n",
+    "docs_vecs = [doc_dict.doc2bow(doc) for doc in headline]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "a54d1768-b069-4936-a156-deaf0b506d93",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of unique tokens: 35192\n",
+      "Number of articles: 1620\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('Number of unique tokens: %d' % len(doc_dict)) \n",
+    "print('Number of articles: %d' % len(docs_vecs)) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "9147fa86-1503-4252-bd9b-92fea1e6a926",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('be', 58147),\n",
+      " ('of', 53324),\n",
+      " ('in', 42710),\n",
+      " ('for', 19846),\n",
+      " ('on', 18899),\n",
+      " ('have', 18769),\n",
+      " ('with', 13029),\n",
+      " ('as', 12079),\n",
+      " ('port', 11548),\n",
+      " ('from', 11033),\n",
+      " ('at', 10589),\n",
+      " ('by', 10527),\n",
+      " ('s', 8527),\n",
+      " ('will', 8465),\n",
+      " ('that', 7825),\n",
+      " ('say', 7752),\n",
+      " ('not', 5664),\n",
+      " ('china', 5654),\n",
+      " ('day', 5304),\n",
+      " ('more', 5266),\n",
+      " ('new', 5198),\n",
+      " ('service', 5001),\n",
+      " ('also', 4707),\n",
+      " ('ship', 4693),\n",
+      " ('time', 4651),\n",
+      " ('than', 3673),\n",
+      " ('year', 3669),\n",
+      " ('trade', 3518),\n",
+      " ('can', 3504),\n",
+      " ('state', 3378),\n",
+      " ('between', 3226),\n",
+      " ('include', 3220),\n",
+      " ('over', 3173),\n",
+      " ('supply', 3094),\n",
+      " ('do', 3083),\n",
+      " ('vessel', 3079),\n",
+      " ('report', 2988),\n",
+      " ('country', 2979),\n",
+      " ('after', 2962),\n",
+      " ('area', 2913),\n",
+      " ('cargo', 2908),\n",
+      " ('one', 2905),\n",
+      " ('people', 2868),\n",
+      " ('other', 2866),\n",
+      " ('increase', 2852),\n",
+      " ('continue', 2787),\n",
+      " ('market', 2766),\n",
+      " ('container', 2765),\n",
+      " ('strike', 2713),\n",
+      " ('chain', 2708),\n",
+      " ('expect', 2656),\n",
+      " ('work', 2624),\n",
+      " ('while', 2590),\n",
+      " ('high', 2579),\n",
+      " ('about', 2514),\n",
+      " ('http', 2489),\n",
+      " ('into', 2470),\n",
+      " ('would', 2461),\n",
+      " ('remain', 2448),\n",
+      " ('pm', 2448),\n",
+      " ('government', 2437),\n",
+      " ('take', 2425),\n",
+      " ('two', 2349),\n",
+      " ('terminal', 2348),\n",
+      " ('may', 2332),\n",
+      " ('due', 2325),\n",
+      " ('use', 2313),\n",
+      " ('update', 2310),\n",
+      " ('city', 2295),\n",
+      " ('train', 2290),\n",
+      " ('make', 2278),\n",
+      " ('info', 2253),\n",
+      " ('demand', 2240),\n",
+      " ('world', 2219),\n",
+      " ('asia', 2194),\n",
+      " ('south', 2160),\n",
+      " ('company', 2148),\n",
+      " ('march', 2125),\n",
+      " ('see', 2101),\n",
+      " ('large', 2097),\n",
+      " ('customer', 2081),\n",
+      " ('through', 2042),\n",
+      " ('most', 2041),\n",
+      " ('plan', 2038),\n",
+      " ('global', 2031),\n",
+      " ('first', 2011),\n",
+      " ('line', 1962),\n",
+      " ('north', 1961),\n",
+      " ('could', 1948),\n",
+      " ('operation', 1931),\n",
+      " ('good', 1928),\n",
+      " ('if', 1906),\n",
+      " ('across', 1893),\n",
+      " ('coast', 1887),\n",
+      " ('business', 1863),\n",
+      " ('when', 1825),\n",
+      " ('week', 1813),\n",
+      " ('during', 1805),\n",
+      " ('last', 1771),\n",
+      " ('million', 1770)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Calculate word frequencies\n",
+    "word_frequencies = {doc_dict[word_id]: freq for word_id, freq in doc_dict.cfs.items()}\n",
+    "sorted_words = sorted(word_frequencies.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "pprint(sorted_words[:100])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5ed78239-2ce1-4784-a8f4-4c7438c8627b",
+   "metadata": {},
+   "source": [
+    "# LDA Modelling"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9db83273-461d-4f70-b23f-ec967579d94f",
+   "metadata": {},
+   "source": [
+    "## Benchmark Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "e6d577bd-9936-4d45-be90-345af2eb4827",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Build LDA benchmark model\n",
+    "lda_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n",
+    "                                       id2word=doc_dict,\n",
+    "                                       num_topics=4, \n",
+    "                                       random_state=42,\n",
+    "                                       chunksize=100,\n",
+    "                                       passes=10,\n",
+    "                                       per_word_topics=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "c4f1521f-5f43-40d2-a3a3-a8ac2ca6fec2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[(0,\n",
+      "  '0.042*\"of\" + 0.030*\"in\" + 0.026*\"be\" + 0.012*\"china\" + 0.012*\"on\" + '\n",
+      "  '0.011*\"s\" + 0.011*\"for\" + 0.011*\"have\" + 0.010*\"that\" + 0.009*\"as\"'),\n",
+      " (1,\n",
+      "  '0.028*\"be\" + 0.028*\"of\" + 0.018*\"in\" + 0.011*\"de\" + 0.009*\"y\" + 0.009*\"for\" '\n",
+      "  '+ 0.008*\"by\" + 0.007*\"http\" + 0.007*\"on\" + 0.007*\"have\"'),\n",
+      " (2,\n",
+      "  '0.041*\"be\" + 0.033*\"of\" + 0.028*\"in\" + 0.015*\"on\" + 0.013*\"have\" + '\n",
+      "  '0.013*\"for\" + 0.009*\"at\" + 0.009*\"say\" + 0.009*\"as\" + 0.008*\"with\"'),\n",
+      " (3,\n",
+      "  '0.039*\"be\" + 0.029*\"of\" + 0.026*\"in\" + 0.018*\"port\" + 0.013*\"for\" + '\n",
+      "  '0.011*\"have\" + 0.009*\"on\" + 0.009*\"with\" + 0.009*\"day\" + 0.008*\"from\"')]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Print the Keyword in the 10 topics\n",
+    "pprint(lda_model.print_topics())\n",
+    "doc_lda = lda_model[docs_vecs]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "fd57b1f4-a6cd-41e8-964f-d8a1d30aa3c9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Coherence Score LDAModel:  0.2757800922361482\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Compute Benchmark Coherence Score\n",
+    "coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n",
+    "coherence_lda = coherence_model_lda.get_coherence()\n",
+    "print('\\nCoherence Score LDAModel: ', coherence_lda)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "152e5a3a-7afe-4fb8-a02f-d7492ad80936",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Perplexity for LDAModel:  -7.407722316633226\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Compute Benchmark Perplexity\n",
+    "perplex= lda_model.log_perplexity(docs_vecs, total_docs=None) #For LDAModel\n",
+    "  # a measure of how good the model is. lower the better.\n",
+    "\n",
+    "print('\\nPerplexity for LDAModel: ', perplex)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "7dd3a60a-5c6f-4249-9868-30528a5b0ac8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=39505) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis\n",
+    "\n",
+    "# feed the LDA model into the pyLDAvis instance\n",
+    "pyLDAvis.enable_notebook()\n",
+    "visual= gensimvis.prepare(lda_model, docs_vecs, doc_dict)\n",
+    "\n",
+    "# Save the output to the html file\n",
+    "pyLDAvis.save_html(visual, \"data/topic_viz_benchmark_minor.html\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1895598f-3e5f-4acd-83a6-4491cc90f695",
+   "metadata": {},
+   "source": [
+    "# Hyper-Perameter Tuning and Evaluation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "47136c89-ff7b-4ac9-840f-04122fe62160",
+   "metadata": {},
+   "source": [
+    "Run the cells below only for re-modelling with new datasets, the whole tuning and evaluation process may take hours to run."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "c79ca5c4-e078-43ce-a430-8c1ed93dcd64",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hyper-perameter tuning (alpha and beta)\n",
+    "def compute_coherence_values(corpus, dictionary, k, a, b):\n",
+    "    \n",
+    "    lda_model = gensim.models.LdaMulticore(corpus=corpus,\n",
+    "                                           id2word=dictionary,\n",
+    "                                           num_topics=k, \n",
+    "                                           random_state=42,\n",
+    "                                           chunksize=100,\n",
+    "                                           passes=10,\n",
+    "                                           alpha=a,\n",
+    "                                           eta=b)\n",
+    "    \n",
+    "    coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n",
+    "    coherence = coherence_model_lda.get_coherence()\n",
+    "    perplex = lda_model.log_perplexity(docs_vecs, total_docs=None) \n",
+    "    \n",
+    "    return coherence, perplex"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "1c3c8478-9336-40f2-bb30-a37db4243b67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# setup\n",
+    "import numpy as np\n",
+    "\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "model_list = []\n",
+    "coherence_values = []\n",
+    "perplexity_values = []\n",
+    "model_topics = []\n",
+    "alpha_result = []\n",
+    "beta_result = []\n",
+    "\n",
+    "# topic ranges\n",
+    "num_topics = range(4, 13)\n",
+    "\n",
+    "# Alpha parameter\n",
+    "alpha = list(np.arange(0.31, 1, 0.3))\n",
+    "alpha.append('symmetric')\n",
+    "alpha.append('asymmetric')\n",
+    "\n",
+    "# Beta parameter\n",
+    "beta = list(np.arange(0.31, 1, 0.3))\n",
+    "beta.append('symmetric')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7e6bc53-0b57-4858-879a-644eca54ddbc",
+   "metadata": {},
+   "source": [
+    "Rational behind the alpha and eta: https://stats.stackexchange.com/questions/37405/natural-interpretation-for-lda-hyperparameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "02877b81-32df-4168-8e62-4cbca2be100b",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Topic range:  range(4, 13)\n",
+      "Alpha:  [0.31, 0.61, 0.9099999999999999, 'symmetric', 'asymmetric']\n",
+      "Beta:  [0.31, 0.61, 0.9099999999999999, 'symmetric']\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Topic range: \",num_topics)\n",
+    "print(\"Alpha: \",alpha)\n",
+    "print(\"Beta: \", beta)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "3c1f703c-4778-467f-a12e-0c18eeb274c5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2024-06-30 11:52:46.026823\n",
+      "#Topics: 4, CV Score: 0.26143167370104753, PV Score: -7.394241913115894, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.29008917581081944, PV Score: -7.35821379289522, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.3075550457683199, PV Score: -7.348746753666222, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.27068500671158163, PV Score: -7.332448354134598, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.2949752069963174, PV Score: -7.355618761173559, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.2883770109840239, PV Score: -7.345756037675643, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.30169838729877146, PV Score: -7.334658077515814, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.28399502160009293, PV Score: -7.337863652882048, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.3098908373731854, PV Score: -7.317278302545894, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.26324058496974073, PV Score: -7.421960949456933, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.28653242913009425, PV Score: -7.406328200019341, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.29614945599419024, PV Score: -7.405829958529572, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.26469167967575336, PV Score: -7.409695513631366, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.29743885397540426, PV Score: -7.416190442593165, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.29464381439032383, PV Score: -7.402408965099985, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.34823725193043276, PV Score: -7.398638060374257, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.31843503124612416, PV Score: -7.4064374372258275, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.31444697036706293, PV Score: -7.405945850874236, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.2712735526187385, PV Score: -7.4600931582453756, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.30311968074865997, PV Score: -7.460908353343469, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.29829568606376594, PV Score: -7.453422852489877, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.29621387204862615, PV Score: -7.46166337203672, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.32127327516432214, PV Score: -7.467013507843407, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.31985991254557, PV Score: -7.431396910423797, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.3224915541388905, PV Score: -7.448438398378084, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.3381018616555288, PV Score: -7.457876704009056, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.30836100224079815, PV Score: -7.45460656674408, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.2643367431201753, PV Score: -7.390794744679982, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.29008917581081944, PV Score: -7.341357499253523, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.27735837350398757, PV Score: -7.323864546372206, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.27207166533136756, PV Score: -7.298831876196072, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.29681559070931296, PV Score: -7.286549304258785, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.29778017634331727, PV Score: -7.271904110257489, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.3012188333925358, PV Score: -7.263682235965553, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.30281626874462, PV Score: -7.267991091418852, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.31307471307490264, PV Score: -7.263340871172593, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.2631472484692373, PV Score: -7.391225160526899, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.28117696184010554, PV Score: -7.358283237848617, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.2946692074353743, PV Score: -7.350030009796271, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.2695848086251494, PV Score: -7.332704133455395, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.2873386785526998, PV Score: -7.365743826339281, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.28209952470043215, PV Score: -7.352132999419965, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.29783175477393864, PV Score: -7.343942952645343, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.2786007151015459, PV Score: -7.345863425524115, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.31534621656869705, PV Score: -7.331805220457104, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.2505923687565112, PV Score: -7.416873289619847, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.28706700596105156, PV Score: -7.403540595451488, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.29473065024977163, PV Score: -7.4106494490711015, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.26820275442412866, PV Score: -7.412576639292029, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.29025456691114637, PV Score: -7.41531726726418, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.29873861091584003, PV Score: -7.410097381301386, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.33489681733700966, PV Score: -7.414464176932596, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.3136515342313888, PV Score: -7.414824058449246, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.29901180359094387, PV Score: -7.425251302616095, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.25606628747612015, PV Score: -7.461432644283255, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.2981919886915416, PV Score: -7.461173440540555, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.3155490657279846, PV Score: -7.456975825811057, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.3083085761113492, PV Score: -7.463721193947859, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.2987637794108595, PV Score: -7.463974937352514, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.30637199754319, PV Score: -7.447224084336919, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.34064164600584956, PV Score: -7.4608295203390345, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.33431924954293984, PV Score: -7.468074450247606, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.3021823876546624, PV Score: -7.474199589973487, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.2592664104701984, PV Score: -7.380958290626252, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.29008917581081944, PV Score: -7.34406277883799, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.2856139513419668, PV Score: -7.332074766991939, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.2721558660968534, PV Score: -7.297199516455353, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.28791066408428134, PV Score: -7.314898967963879, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.2972722790111482, PV Score: -7.283806173931616, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.3082121035138333, PV Score: -7.29151183400522, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.30375150084375463, PV Score: -7.280134176808763, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.3026143223444013, PV Score: -7.277233833271221, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.2587849246611012, PV Score: -7.391745121978334, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.29232718870337143, PV Score: -7.359033621013408, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.298114743775785, PV Score: -7.351153460394994, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.26903864999105614, PV Score: -7.334371538653122, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.28622468428206693, PV Score: -7.372271837301099, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.29758804437495606, PV Score: -7.358497856224319, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.29746253535912703, PV Score: -7.343247363065808, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.2842073825386278, PV Score: -7.3531033741821386, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.32301209366405703, PV Score: -7.344357219935557, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.2620085857568131, PV Score: -7.431829725624846, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.3148122601459173, PV Score: -7.4118133900365955, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.29369633574952747, PV Score: -7.411592442393478, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.2702663182798373, PV Score: -7.408869634701454, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.29626015190055904, PV Score: -7.4218524805240165, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.3029984127461481, PV Score: -7.413846845692629, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.3328224542610766, PV Score: -7.42022182252732, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.30475127610761077, PV Score: -7.422669034180523, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.30231311065405014, PV Score: -7.434846448528838, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.258672617830039, PV Score: -7.469954125650798, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.3141300753700526, PV Score: -7.465439773550294, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.33646277127762847, PV Score: -7.459430389322742, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.2989500059387701, PV Score: -7.464860494675141, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.3067664143272131, PV Score: -7.460823334723379, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.32230471519058956, PV Score: -7.462203609695669, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.355651395118116, PV Score: -7.466906313924779, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.3074563046801128, PV Score: -7.4762246058984125, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.292184655691141, PV Score: -7.48462689762355, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.27425502082773556, PV Score: -7.381959277922089, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.2792475123330335, PV Score: -7.340967173529193, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.2748288743412546, PV Score: -7.338412973991607, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.27272711338157735, PV Score: -7.304992530741422, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.27875956141406333, PV Score: -7.310583890409706, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.2958113176625343, PV Score: -7.290099223904081, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.30039391830183415, PV Score: -7.29361844332684, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.30238837300731736, PV Score: -7.293498894544746, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.30057296940845457, PV Score: -7.286728893856696, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.28045576981660336, PV Score: -7.3810882171558, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.27516707115009786, PV Score: -7.348322538492822, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.30174281503830513, PV Score: -7.357661750753335, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.26452009940043947, PV Score: -7.332995538212589, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.27957941981373957, PV Score: -7.34520072112036, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.2921761447063516, PV Score: -7.3174465018975985, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.2819456892021096, PV Score: -7.3271012408060425, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.2858145693331318, PV Score: -7.324243303702109, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.3162268304900056, PV Score: -7.301149412893058, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.26200858575681313, PV Score: -7.435025163734385, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.28653242913009425, PV Score: -7.402808127824116, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.30034116357398466, PV Score: -7.403041768564758, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.27713631118988463, PV Score: -7.401296493610201, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.305005574283436, PV Score: -7.403063894303076, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.319284842602941, PV Score: -7.381097776990529, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.3187835569771548, PV Score: -7.392442067244151, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.31619116554114374, PV Score: -7.398110003036577, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.30620936485014044, PV Score: -7.389060738256993, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.25861742292536233, PV Score: -7.463031931556786, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.30612131816590366, PV Score: -7.459187288641012, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.29856036681931464, PV Score: -7.451419283754892, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.2963932433750025, PV Score: -7.451642157595998, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.3175863739616889, PV Score: -7.460112275446344, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.30091910989824133, PV Score: -7.423042776887486, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.3221986755554972, PV Score: -7.4413791780149126, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.3400624797510634, PV Score: -7.4475561490332405, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.29577010557935146, PV Score: -7.438896552351397, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.2875150010691004, PV Score: -7.398657443181138, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.2900891758108194, PV Score: -7.337054653335996, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.2872501529046281, PV Score: -7.319444085859869, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.27501057403010526, PV Score: -7.290723834382987, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.2923043477830799, PV Score: -7.2829244272999825, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.2978356724583504, PV Score: -7.263979371448642, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.3098726690096235, PV Score: -7.249251508569862, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.31266100984189077, PV Score: -7.252673302755199, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.31275264383294515, PV Score: -7.254777083051176, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.2602975588122612, PV Score: -7.381821385441839, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.2792475123330335, PV Score: -7.350616172255018, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.3015492641821192, PV Score: -7.349068241683937, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.2668593619258581, PV Score: -7.334972757155578, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.2916111243636439, PV Score: -7.3504381054918735, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.3104237788212182, PV Score: -7.3330296284925875, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.3282155213609229, PV Score: -7.336602404389545, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.2878177794271231, PV Score: -7.319097172570247, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.309148501118687, PV Score: -7.299840665764544, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.2643914598991108, PV Score: -7.439997400522905, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.30680350294176834, PV Score: -7.406369360079358, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.30273133375875494, PV Score: -7.409977374651039, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.2895606942065561, PV Score: -7.404088495718267, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.30220942123026345, PV Score: -7.41654700444938, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.30085511300217477, PV Score: -7.3893922736333995, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.32576991286555973, PV Score: -7.400148584021936, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.30570350016100467, PV Score: -7.3930680136359115, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.27592390657639637, PV Score: -7.390687284583513, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.26415862588678707, PV Score: -7.4660902988837075, Alpha: asymmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.3031196807486601, PV Score: -7.459922366408937, Alpha: asymmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.28267000810499315, PV Score: -7.446406611219757, Alpha: asymmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.30409252212111715, PV Score: -7.458109944307844, Alpha: asymmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.3186545711929555, PV Score: -7.460394137450762, Alpha: asymmetric, Beta: 0.9099999999999999\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "File \u001b[0;32m<timed exec>:10\u001b[0m\n",
+      "Cell \u001b[0;32mIn[19], line 4\u001b[0m, in \u001b[0;36mcompute_coherence_values\u001b[0;34m(corpus, dictionary, k, a, b)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcompute_coherence_values\u001b[39m(corpus, dictionary, k, a, b):\n\u001b[0;32m----> 4\u001b[0m     lda_model \u001b[38;5;241m=\u001b[39m \u001b[43mgensim\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLdaMulticore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcorpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mid2word\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdictionary\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mnum_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m42\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mpasses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43meta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     13\u001b[0m     coherence_model_lda \u001b[38;5;241m=\u001b[39m CoherenceModel(model\u001b[38;5;241m=\u001b[39mlda_model, texts\u001b[38;5;241m=\u001b[39mheadline, dictionary\u001b[38;5;241m=\u001b[39mdoc_dict, coherence\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mc_v\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     14\u001b[0m     coherence \u001b[38;5;241m=\u001b[39m coherence_model_lda\u001b[38;5;241m.\u001b[39mget_coherence()\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamulticore.py:186\u001b[0m, in \u001b[0;36mLdaMulticore.__init__\u001b[0;34m(self, corpus, num_topics, id2word, workers, chunksize, passes, batch, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, random_state, minimum_probability, minimum_phi_value, per_word_topics, dtype)\u001b[0m\n\u001b[1;32m    183\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(alpha, \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m alpha \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mauto\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m    184\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto-tuning alpha not implemented in LdaMulticore; use plain LdaModel.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 186\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mLdaMulticore\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m    187\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcorpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_topics\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    188\u001b[0m \u001b[43m    \u001b[49m\u001b[43mid2word\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mid2word\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunksize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpasses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpasses\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43malpha\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    189\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdecay\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecay\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meval_every\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meval_every\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miterations\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miterations\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    190\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgamma_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgamma_threshold\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrandom_state\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mminimum_probability\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mminimum_probability\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    191\u001b[0m \u001b[43m    \u001b[49m\u001b[43mminimum_phi_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mminimum_phi_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mper_word_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mper_word_topics\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    192\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamodel.py:521\u001b[0m, in \u001b[0;36mLdaModel.__init__\u001b[0;34m(self, corpus, num_topics, id2word, distributed, chunksize, passes, update_every, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, minimum_probability, random_state, ns_conf, minimum_phi_value, per_word_topics, callbacks, dtype)\u001b[0m\n\u001b[1;32m    519\u001b[0m use_numpy \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdispatcher \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    520\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[0;32m--> 521\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupdate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchunks_as_numpy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_numpy\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    522\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39madd_lifecycle_event(\n\u001b[1;32m    523\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcreated\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    524\u001b[0m     msg\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrained \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtime\u001b[38;5;241m.\u001b[39mtime()\u001b[38;5;250m \u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;250m \u001b[39mstart\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124ms\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    525\u001b[0m )\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamulticore.py:316\u001b[0m, in \u001b[0;36mLdaMulticore.update\u001b[0;34m(self, corpus, chunks_as_numpy)\u001b[0m\n\u001b[1;32m    312\u001b[0m \u001b[38;5;66;03m# endfor single corpus pass\u001b[39;00m\n\u001b[1;32m    313\u001b[0m \n\u001b[1;32m    314\u001b[0m \u001b[38;5;66;03m# wait for all outstanding jobs to finish\u001b[39;00m\n\u001b[1;32m    315\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m queue_size[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m--> 316\u001b[0m     \u001b[43mprocess_result_queue\u001b[49m\u001b[43m(\u001b[49m\u001b[43mforce\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m    318\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m reallen \u001b[38;5;241m!=\u001b[39m lencorpus:\n\u001b[1;32m    319\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minput corpus size changed during training (don\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt use generators as input)\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamulticore.py:274\u001b[0m, in \u001b[0;36mLdaMulticore.update.<locals>.process_result_queue\u001b[0;34m(force)\u001b[0m\n\u001b[1;32m    268\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    269\u001b[0m \u001b[38;5;124;03mClear the result queue, merging all intermediate results, and update the\u001b[39;00m\n\u001b[1;32m    270\u001b[0m \u001b[38;5;124;03mLDA model if necessary.\u001b[39;00m\n\u001b[1;32m    271\u001b[0m \n\u001b[1;32m    272\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    273\u001b[0m merged_new \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 274\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[43mresult_queue\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mempty\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m    275\u001b[0m     other\u001b[38;5;241m.\u001b[39mmerge(result_queue\u001b[38;5;241m.\u001b[39mget())\n\u001b[1;32m    276\u001b[0m     queue_size[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/queues.py:129\u001b[0m, in \u001b[0;36mQueue.empty\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mempty\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 129\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_poll\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/connection.py:257\u001b[0m, in \u001b[0;36m_ConnectionBase.poll\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    255\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_closed()\n\u001b[1;32m    256\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_readable()\n\u001b[0;32m--> 257\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_poll\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/connection.py:440\u001b[0m, in \u001b[0;36mConnection._poll\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    439\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_poll\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout):\n\u001b[0;32m--> 440\u001b[0m     r \u001b[38;5;241m=\u001b[39m \u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    441\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mbool\u001b[39m(r)\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/connection.py:1130\u001b[0m, in \u001b[0;36mwait\u001b[0;34m(object_list, timeout)\u001b[0m\n\u001b[1;32m   1128\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _WaitSelector() \u001b[38;5;28;01mas\u001b[39;00m selector:\n\u001b[1;32m   1129\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m obj \u001b[38;5;129;01min\u001b[39;00m object_list:\n\u001b[0;32m-> 1130\u001b[0m         \u001b[43mselector\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mregister\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mselectors\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mEVENT_READ\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1132\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1133\u001b[0m         deadline \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mmonotonic() \u001b[38;5;241m+\u001b[39m timeout\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/selectors.py:352\u001b[0m, in \u001b[0;36m_PollLikeSelector.register\u001b[0;34m(self, fileobj, events, data)\u001b[0m\n\u001b[1;32m    351\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mregister\u001b[39m(\u001b[38;5;28mself\u001b[39m, fileobj, events, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m--> 352\u001b[0m     key \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mregister\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfileobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevents\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    353\u001b[0m     poller_events \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    354\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m events \u001b[38;5;241m&\u001b[39m EVENT_READ:\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/selectors.py:238\u001b[0m, in \u001b[0;36m_BaseSelectorImpl.register\u001b[0;34m(self, fileobj, events, data)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;129;01mnot\u001b[39;00m events) \u001b[38;5;129;01mor\u001b[39;00m (events \u001b[38;5;241m&\u001b[39m \u001b[38;5;241m~\u001b[39m(EVENT_READ \u001b[38;5;241m|\u001b[39m EVENT_WRITE)):\n\u001b[1;32m    236\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid events: \u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(events))\n\u001b[0;32m--> 238\u001b[0m key \u001b[38;5;241m=\u001b[39m \u001b[43mSelectorKey\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfileobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fileobj_lookup\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfileobj\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mevents\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    240\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m key\u001b[38;5;241m.\u001b[39mfd \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fd_to_key:\n\u001b[1;32m    241\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{!r}\u001b[39;00m\u001b[38;5;124m (FD \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m) is already registered\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    242\u001b[0m                    \u001b[38;5;241m.\u001b[39mformat(fileobj, key\u001b[38;5;241m.\u001b[39mfd))\n",
+      "File \u001b[0;32m<string>:1\u001b[0m, in \u001b[0;36m<lambda>\u001b[0;34m(_cls, fileobj, fd, events, data)\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "import datetime\n",
+    "import numpy as np\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "print(datetime.datetime.now())\n",
+    "\n",
+    "for a in alpha:\n",
+    "    for b in beta:\n",
+    "        for num in num_topics:\n",
+    "            cv, pv = compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=num, a=a, b=b) \n",
+    "\n",
+    "            model_topics.append(num)    \n",
+    "            coherence_values.append(cv)  \n",
+    "            perplexity_values.append(pv)\n",
+    "            alpha_result.append(a)\n",
+    "            beta_result.append(b)\n",
+    "            print(\"#Topics: \" + str(num) + \", CV Score: \" + str(coherence_values[-1]) + \", PV Score: \" + str(perplexity_values[-1]) + \", Alpha: \" + str(alpha_result[-1]) + \", Beta: \" + str(beta_result[-1]))\n",
+    "    \n",
+    "print(datetime.datetime.now())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "364ff6d5-e3da-4dde-a2c8-5375fc5d711f",
+   "metadata": {},
+   "source": [
+    "The table below reveals the top 20 fine tuned models with best combinations of coherence score and perplexity score. It was sorted by the coherence score in descending order as a higher coherence score indicates a better model, and sorted the perplexity score in ascending order as a lower perplexity score indicates a better model. While coherence score evaluates the quality of the topics, the perplexity score evaluates the overall performance of the model in predicting new documents. Usually, the coherence score is a better metric to use if the goal is to obtain topics that are semantically coherent and interpretable. Perplexity score, on the other hand, is a better metric to use if the goal is to build a model that generalises well to new data, in other words, how confident the model is in predicting the new data (Sánchez-Aguayo, et al., 2022). Ultimately, we aim to get a balance between the perplexity value and coherence score when determining our final model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "78a60032-a4d7-44d4-841c-a1bd3740d5dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Find the top 20 combinations based on Coherence Score and Perplexity Score\n",
+    "result = pd.DataFrame(\n",
+    "    {'Topics': model_topics,\n",
+    "     'Coherence Score': coherence_values,\n",
+    "     'Perplexity Score': perplexity_values,\n",
+    "     'Alpha': alpha_result,\n",
+    "     'Beta': beta_result\n",
+    "    })\n",
+    "result.sort_values(by=['Coherence Score', 'Perplexity Score'], ascending=[False, True]).head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3461df57-c069-4ad2-80d7-8890dec9438e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result.to_csv('data/lda_fine_tuning_result_minor.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "800e5a4b-7302-42e8-97b0-5b598c1c80ae",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "# Show graph Topics vs Coherence Score\n",
+    "result.groupby('Alpha').plot(x='Topics', y='Coherence Score', legend = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26996b89-0e7a-4f2d-8cf7-c4a716569bc2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Show graph Topics vs Perplexity Score\n",
+    "\n",
+    "plt.plot(model_topics, coherence_values)\n",
+    "plt.xlabel(\"Num Topics\")\n",
+    "plt.ylabel(\"Coherence Score\")\n",
+    "plt.legend((\"Coherence Score\"), loc='best')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "91d2f4c1-de77-44b6-b41b-fcc9a07233e8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Show graph Topics vs Perplexity Score\n",
+    "\n",
+    "plt.plot(model_topics, perplexity_values)\n",
+    "plt.xlabel(\"Num Topics\")\n",
+    "plt.ylabel(\"Perplexity score\")\n",
+    "plt.legend((\"perplexity_values\"), loc='best')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cdc3ddd2-f743-4e5b-b6c6-2656e0b77aec",
+   "metadata": {},
+   "source": [
+    "## Final Model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e86c2bfe-264b-4530-9d81-10b1cdc5071c",
+   "metadata": {},
+   "source": [
+    "refer to the script topic_modelling_severe for detailed explanation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "490734ed-077c-4fb0-930c-0b42f4f63c94",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# realised that there may be some overlaps for 8 topics, thus 4-6 topics are optimal\n",
+    "k = 2\n",
+    "# a = 'asymmetric'\n",
+    "a = 0.31\n",
+    "# b = 0.31\n",
+    "b = 'symmetric'\n",
+    "\n",
+    "\n",
+    "final_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n",
+    "                                           id2word=doc_dict,\n",
+    "                                           num_topics=k, \n",
+    "                                           random_state=42,\n",
+    "                                           chunksize=100,\n",
+    "                                           passes=10,\n",
+    "                                           alpha=a,\n",
+    "                                           eta=b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "afe8abf0-2d12-414e-92be-a655865addb1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=k, a=a, b=b) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8430a827-6dbb-4737-8ccc-78ed17a01234",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Set up the environment to display the graphical outputs\n",
+    "# feed the LDA model into the pyLDAvis instance\n",
+    "pyLDAvis.enable_notebook()\n",
+    "visual= gensimvis.prepare(final_model, docs_vecs, doc_dict)\n",
+    "\n",
+    "# Save the output to the html file\n",
+    "pyLDAvis.save_html(visual, \"data/topic_viz2_minor_training.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e30d71a-a3c7-40c7-94c0-7eea1bedc887",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "final_model.print_topics(num_words=30)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b958c148-5dbe-4896-bcba-85e0f78b2941",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1939c00d-12e4-4670-9f18-f287f8b86bef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Map the topic ID with appropriate topic names, this part should be updated accordinly whenever the model is updated\n",
+    "topic_mapping = {0: \"finance\", 1: \"tech\", 2: \"education\", 3: \"sports\", 4: \"leisure\"}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a9a921e-3cdd-4fe5-a58f-7b50e5feeecd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get the topics and their top keywords into a dataframe\n",
+    "topics = final_model.show_topics(num_words=30) \n",
+    "\n",
+    "topic_keywords = pd.DataFrame()\n",
+    "for topic_id, topic in topics:\n",
+    "    topic_keywords.at[topic_id, 'Topic Keywords'] = topic\n",
+    "\n",
+    "topic_keywords['Topic ID'] = topic_keywords.index\n",
+    "topic_keywords['Topic Name'] = topic_mapping \n",
+    "topic_keywords"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "607d2cfd-b3ca-4f99-9e01-d320ca98a2a0",
+   "metadata": {},
+   "source": [
+    "# Save the final model "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "84eb2746-173a-4283-bca5-681f77548698",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Save a model to disk, or reload a pre-trained model\n",
+    "# naming convention: final_model_topic_alpha_eta\n",
+    "final_model.save(\"models/final_model_5_asym_91\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7b6e4d9-a577-4dfb-ba6e-fc74365880f4",
+   "metadata": {},
+   "source": [
+    "# Find dominant topic(s) for each news article"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0eeecbcb-358c-44f9-8463-75cdfac0ba90",
+   "metadata": {},
+   "source": [
+    "Attach the dominant topics back to the news dataset for classifying purpose."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8bebb269-dbb0-4c46-925c-38de0f2bcfd7",
+   "metadata": {},
+   "source": [
+    "Made use of gensim lda's own function: https://radimrehurek.com/gensim/models/ldamodel.html"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f585ff52-b60d-4d70-ae64-a7c23d2cc6c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "def format_topics_sentences(ldamodel, corpus, data):\n",
+    "    # Preallocate memory for the DataFrame\n",
+    "    num_docs = len(corpus)\n",
+    "    sent_topics = {'Dominant_Topic': [0] * num_docs, 'Perc_Contribution': [0.0] * num_docs, 'Topic_Distribution': [()] * num_docs}\n",
+    "    \n",
+    "    # Get main topic in each document\n",
+    "    for i, row in enumerate(ldamodel[corpus]):\n",
+    "        row = sorted(row, key=lambda x: (x[1]), reverse=True)\n",
+    "        if row:\n",
+    "            # Get the Dominant topic, Perc Contribution and Keywords for each document\n",
+    "            dominant_topic, perc_contribution = row[0]\n",
+    "            topic_distribution = row\n",
+    "            sent_topics['Dominant_Topic'][i] = int(dominant_topic)\n",
+    "            sent_topics['Perc_Contribution'][i] = round(perc_contribution, 4)\n",
+    "            sent_topics['Topic_Distribution'][i] = topic_distribution\n",
+    "\n",
+    "    # Create the DataFrame\n",
+    "    sent_topics_df = pd.DataFrame(sent_topics)\n",
+    "    sent_topics_df['Text'] = data\n",
+    "\n",
+    "    return sent_topics_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24d3ff60-035e-4133-9ffd-88cce5cdccb1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_topic_sents_keywords = format_topics_sentences(ldamodel=final_model, corpus=docs_vecs, data=cleaned.Headline_Details)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c88b088b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Format\n",
+    "df_dominant_topic = df_topic_sents_keywords.reset_index()\n",
+    "df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Topic_Distribution', 'Text']\n",
+    "\n",
+    "# Show\n",
+    "df_dominant_topic.head(10)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "560da382-aa86-4df1-8b85-56b057a27cd4",
+   "metadata": {},
+   "source": [
+    "# Result Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4fe6b40b-6922-4de3-8d9e-dac7474b6303",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_dominant_topic[\"Dominant_Topic\"].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b9917340-31cf-48af-871f-b481128fdf22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "# Get value counts of each topic\n",
+    "topic_counts = df_dominant_topic[\"Dominant_Topic\"].value_counts()\n",
+    "\n",
+    "# Create a bar plot\n",
+    "plt.figure(figsize=(8, 6))\n",
+    "topic_counts.plot(kind=\"bar\", color=\"skyblue\")\n",
+    "\n",
+    "# Add labels to the bars\n",
+    "for i, count in enumerate(topic_counts):\n",
+    "    plt.text(i, count, str(count), ha=\"center\", va=\"bottom\")\n",
+    "\n",
+    "# Add labels and title\n",
+    "plt.xlabel(\"Topics\")\n",
+    "plt.ylabel(\"Number of News\")\n",
+    "plt.title(\"Topic Distribution\")\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.xticks(rotation=45)  # Rotate x-axis labels for better readability\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fffa1e57-f975-4469-a42b-19d76c60fb66",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_dominant_topic.sort_values(by='Topic_Perc_Contrib', ascending=True).head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8510f506-141f-4382-b668-251df1afc95f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Sample 100 rows, can change the random_state for different samples\n",
+    "sampled_data = df_dominant_topic.sample(n=100, random_state=42)  \n",
+    "sampled_df = pd.DataFrame(sampled_data).reset_index()\n",
+    "sampled_df.to_csv('data/sample_minor.csv')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/08_topic_modelling_moderate.ipynb

@@ -0,0 +1,1760 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "46322fb5-5918-4b70-9689-9e0781439ac4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n",
+      "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "daf1e3d1-75ac-4299-8bed-2f413a49f9a6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import sent_tokenize\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "\n",
+    "import gensim\n",
+    "from gensim import corpora\n",
+    "from gensim import similarities\n",
+    "from gensim import models\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "# from wordcloud import WordCloud, ImageColorGenerator\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import os\n",
+    "import datetime\n",
+    "\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "49e6de6b-71bd-4948-8827-52601406058f",
+   "metadata": {},
+   "source": [
+    "# Import Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "49222182-7811-4fa6-8c0a-21d3a546863e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_parquet('data/processed_data2.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "3fb59a30",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>lat</th>\n",
+       "      <th>lon</th>\n",
+       "      <th>...</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "      <th>Headline_Details</th>\n",
+       "      <th>url</th>\n",
+       "      <th>title</th>\n",
+       "      <th>content</th>\n",
+       "      <th>cleaned_content</th>\n",
+       "      <th>binary_content</th>\n",
+       "      <th>word_count</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-4.05608</td>\n",
+       "      <td>137.11302</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZ2h0d...</td>\n",
+       "      <td>Freeport Indonesia mine workers extend strike ...</td>\n",
+       "      <td>Trucks are seen on a road in the Grasberg copp...</td>\n",
+       "      <td>[truck, be, see, on, road, in, grasberg, coppe...</td>\n",
+       "      <td>[adkerson_jakarta_try, agreement_freeport_indo...</td>\n",
+       "      <td>53</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Congestion</td>\n",
+       "      <td>China</td>\n",
+       "      <td>27/4/17 9:16</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>29.52000</td>\n",
+       "      <td>121.33190</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiVWh0d...</td>\n",
+       "      <td>Typhoon Muifa to shut China ports for second t...</td>\n",
+       "      <td>By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...</td>\n",
+       "      <td>[by, sam, whelan, typhoon, have, prompt, port,...</td>\n",
+       "      <td>[additional_ripple_effect, avoid_path_typhoon,...</td>\n",
+       "      <td>44</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>5</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 16:20</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZWh0d...</td>\n",
+       "      <td>Jakarta Police Receive 2 More Reports on Coldp...</td>\n",
+       "      <td>TEMPO.CO, Jakarta - South Jakarta Metro Police...</td>\n",
+       "      <td>[jakarta, south, jakarta, metro, police, recei...</td>\n",
+       "      <td>[actress_accord, available_day_concert, click_...</td>\n",
+       "      <td>24</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>6</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>Severe winds have downed billboards and trees ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Roadway Closure / Disruption, Flooding, Severe...</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>19/4/17 9:10</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-6.91264</td>\n",
+       "      <td>107.65700</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>16.0</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiSWh0d...</td>\n",
+       "      <td>Indonesia hit by some of strongest winds recorded</td>\n",
+       "      <td>A man stands near damaged houses following a t...</td>\n",
+       "      <td>[man, stand, near, damage, house, follow, torn...</td>\n",
+       "      <td>[bbc_indonesia, climatologist_government_resea...</td>\n",
+       "      <td>28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>14</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>Government sources are reporting a tornado has...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Tornado</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>17/9/18 19:55</td>\n",
+       "      <td>2018.0</td>\n",
+       "      <td>37.51000</td>\n",
+       "      <td>-77.61000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>9.0</td>\n",
+       "      <td>38.0</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMigAFod...</td>\n",
+       "      <td>UPDATE: Number of homes without power down to ...</td>\n",
+       "      <td>More than 90,000 homes and businesses across t...</td>\n",
+       "      <td>[more, than, home, business, across, richmond,...</td>\n",
+       "      <td>[advise_seek_alternate, affect_richmond, alter...</td>\n",
+       "      <td>134</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 23 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   id                                           Headline  \\\n",
+       "0   1  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1   3  Shanghai port congestion impacts terminals in ...   \n",
+       "2   5  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "3   6  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "4  14  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "\n",
+       "                                             Details  Severity  \\\n",
+       "0  Media sources indicate that workers at the Gra...  Moderate   \n",
+       "1  The persisting port congestion at Shanghai’s Y...     Minor   \n",
+       "2  According to local police in Jakarta, two expl...   Extreme   \n",
+       "3  Severe winds have downed billboards and trees ...  Moderate   \n",
+       "4  Government sources are reporting a tornado has...     Minor   \n",
+       "\n",
+       "                                            Category         Region  \\\n",
+       "0                                Mine Workers Strike      Indonesia   \n",
+       "1                                    Port Congestion          China   \n",
+       "2                         Bombing, Police Operations      Indonesia   \n",
+       "3  Roadway Closure / Disruption, Flooding, Severe...      Indonesia   \n",
+       "4                                            Tornado  United States   \n",
+       "\n",
+       "        Datetime    Year       lat        lon  ...  if_labeled Month  Week  \\\n",
+       "0  28/5/17 17:08  2017.0  -4.05608  137.11302  ...       False   5.0  21.0   \n",
+       "1   27/4/17 9:16  2017.0  29.52000  121.33190  ...       False   4.0  17.0   \n",
+       "2  24/5/17 16:20  2017.0       NaN        NaN  ...        True   5.0  21.0   \n",
+       "3   19/4/17 9:10  2017.0  -6.91264  107.65700  ...        True   4.0  16.0   \n",
+       "4  17/9/18 19:55  2018.0  37.51000  -77.61000  ...        True   9.0  38.0   \n",
+       "\n",
+       "                                    Headline_Details  \\\n",
+       "0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1  Shanghai port congestion impacts terminals in ...   \n",
+       "2  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "3  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "4  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "\n",
+       "                                                 url  \\\n",
+       "0  https://news.google.com/rss/articles/CBMiZ2h0d...   \n",
+       "1  https://news.google.com/rss/articles/CBMiVWh0d...   \n",
+       "2  https://news.google.com/rss/articles/CBMiZWh0d...   \n",
+       "3  https://news.google.com/rss/articles/CBMiSWh0d...   \n",
+       "4  https://news.google.com/rss/articles/CBMigAFod...   \n",
+       "\n",
+       "                                               title  \\\n",
+       "0  Freeport Indonesia mine workers extend strike ...   \n",
+       "1  Typhoon Muifa to shut China ports for second t...   \n",
+       "2  Jakarta Police Receive 2 More Reports on Coldp...   \n",
+       "3  Indonesia hit by some of strongest winds recorded   \n",
+       "4  UPDATE: Number of homes without power down to ...   \n",
+       "\n",
+       "                                             content  \\\n",
+       "0  Trucks are seen on a road in the Grasberg copp...   \n",
+       "1  By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...   \n",
+       "2  TEMPO.CO, Jakarta - South Jakarta Metro Police...   \n",
+       "3  A man stands near damaged houses following a t...   \n",
+       "4  More than 90,000 homes and businesses across t...   \n",
+       "\n",
+       "                                     cleaned_content  \\\n",
+       "0  [truck, be, see, on, road, in, grasberg, coppe...   \n",
+       "1  [by, sam, whelan, typhoon, have, prompt, port,...   \n",
+       "2  [jakarta, south, jakarta, metro, police, recei...   \n",
+       "3  [man, stand, near, damage, house, follow, torn...   \n",
+       "4  [more, than, home, business, across, richmond,...   \n",
+       "\n",
+       "                                      binary_content word_count  \n",
+       "0  [adkerson_jakarta_try, agreement_freeport_indo...         53  \n",
+       "1  [additional_ripple_effect, avoid_path_typhoon,...         44  \n",
+       "2  [actress_accord, available_day_concert, click_...         24  \n",
+       "3  [bbc_indonesia, climatologist_government_resea...         28  \n",
+       "4  [advise_seek_alternate, affect_richmond, alter...        134  \n",
+       "\n",
+       "[5 rows x 23 columns]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "09113e88-66cc-414c-a953-da04db83c4ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(3681, 23)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "037e74fc-bbcd-43e3-8346-799920cca8d8",
+   "metadata": {},
+   "source": [
+    "# Vectorisation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d67cef3a-59fb-4dd8-adc8-2cf288b90728",
+   "metadata": {},
+   "source": [
+    "NLP vectorization refers to the process of converting text data into numerical vectors that machine learning algorithms can understand and process. \n",
+    "\n",
+    "Bag-of-Words (BoW) is used here that represents text as a collection of unique words along with their frequencies. Each word is assigned an index, and the vector contains the count of each word present in the document."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c95b7b8a-9767-469d-812d-c9a9d9fee0e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_copy = df.copy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "dfb2001e-04c1-49dc-b423-a64ea47af5a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# choose only the extreme and severe cases for modelling\n",
+    "cleaned = df_copy[df_copy['Severity'].isin(['Moderate'])]\n",
+    "cleaned.reset_index(drop=True, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "de71c523-a59e-44b2-aa96-5f17d872c9c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "headline = cleaned.binary_content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "5b1e34e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array(['accord_video', 'aftermath_shootout', 'allow_couple_flee',\n",
+       "       'approach_truck', 'barricade_patrol_car', 'cam_footage',\n",
+       "       'chaotic_morning_begin', 'chase_truck', 'decide_attempt',\n",
+       "       'emergency_room_stab', 'entry_suspect_fire', 'escape_vehicle_hand',\n",
+       "       'explain_officer_form', 'force_cop_dive', 'free_driver',\n",
+       "       'give_police', 'greene_madison_montgomery', 'harrow_footage',\n",
+       "       'hijack_attempt', 'hold_hostage', 'hole_truck', 'hospital_dayton',\n",
+       "       'hostage_access_road', 'id_woman_purse', 'initial_officer_catch',\n",
+       "       'international_airport_access', 'international_airport_london',\n",
+       "       'late_die_morning', 'lead_officer', 'left_front_tire',\n",
+       "       'london_police_officer', 'man_wife', 'minivan_foot_chase',\n",
+       "       'montgomery_truck_roll', 'multiple_bullet_hole',\n",
+       "       'name_ronald_barbara', 'negotiate_couple_release',\n",
+       "       'numerous_bullet_hole', 'officer_stop_minivan', 'ohio_highway_cop',\n",
+       "       'ohio_highway_patrol', 'ohio_state_highway', 'open_fire',\n",
+       "       'original_driver_truck', 'parked_car', 'parking_lot',\n",
+       "       'police_car_chase', 'rear_light', 'return_fire', 'road_accord',\n",
+       "       'several_bullet', 'several_county', 'several_county_stop',\n",
+       "       'shoot_truck', 'shut_negotiation', 'special_response_team',\n",
+       "       'spike_strip_stop', 'stop_dayton', 'stop_till_morning',\n",
+       "       'strip_chase', 'suspect_point_handgun', 'tactical_gear_approach',\n",
+       "       'taylor_male_driver', 'transport_miami', 'trooper_return_fire',\n",
+       "       'trooper_see', 'truck_drive', 'truck_drive_police',\n",
+       "       'turn_hostage_standoff', 'tv_officer_multiple',\n",
+       "       'unload_round_pistol', 'victim_truck',\n",
+       "       'visible_registration_harrow', 'wbn_body', 'wbns_truck',\n",
+       "       'white_werner'], dtype=object)"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "headline[5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "677055b4-978e-4253-90f4-3f903662e225",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# vectorise the words\n",
+    "doc_dict = gensim.corpora.Dictionary(headline)\n",
+    "docs_vecs = [doc_dict.doc2bow(doc) for doc in headline]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "a54d1768-b069-4936-a156-deaf0b506d93",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of unique tokens: 128764\n",
+      "Number of articles: 1761\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('Number of unique tokens: %d' % len(doc_dict)) \n",
+    "print('Number of articles: %d' % len(docs_vecs)) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "9147fa86-1503-4252-bd9b-92fea1e6a926",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('heavy_rain', 134),\n",
+      " ('supply_chain', 92),\n",
+      " ('strong_wind', 77),\n",
+      " ('national_weather_service', 72),\n",
+      " ('tropical_storm', 59),\n",
+      " ('critical_destination_port', 58),\n",
+      " ('industrial_action', 53),\n",
+      " ('global_supply_chain', 53),\n",
+      " ('heavy_rainfall', 49),\n",
+      " ('international_airport', 46),\n",
+      " ('united_state', 46),\n",
+      " ('high_yard_density', 43),\n",
+      " ('social_medium', 41),\n",
+      " ('hong_kong', 41),\n",
+      " ('national_hurricane_center', 41),\n",
+      " ('global_port_tracker', 39),\n",
+      " ('global_shipping_disruption', 38),\n",
+      " ('sign_confidence_consumer', 38),\n",
+      " ('upgrade_import_forecast', 38),\n",
+      " ('port_authority', 37),\n",
+      " ('new_york_city', 36),\n",
+      " ('schedule_reliability', 36),\n",
+      " ('new_york', 34),\n",
+      " ('coastal_area', 34),\n",
+      " ('east_coast', 34),\n",
+      " ('global_economy', 34),\n",
+      " ('economic_growth', 33),\n",
+      " ('mediterranean_demand', 33),\n",
+      " ('trade_trade_statement', 33),\n",
+      " ('high_wind', 32),\n",
+      " ('customer_demand', 32),\n",
+      " ('trade_statement', 32),\n",
+      " ('severe_weather', 31),\n",
+      " ('american_market', 31),\n",
+      " ('relevant_information', 31),\n",
+      " ('tropical_cyclone', 31),\n",
+      " ('severe_thunderstorm', 31),\n",
+      " ('several_day', 31),\n",
+      " ('matadi_cape_town', 31),\n",
+      " ('pacific_trade_statement', 31),\n",
+      " ('situation_port_face', 31),\n",
+      " ('boost_business', 30),\n",
+      " ('important_transport_route', 30),\n",
+      " ('industry_share_market', 30),\n",
+      " ('part_commitment_provide', 30),\n",
+      " ('relevant_news_hope', 30),\n",
+      " ('state_port', 30),\n",
+      " ('current_situation', 29),\n",
+      " ('fourth_quarter', 29),\n",
+      " ('shanghai_ningbo_shekou', 29),\n",
+      " ('additional_capacity_cater', 29),\n",
+      " ('new_zealand', 29),\n",
+      " ('full_network', 29),\n",
+      " ('coast_port', 28),\n",
+      " ('cargo_move_supply', 28),\n",
+      " ('crew_member', 28),\n",
+      " ('storm_surge', 28),\n",
+      " ('great_china', 28),\n",
+      " ('many_company', 28),\n",
+      " ('day_trade_asia', 28),\n",
+      " ('dp_world', 27),\n",
+      " ('international_container', 27),\n",
+      " ('power_outage', 27),\n",
+      " ('abijian_conakry_maputo', 27),\n",
+      " ('america_latin', 27),\n",
+      " ('america_trade_maersk', 27),\n",
+      " ('arrival_trade_asia', 27),\n",
+      " ('click_link_stay', 27),\n",
+      " ('connect_customer_supply', 27),\n",
+      " ('currency_depreciation_demand', 27),\n",
+      " ('day_coast_day', 27),\n",
+      " ('fiscal_support_consumer', 27),\n",
+      " ('lift_outlook', 27),\n",
+      " ('lome_onne', 27),\n",
+      " ('monthly_please', 27),\n",
+      " ('pacific_america_trade', 27),\n",
+      " ('pointe_noire_balboa', 27),\n",
+      " ('post_subscribe_asia', 27),\n",
+      " ('professional_find_market', 27),\n",
+      " ('question_supply_chain', 27),\n",
+      " ('service_china', 27),\n",
+      " ('useful_subscribe_maersk', 27),\n",
+      " ('help_business', 26),\n",
+      " ('new_jersey', 26),\n",
+      " ('port_congestion', 26),\n",
+      " ('asia_trade_statement', 26),\n",
+      " ('america_trade_statement', 26),\n",
+      " ('international_longshore_warehouse', 25),\n",
+      " ('next_day', 25),\n",
+      " ('many_country', 25),\n",
+      " ('meet_firm', 25),\n",
+      " ('website_see_service', 25),\n",
+      " ('situation_trade', 25),\n",
+      " ('vietnam_cambodia_myanmar', 25),\n",
+      " ('main_port', 25),\n",
+      " ('high_inflation', 24),\n",
+      " ('port_situation', 24),\n",
+      " ('high_level', 24),\n",
+      " ('china_area', 24),\n",
+      " ('passenger_service', 23)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Calculate word frequencies\n",
+    "word_frequencies = {doc_dict[word_id]: freq for word_id, freq in doc_dict.cfs.items()}\n",
+    "sorted_words = sorted(word_frequencies.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "pprint(sorted_words[:100])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5ed78239-2ce1-4784-a8f4-4c7438c8627b",
+   "metadata": {},
+   "source": [
+    "# LDA Modelling"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9db83273-461d-4f70-b23f-ec967579d94f",
+   "metadata": {},
+   "source": [
+    "## Benchmark Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "e6d577bd-9936-4d45-be90-345af2eb4827",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 1.32 s, sys: 837 ms, total: 2.15 s\n",
+      "Wall time: 1min 21s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "# Build LDA benchmark model\n",
+    "lda_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n",
+    "                                       id2word=doc_dict,\n",
+    "                                       num_topics=4, \n",
+    "                                       random_state=42,\n",
+    "                                       chunksize=100,\n",
+    "                                       passes=10,\n",
+    "                                       per_word_topics=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "c4f1521f-5f43-40d2-a3a3-a8ac2ca6fec2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[(0,\n",
+      "  '0.000*\"heavy_rain\" + 0.000*\"tropical_storm\" + '\n",
+      "  '0.000*\"national_weather_service\" + 0.000*\"strong_wind\" + '\n",
+      "  '0.000*\"average_day\" + 0.000*\"united_state\" + 0.000*\"supply_chain\" + '\n",
+      "  '0.000*\"port_congestion\" + 0.000*\"heavy_rainfall\" + '\n",
+      "  '0.000*\"national_hurricane_center\"'),\n",
+      " (1,\n",
+      "  '0.000*\"global_port_tracker\" + 0.000*\"global_shipping_disruption\" + '\n",
+      "  '0.000*\"upgrade_import_forecast\" + 0.000*\"sign_confidence_consumer\" + '\n",
+      "  '0.000*\"supply_chain\" + 0.000*\"american_market\" + '\n",
+      "  '0.000*\"relevant_information\" + 0.000*\"relevant_news_hope\" + '\n",
+      "  '0.000*\"state_port\" + 0.000*\"industry_share_market\"'),\n",
+      " (2,\n",
+      "  '0.001*\"heavy_rain\" + 0.000*\"supply_chain\" + 0.000*\"united_state\" + '\n",
+      "  '0.000*\"critical_destination_port\" + 0.000*\"national_weather_service\" + '\n",
+      "  '0.000*\"strong_wind\" + 0.000*\"high_yard_density\" + '\n",
+      "  '0.000*\"mediterranean_demand\" + 0.000*\"pacific_trade_statement\" + '\n",
+      "  '0.000*\"trade_trade_statement\"'),\n",
+      " (3,\n",
+      "  '0.000*\"heavy_rain\" + 0.000*\"critical_destination_port\" + '\n",
+      "  '0.000*\"global_supply_chain\" + 0.000*\"trade_statement\" + '\n",
+      "  '0.000*\"schedule_reliability\" + 0.000*\"day_trade_asia\" + '\n",
+      "  '0.000*\"full_network\" + 0.000*\"full_truck\" + 0.000*\"january_trade_maersk\" + '\n",
+      "  '0.000*\"stable_network\"')]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Print the Keyword in the 10 topics\n",
+    "pprint(lda_model.print_topics())\n",
+    "doc_lda = lda_model[docs_vecs]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "fd57b1f4-a6cd-41e8-964f-d8a1d30aa3c9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Coherence Score LDAModel:  0.5135998374037359\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Compute Benchmark Coherence Score\n",
+    "coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n",
+    "coherence_lda = coherence_model_lda.get_coherence()\n",
+    "print('\\nCoherence Score LDAModel: ', coherence_lda)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "152e5a3a-7afe-4fb8-a02f-d7492ad80936",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Perplexity for LDAModel:  -11.69195177074239\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Compute Benchmark Perplexity\n",
+    "perplex= lda_model.log_perplexity(docs_vecs, total_docs=None) #For LDAModel\n",
+    "  # a measure of how good the model is. lower the better.\n",
+    "\n",
+    "print('\\nPerplexity for LDAModel: ', perplex)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "7dd3a60a-5c6f-4249-9868-30528a5b0ac8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis\n",
+    "\n",
+    "# feed the LDA model into the pyLDAvis instance\n",
+    "pyLDAvis.enable_notebook()\n",
+    "visual= gensimvis.prepare(lda_model, docs_vecs, doc_dict)\n",
+    "\n",
+    "# Save the output to the html file\n",
+    "pyLDAvis.save_html(visual, \"data/topic_viz_benchmark_moderate.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "3a5612f7-6358-49c8-aba9-8aa54e275c6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Topic Keywords</th>\n",
+       "      <th>Topic ID</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.000*\"heavy_rain\" + 0.000*\"tropical_storm\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\" + 0.000*\"average_day\" + 0.000*\"united_state\"</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.000*\"global_port_tracker\" + 0.000*\"global_shipping_disruption\" + 0.000*\"upgrade_import_forecast\" + 0.000*\"sign_confidence_consumer\" + 0.000*\"supply_chain\" + 0.000*\"american_market\"</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0.001*\"heavy_rain\" + 0.000*\"supply_chain\" + 0.000*\"united_state\" + 0.000*\"critical_destination_port\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\"</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0.000*\"heavy_rain\" + 0.000*\"critical_destination_port\" + 0.000*\"global_supply_chain\" + 0.000*\"trade_statement\" + 0.000*\"schedule_reliability\" + 0.000*\"day_trade_asia\"</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                           Topic Keywords  \\\n",
+       "0                                       0.000*\"heavy_rain\" + 0.000*\"tropical_storm\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\" + 0.000*\"average_day\" + 0.000*\"united_state\"   \n",
+       "1  0.000*\"global_port_tracker\" + 0.000*\"global_shipping_disruption\" + 0.000*\"upgrade_import_forecast\" + 0.000*\"sign_confidence_consumer\" + 0.000*\"supply_chain\" + 0.000*\"american_market\"   \n",
+       "2                           0.001*\"heavy_rain\" + 0.000*\"supply_chain\" + 0.000*\"united_state\" + 0.000*\"critical_destination_port\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\"   \n",
+       "3                  0.000*\"heavy_rain\" + 0.000*\"critical_destination_port\" + 0.000*\"global_supply_chain\" + 0.000*\"trade_statement\" + 0.000*\"schedule_reliability\" + 0.000*\"day_trade_asia\"   \n",
+       "\n",
+       "   Topic ID  \n",
+       "0         0  \n",
+       "1         1  \n",
+       "2         2  \n",
+       "3         3  "
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.set_option('max_colwidth', 200)\n",
+    "# Get the topics and their top keywords into a dataframe\n",
+    "topics = lda_model.show_topics(num_words=6) \n",
+    "\n",
+    "topic_keywords = pd.DataFrame()\n",
+    "for topic_id, topic in topics:\n",
+    "    topic_keywords.at[topic_id, 'Topic Keywords'] = topic\n",
+    "\n",
+    "topic_keywords['Topic ID'] = topic_keywords.index\n",
+    "# topic_keywords['Topic Name'] = topic_mapping \n",
+    "topic_keywords"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "26da4eea-06a0-4ff7-ae14-2f40fa0db04b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# break "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1895598f-3e5f-4acd-83a6-4491cc90f695",
+   "metadata": {},
+   "source": [
+    "# Hyper-Perameter Tuning and Evaluation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "47136c89-ff7b-4ac9-840f-04122fe62160",
+   "metadata": {},
+   "source": [
+    "Run the cells below only for re-modelling with new datasets, the whole tuning and evaluation process may take hours to run."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "c79ca5c4-e078-43ce-a430-8c1ed93dcd64",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hyper-perameter tuning (alpha and beta)\n",
+    "def compute_coherence_values(corpus, dictionary, k, a, b):\n",
+    "    \n",
+    "    lda_model = gensim.models.LdaMulticore(corpus=corpus,\n",
+    "                                           id2word=dictionary,\n",
+    "                                           num_topics=k, \n",
+    "                                           random_state=42,\n",
+    "                                           chunksize=100,\n",
+    "                                           passes=10,\n",
+    "                                           alpha=a,\n",
+    "                                           eta=b)\n",
+    "    \n",
+    "    coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n",
+    "    coherence = coherence_model_lda.get_coherence()\n",
+    "    perplex = lda_model.log_perplexity(docs_vecs, total_docs=None) \n",
+    "    \n",
+    "    return coherence, perplex"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "1c3c8478-9336-40f2-bb30-a37db4243b67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# setup\n",
+    "import numpy as np\n",
+    "\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "model_list = []\n",
+    "coherence_values = []\n",
+    "perplexity_values = []\n",
+    "model_topics = []\n",
+    "alpha_result = []\n",
+    "beta_result = []\n",
+    "\n",
+    "# topic ranges\n",
+    "num_topics = range(4, 13)\n",
+    "\n",
+    "# Alpha parameter\n",
+    "alpha = list(np.arange(0.31, 1, 0.3))\n",
+    "alpha.append('symmetric')\n",
+    "alpha.append('asymmetric')\n",
+    "\n",
+    "# Beta parameter\n",
+    "beta = list(np.arange(0.31, 1, 0.3))\n",
+    "beta.append('symmetric')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7e6bc53-0b57-4858-879a-644eca54ddbc",
+   "metadata": {},
+   "source": [
+    "Rational behind the alpha and eta: https://stats.stackexchange.com/questions/37405/natural-interpretation-for-lda-hyperparameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "02877b81-32df-4168-8e62-4cbca2be100b",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Topic range:  range(4, 13)\n",
+      "Alpha:  [0.31, 0.61, 0.9099999999999999, 'symmetric', 'asymmetric']\n",
+      "Beta:  [0.31, 0.61, 0.9099999999999999, 'symmetric']\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Topic range: \",num_topics)\n",
+    "print(\"Alpha: \",alpha)\n",
+    "print(\"Beta: \", beta)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "3c1f703c-4778-467f-a12e-0c18eeb274c5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2024-06-30 15:33:21.408395\n",
+      "#Topics: 4, CV Score: 0.5164676429141806, PV Score: -11.613380568733923, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.5683002282838892, PV Score: -11.563534433528984, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.46555259667124016, PV Score: -11.53219474749234, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.5159944038644447, PV Score: -11.50783852707268, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.4639539916466071, PV Score: -11.49178161514173, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.6087247241979915, PV Score: -11.474821405795332, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.5444179957484441, PV Score: -11.469740741939871, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.5694955423900976, PV Score: -11.473400251077415, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.49528605221915506, PV Score: -11.464966921298775, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.4846077269482417, PV Score: -11.615188474292983, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.40044534503210266, PV Score: -11.571275497697407, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.47710647516703003, PV Score: -11.542620661350824, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.41352955820583087, PV Score: -11.540942420187077, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.4289987234137122, PV Score: -11.52253612455723, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.4870040213227611, PV Score: -11.517891900057037, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.550960377654979, PV Score: -11.515178838931973, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.54629246756211, PV Score: -11.521576629251783, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.4734093628757035, PV Score: -11.663901784573016, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.5226264223935461, PV Score: -11.581600670664274, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.4815626090375563, PV Score: -11.57887038938498, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.4822722108681728, PV Score: -11.559852785176927, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.5190214814534763, PV Score: -11.58883143718404, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.45997514950408147, PV Score: -11.55820866243133, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.36374367474380254, PV Score: -11.63640550749839, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.5643564534964567, PV Score: -11.557503103730841, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.5469185375162117, PV Score: -11.5647045020449, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.5792875415068292, PV Score: -11.806654395764777, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.49804879882635616, PV Score: -11.642897316807662, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.5778525494321111, PV Score: -11.619415198399734, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.46263841554243873, PV Score: -11.603945568112582, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.4751517261203773, PV Score: -11.610468303646345, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.3914935846000445, PV Score: -11.642292032598931, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.6211150499938938, PV Score: -11.582063142378686, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.6020085770917671, PV Score: -11.583284044840946, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.6175933853797404, PV Score: -11.59165919505012, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.5908368128582339, PV Score: -11.564916248235656, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.5003561625277873, PV Score: -11.637393333313193, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.5317007800262091, PV Score: -11.641278862591506, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.557100339849869, PV Score: -11.584370827514544, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.5172402989720184, PV Score: -11.566723792196806, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.4886338324086602, PV Score: -11.54297247369616, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.5747517437985423, PV Score: -11.543691538358624, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6423710177739427, PV Score: -11.54859461537098, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.5144410257534869, PV Score: -11.73660202888069, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.5717578206426642, PV Score: -11.530830842170962, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.5180085860214778, PV Score: -11.597568405435847, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.48198140101849224, PV Score: -11.580472178461592, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.5224885861991673, PV Score: -11.607306821970512, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.5116028282066244, PV Score: -11.566861788342026, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.4497755401591277, PV Score: -11.575814895292964, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.5327566529438476, PV Score: -11.586548862112604, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.5214711529865992, PV Score: -11.628991593567058, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.6139640248470433, PV Score: -11.594641144555219, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.5356515378529471, PV Score: -11.59298627864858, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.49160818384314564, PV Score: -11.60400992430138, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.489468146274511, PV Score: -11.596778903516313, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.47456800821881234, PV Score: -11.59473954643288, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.5576401494669467, PV Score: -11.594596066878076, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.4667184261166916, PV Score: -11.60378701602699, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.5695086900271756, PV Score: -11.609075892863483, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.6005670921966928, PV Score: -11.614071327825993, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.5480401775800159, PV Score: -11.623267686418437, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.5396539221868254, PV Score: -11.712120797560939, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.45957893891496127, PV Score: -11.6729179593589, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.5497056258415677, PV Score: -11.65337679958747, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.5704158041118883, PV Score: -11.645450690343361, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.5268609558304878, PV Score: -11.638214852770608, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.4658802544250874, PV Score: -11.63919464483393, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.6506499863493337, PV Score: -11.63849523064737, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.649136503615956, PV Score: -11.616778698707527, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.6461235115651532, PV Score: -11.660982724024588, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.5795686612759658, PV Score: -11.654057997902278, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.43540817795833214, PV Score: -11.686659200023914, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.5462127792495466, PV Score: -11.62457497965543, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.5524384036237932, PV Score: -11.604354138411734, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.5902480026727759, PV Score: -11.588582708471723, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.47181028096737715, PV Score: -11.583687892570778, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.6963152362329474, PV Score: -11.581137216572229, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6356546949964338, PV Score: -11.583413852955387, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.6110954828509537, PV Score: -11.578235683186872, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.5948748370571434, PV Score: -11.591490601197242, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.48818167146918845, PV Score: -11.61574038870984, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.4509752119451145, PV Score: -11.604195597608339, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.35645184464437624, PV Score: -11.637862647568788, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.526882700494322, PV Score: -11.653081895856653, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.42882071728968496, PV Score: -11.609087232613648, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.6693854566085815, PV Score: -11.614610403392634, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.6179132712789255, PV Score: -11.623350711437743, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.6085517947565203, PV Score: -11.63547085027884, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.5577009173064904, PV Score: -11.651522052657057, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.5230671592592115, PV Score: -11.623717800797593, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.42768193852006214, PV Score: -11.62144694067679, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.5364419205784232, PV Score: -11.624175638013885, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.4857096769423311, PV Score: -11.63428665005544, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.5044375689801471, PV Score: -11.642138170613716, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.5696949744243359, PV Score: -11.654881948442265, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.6326338442828181, PV Score: -11.66680515031184, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.5727395073356502, PV Score: -11.69273499062231, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.559542950238316, PV Score: -11.748344593064868, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.3804434166795542, PV Score: -11.768219382154312, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.5346775324429552, PV Score: -11.668832927089504, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.35761369835980755, PV Score: -11.7234783761902, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.5739180802234388, PV Score: -11.650366733021364, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.4690450833134745, PV Score: -11.671911408908096, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.6456959957990995, PV Score: -11.692264722739667, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.6358178052250946, PV Score: -11.694376238908927, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.6545497380387766, PV Score: -11.728265629814926, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.5668227847904856, PV Score: -11.831189552283824, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.5079465498144028, PV Score: -11.744019308081587, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.5207685525937951, PV Score: -11.56250304658784, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.4520436901506503, PV Score: -11.517440615925524, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.3984484028061592, PV Score: -11.49927289529406, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.46282464845379434, PV Score: -11.447392562926892, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.5628759566317312, PV Score: -11.431744800328639, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.5550416515331734, PV Score: -11.414885267043056, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.5292928552625016, PV Score: -11.412137160368818, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.4300247772463878, PV Score: -11.42635756927043, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.551938709076145, PV Score: -11.646172984411194, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.4809237301776637, PV Score: -11.559942575549936, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.46228300273399897, PV Score: -11.515382800949316, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.46287366351823206, PV Score: -11.4950582564519, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.4606576501635787, PV Score: -11.486398241110296, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.4481608851641272, PV Score: -11.520590899826077, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.5543837883238459, PV Score: -11.57085700936084, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.5316944180923253, PV Score: -11.578237826778107, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.4687550311670818, PV Score: -11.453118532016195, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.5365308671952822, PV Score: -11.58505859292843, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.4573271215291844, PV Score: -11.55400501711275, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.4845637557935425, PV Score: -11.538682634708005, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.43505334283558816, PV Score: -11.560310334374622, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.48972094471582783, PV Score: -11.52262741120793, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.5105059500845196, PV Score: -11.51304907591031, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.5591753304786676, PV Score: -11.5054622260105, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.5528861076804188, PV Score: -11.504385112555493, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.49367285382840453, PV Score: -11.500990267519892, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.46594738594111, PV Score: -11.638161524533613, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.5212817101806027, PV Score: -11.605645781016982, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.44807641083986144, PV Score: -11.582140562178676, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.509705458396568, PV Score: -11.559129856046134, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.45072428914784013, PV Score: -11.521973552776846, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.6163956153172733, PV Score: -11.532030259535352, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.5731211242304707, PV Score: -11.526033204967101, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.6000508615312063, PV Score: -11.49509548343454, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.4536127133023553, PV Score: -11.499639773335504, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.5517238147309061, PV Score: -11.613287330943077, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.653586278160635, PV Score: -11.544483709714015, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.5297708553518212, PV Score: -11.565107511094201, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.4308728891563382, PV Score: -11.482917171438388, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.5225479879550474, PV Score: -11.452961250714646, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.6266251392086057, PV Score: -11.436045988851452, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6244003133589429, PV Score: -11.420447490550442, Alpha: asymmetric, Beta: 0.31\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "File \u001b[0;32m<timed exec>:10\u001b[0m\n",
+      "Cell \u001b[0;32mIn[23], line 4\u001b[0m, in \u001b[0;36mcompute_coherence_values\u001b[0;34m(corpus, dictionary, k, a, b)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcompute_coherence_values\u001b[39m(corpus, dictionary, k, a, b):\n\u001b[0;32m----> 4\u001b[0m     lda_model \u001b[38;5;241m=\u001b[39m \u001b[43mgensim\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLdaMulticore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcorpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mid2word\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdictionary\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mnum_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m42\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43mpasses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m                                           \u001b[49m\u001b[43meta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     13\u001b[0m     coherence_model_lda \u001b[38;5;241m=\u001b[39m CoherenceModel(model\u001b[38;5;241m=\u001b[39mlda_model, texts\u001b[38;5;241m=\u001b[39mheadline, dictionary\u001b[38;5;241m=\u001b[39mdoc_dict, coherence\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mc_v\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     14\u001b[0m     coherence \u001b[38;5;241m=\u001b[39m coherence_model_lda\u001b[38;5;241m.\u001b[39mget_coherence()\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamulticore.py:186\u001b[0m, in \u001b[0;36mLdaMulticore.__init__\u001b[0;34m(self, corpus, num_topics, id2word, workers, chunksize, passes, batch, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, random_state, minimum_probability, minimum_phi_value, per_word_topics, dtype)\u001b[0m\n\u001b[1;32m    183\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(alpha, \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m alpha \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mauto\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m    184\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto-tuning alpha not implemented in LdaMulticore; use plain LdaModel.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 186\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mLdaMulticore\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m    187\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcorpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_topics\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    188\u001b[0m \u001b[43m    \u001b[49m\u001b[43mid2word\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mid2word\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunksize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpasses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpasses\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43malpha\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    189\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdecay\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecay\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meval_every\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meval_every\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miterations\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miterations\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    190\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgamma_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgamma_threshold\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrandom_state\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mminimum_probability\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mminimum_probability\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    191\u001b[0m \u001b[43m    \u001b[49m\u001b[43mminimum_phi_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mminimum_phi_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mper_word_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mper_word_topics\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    192\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamodel.py:521\u001b[0m, in \u001b[0;36mLdaModel.__init__\u001b[0;34m(self, corpus, num_topics, id2word, distributed, chunksize, passes, update_every, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, minimum_probability, random_state, ns_conf, minimum_phi_value, per_word_topics, callbacks, dtype)\u001b[0m\n\u001b[1;32m    519\u001b[0m use_numpy \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdispatcher \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    520\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[0;32m--> 521\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupdate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchunks_as_numpy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_numpy\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    522\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39madd_lifecycle_event(\n\u001b[1;32m    523\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcreated\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    524\u001b[0m     msg\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrained \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtime\u001b[38;5;241m.\u001b[39mtime()\u001b[38;5;250m \u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;250m \u001b[39mstart\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124ms\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    525\u001b[0m )\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamulticore.py:286\u001b[0m, in \u001b[0;36mLdaMulticore.update\u001b[0;34m(self, corpus, chunks_as_numpy)\u001b[0m\n\u001b[1;32m    283\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlog_perplexity(chunk, total_docs\u001b[38;5;241m=\u001b[39mlencorpus)\n\u001b[1;32m    285\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtraining LDA model using \u001b[39m\u001b[38;5;132;01m%i\u001b[39;00m\u001b[38;5;124m processes\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mworkers)\n\u001b[0;32m--> 286\u001b[0m pool \u001b[38;5;241m=\u001b[39m \u001b[43mPool\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mworkers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mworker_e_step\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mjob_queue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresult_queue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    287\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m pass_ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpasses):\n\u001b[1;32m    288\u001b[0m     queue_size, reallen \u001b[38;5;241m=\u001b[39m [\u001b[38;5;241m0\u001b[39m], \u001b[38;5;241m0\u001b[39m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/context.py:119\u001b[0m, in \u001b[0;36mBaseContext.Pool\u001b[0;34m(self, processes, initializer, initargs, maxtasksperchild)\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''Returns a process pool object'''\u001b[39;00m\n\u001b[1;32m    118\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpool\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Pool\n\u001b[0;32m--> 119\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mPool\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocesses\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitializer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmaxtasksperchild\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    120\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_context\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/pool.py:215\u001b[0m, in \u001b[0;36mPool.__init__\u001b[0;34m(self, processes, initializer, initargs, maxtasksperchild, context)\u001b[0m\n\u001b[1;32m    213\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_processes \u001b[38;5;241m=\u001b[39m processes\n\u001b[1;32m    214\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 215\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_repopulate_pool\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    216\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m    217\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pool:\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/pool.py:306\u001b[0m, in \u001b[0;36mPool._repopulate_pool\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    305\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_repopulate_pool\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 306\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_repopulate_pool_static\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ctx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mProcess\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    307\u001b[0m \u001b[43m                                        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_processes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    308\u001b[0m \u001b[43m                                        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_pool\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inqueue\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    309\u001b[0m \u001b[43m                                        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_outqueue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_initializer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    310\u001b[0m \u001b[43m                                        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_initargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    311\u001b[0m \u001b[43m                                        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_maxtasksperchild\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    312\u001b[0m \u001b[43m                                        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_wrap_exception\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/pool.py:329\u001b[0m, in \u001b[0;36mPool._repopulate_pool_static\u001b[0;34m(ctx, Process, processes, pool, inqueue, outqueue, initializer, initargs, maxtasksperchild, wrap_exception)\u001b[0m\n\u001b[1;32m    327\u001b[0m w\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m=\u001b[39m w\u001b[38;5;241m.\u001b[39mname\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcess\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPoolWorker\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m    328\u001b[0m w\u001b[38;5;241m.\u001b[39mdaemon \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 329\u001b[0m \u001b[43mw\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    330\u001b[0m pool\u001b[38;5;241m.\u001b[39mappend(w)\n\u001b[1;32m    331\u001b[0m util\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madded worker\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/process.py:121\u001b[0m, in \u001b[0;36mBaseProcess.start\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _current_process\u001b[38;5;241m.\u001b[39m_config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdaemon\u001b[39m\u001b[38;5;124m'\u001b[39m), \\\n\u001b[1;32m    119\u001b[0m        \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdaemonic processes are not allowed to have children\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    120\u001b[0m _cleanup()\n\u001b[0;32m--> 121\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_popen \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_Popen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    122\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sentinel \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_popen\u001b[38;5;241m.\u001b[39msentinel\n\u001b[1;32m    123\u001b[0m \u001b[38;5;66;03m# Avoid a refcycle if the target function holds an indirect\u001b[39;00m\n\u001b[1;32m    124\u001b[0m \u001b[38;5;66;03m# reference to the process object (see bpo-30775)\u001b[39;00m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/context.py:289\u001b[0m, in \u001b[0;36mSpawnProcess._Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m    286\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m    287\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_Popen\u001b[39m(process_obj):\n\u001b[1;32m    288\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpopen_spawn_posix\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Popen\n\u001b[0;32m--> 289\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mPopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_spawn_posix.py:32\u001b[0m, in \u001b[0;36mPopen.__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, process_obj):\n\u001b[1;32m     31\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fds \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 32\u001b[0m     \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_fork.py:19\u001b[0m, in \u001b[0;36mPopen.__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturncode \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfinalizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_launch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_spawn_posix.py:62\u001b[0m, in \u001b[0;36mPopen._launch\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msentinel \u001b[38;5;241m=\u001b[39m parent_r\n\u001b[1;32m     61\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(parent_w, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwb\u001b[39m\u001b[38;5;124m'\u001b[39m, closefd\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m---> 62\u001b[0m         \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetbuffer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     64\u001b[0m     fds_to_close \u001b[38;5;241m=\u001b[39m []\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "import datetime\n",
+    "import numpy as np\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "print(datetime.datetime.now())\n",
+    "\n",
+    "for a in alpha:\n",
+    "    for b in beta:\n",
+    "        for num in num_topics:\n",
+    "            cv, pv = compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=num, a=a, b=b) \n",
+    "\n",
+    "            model_topics.append(num)    \n",
+    "            coherence_values.append(cv)  \n",
+    "            perplexity_values.append(pv)\n",
+    "            alpha_result.append(a)\n",
+    "            beta_result.append(b)\n",
+    "            print(\"#Topics: \" + str(num) + \", CV Score: \" + str(coherence_values[-1]) + \", PV Score: \" + str(perplexity_values[-1]) + \", Alpha: \" + str(alpha_result[-1]) + \", Beta: \" + str(beta_result[-1]))\n",
+    "    \n",
+    "print(datetime.datetime.now())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "364ff6d5-e3da-4dde-a2c8-5375fc5d711f",
+   "metadata": {},
+   "source": [
+    "The table below reveals the top 20 fine tuned models with best combinations of coherence score and perplexity score. It was sorted by the coherence score in descending order as a higher coherence score indicates a better model, and sorted the perplexity score in ascending order as a lower perplexity score indicates a better model. While coherence score evaluates the quality of the topics, the perplexity score evaluates the overall performance of the model in predicting new documents. Usually, the coherence score is a better metric to use if the goal is to obtain topics that are semantically coherent and interpretable. Perplexity score, on the other hand, is a better metric to use if the goal is to build a model that generalises well to new data, in other words, how confident the model is in predicting the new data (Sánchez-Aguayo, et al., 2022). Ultimately, we aim to get a balance between the perplexity value and coherence score when determining our final model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "78a60032-a4d7-44d4-841c-a1bd3740d5dd",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Find the top 20 combinations based on Coherence Score and Perplexity Score\n",
+    "result = pd.DataFrame(\n",
+    "    {'Topics': model_topics,\n",
+    "     'Coherence Score': coherence_values,\n",
+    "     'Perplexity Score': perplexity_values,\n",
+    "     'Alpha': alpha_result,\n",
+    "     'Beta': beta_result\n",
+    "    })\n",
+    "result.sort_values(by=['Coherence Score', 'Perplexity Score'], ascending=[False, True]).head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3461df57-c069-4ad2-80d7-8890dec9438e",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "result.to_csv('data/lda_fine_tuning_result_moderate.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "800e5a4b-7302-42e8-97b0-5b598c1c80ae",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Show graph Topics vs Coherence Score\n",
+    "result.groupby('Alpha').plot(x='Topics', y='Coherence Score', legend = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26996b89-0e7a-4f2d-8cf7-c4a716569bc2",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Show graph Topics vs Perplexity Score\n",
+    "\n",
+    "plt.plot(model_topics, coherence_values)\n",
+    "plt.xlabel(\"Num Topics\")\n",
+    "plt.ylabel(\"Coherence Score\")\n",
+    "plt.legend((\"Coherence Score\"), loc='best')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "91d2f4c1-de77-44b6-b41b-fcc9a07233e8",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Show graph Topics vs Perplexity Score\n",
+    "\n",
+    "plt.plot(model_topics, perplexity_values)\n",
+    "plt.xlabel(\"Num Topics\")\n",
+    "plt.ylabel(\"Perplexity score\")\n",
+    "plt.legend((\"perplexity_values\"), loc='best')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cdc3ddd2-f743-4e5b-b6c6-2656e0b77aec",
+   "metadata": {},
+   "source": [
+    "## Final Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "490734ed-077c-4fb0-930c-0b42f4f63c94",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# realised that there may be some overlaps for more than 5 topics, but below 5 topics results in low differentiation and high ambiguity among the topics.\n",
+    "# LDA is not suitable for this dataset\n",
+    "k = 9\n",
+    "a = 'symmetric'\n",
+    "# a = 0.31\n",
+    "# b = 0.31\n",
+    "b = 'symmetric'\n",
+    "\n",
+    "\n",
+    "final_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n",
+    "                                           id2word=doc_dict,\n",
+    "                                           num_topics=k, \n",
+    "                                           random_state=42,\n",
+    "                                           chunksize=100,\n",
+    "                                           passes=10,\n",
+    "                                           alpha=a,\n",
+    "                                           eta=b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "afe8abf0-2d12-414e-92be-a655865addb1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=k, a=a, b=b) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8430a827-6dbb-4737-8ccc-78ed17a01234",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "#Set up the environment to display the graphical outputs\n",
+    "# feed the LDA model into the pyLDAvis instance\n",
+    "pyLDAvis.enable_notebook()\n",
+    "visual= gensimvis.prepare(final_model, docs_vecs, doc_dict)\n",
+    "\n",
+    "#Save the output to the html file\n",
+    "pyLDAvis.save_html(visual, \"data/topic_viz12_mod_training.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e30d71a-a3c7-40c7-94c0-7eea1bedc887",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "final_model.print_topics(num_words=30)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "607d2cfd-b3ca-4f99-9e01-d320ca98a2a0",
+   "metadata": {},
+   "source": [
+    "# Save the final model "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "84eb2746-173a-4283-bca5-681f77548698",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Save a model to disk, or reload a pre-trained model\n",
+    "# naming convention: final_model_topic_alpha_eta\n",
+    "final_model.save(\"models/final_model_9_sym_sym\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7b6e4d9-a577-4dfb-ba6e-fc74365880f4",
+   "metadata": {},
+   "source": [
+    "# Find dominant topic(s) for each news article"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0eeecbcb-358c-44f9-8463-75cdfac0ba90",
+   "metadata": {},
+   "source": [
+    "Attach the dominant topics back to the news dataset for classifying purpose."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8bebb269-dbb0-4c46-925c-38de0f2bcfd7",
+   "metadata": {},
+   "source": [
+    "Made use of gensim lda's own function: https://radimrehurek.com/gensim/models/ldamodel.html"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f585ff52-b60d-4d70-ae64-a7c23d2cc6c1",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "def format_topics_sentences(ldamodel, corpus, data):\n",
+    "    # Preallocate memory for the DataFrame\n",
+    "    num_docs = len(corpus)\n",
+    "    sent_topics = {'Dominant_Topic': [0] * num_docs, 'Perc_Contribution': [0.0] * num_docs, 'Topic_Distribution': [()] * num_docs}\n",
+    "    \n",
+    "    # Get main topic in each document\n",
+    "    for i, row in enumerate(ldamodel[corpus]):\n",
+    "        row = sorted(row, key=lambda x: (x[1]), reverse=True)\n",
+    "        if row:\n",
+    "            # Get the Dominant topic, Perc Contribution and Keywords for each document\n",
+    "            dominant_topic, perc_contribution = row[0]\n",
+    "            topic_distribution = row\n",
+    "            sent_topics['Dominant_Topic'][i] = int(dominant_topic)\n",
+    "            sent_topics['Perc_Contribution'][i] = round(perc_contribution, 4)\n",
+    "            sent_topics['Topic_Distribution'][i] = topic_distribution\n",
+    "\n",
+    "    # Create the DataFrame\n",
+    "    sent_topics_df = pd.DataFrame(sent_topics)\n",
+    "    sent_topics_df['Text'] = data\n",
+    "\n",
+    "    return sent_topics_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24d3ff60-035e-4133-9ffd-88cce5cdccb1",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_topic_sents_keywords = format_topics_sentences(ldamodel=final_model, corpus=docs_vecs, data=cleaned.Headline_Details)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c88b088b",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Format\n",
+    "df_dominant_topic = df_topic_sents_keywords.reset_index()\n",
+    "df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Topic_Distribution', 'Text']\n",
+    "\n",
+    "# Show\n",
+    "df_dominant_topic.head(10)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "560da382-aa86-4df1-8b85-56b057a27cd4",
+   "metadata": {},
+   "source": [
+    "# Result Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4fe6b40b-6922-4de3-8d9e-dac7474b6303",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_dominant_topic[\"Dominant_Topic\"].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b9917340-31cf-48af-871f-b481128fdf22",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "# Get value counts of each topic\n",
+    "topic_counts = df_dominant_topic[\"Dominant_Topic\"].value_counts()\n",
+    "\n",
+    "# Create a bar plot\n",
+    "plt.figure(figsize=(8, 6))\n",
+    "topic_counts.plot(kind=\"bar\", color=\"skyblue\")\n",
+    "\n",
+    "# Add labels to the bars\n",
+    "for i, count in enumerate(topic_counts):\n",
+    "    plt.text(i, count, str(count), ha=\"center\", va=\"bottom\")\n",
+    "\n",
+    "# Add labels and title\n",
+    "plt.xlabel(\"Topics\")\n",
+    "plt.ylabel(\"Number of News\")\n",
+    "plt.title(\"Topic Distribution\")\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.xticks(rotation=45)  # Rotate x-axis labels for better readability\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "70e7d652-4421-45e0-93f8-aaa51c186422",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_dominant_topic.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69932a6e-7159-46b1-98f8-827d99b95c54",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Sample 100 rows, can change the random_state for different samples\n",
+    "sampled_data = df_dominant_topic.sample(n=100, random_state=42)  \n",
+    "sampled_df = pd.DataFrame(sampled_data).reset_index()\n",
+    "sampled_df.to_csv('data/sample_moderate.csv')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/09_topic_modelling_severe.ipynb

@@ -0,0 +1,1859 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "46322fb5-5918-4b70-9689-9e0781439ac4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n",
+      "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "daf1e3d1-75ac-4299-8bed-2f413a49f9a6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import nltk\n",
+    "from nltk.tokenize import sent_tokenize\n",
+    "from nltk.tokenize import word_tokenize\n",
+    "\n",
+    "import gensim\n",
+    "from gensim import corpora\n",
+    "from gensim import similarities\n",
+    "from gensim import models\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "# from wordcloud import WordCloud, ImageColorGenerator\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import os\n",
+    "import datetime\n",
+    "\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "c673c907-e1d8-4d64-9a73-c15c15b78e7f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2024-06-30 15:39:16.255404\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(datetime.datetime.now())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "49e6de6b-71bd-4948-8827-52601406058f",
+   "metadata": {},
+   "source": [
+    "# Import the data with full news content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "49222182-7811-4fa6-8c0a-21d3a546863e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_parquet('data/processed_data2.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "3fb59a30",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>lat</th>\n",
+       "      <th>lon</th>\n",
+       "      <th>...</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "      <th>Headline_Details</th>\n",
+       "      <th>url</th>\n",
+       "      <th>title</th>\n",
+       "      <th>content</th>\n",
+       "      <th>cleaned_content</th>\n",
+       "      <th>binary_content</th>\n",
+       "      <th>word_count</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-4.05608</td>\n",
+       "      <td>137.11302</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZ2h0d...</td>\n",
+       "      <td>Freeport Indonesia mine workers extend strike ...</td>\n",
+       "      <td>Trucks are seen on a road in the Grasberg copp...</td>\n",
+       "      <td>[truck, be, see, on, road, in, grasberg, coppe...</td>\n",
+       "      <td>[adkerson_jakarta_try, agreement_freeport_indo...</td>\n",
+       "      <td>53</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Congestion</td>\n",
+       "      <td>China</td>\n",
+       "      <td>27/4/17 9:16</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>29.52000</td>\n",
+       "      <td>121.33190</td>\n",
+       "      <td>...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiVWh0d...</td>\n",
+       "      <td>Typhoon Muifa to shut China ports for second t...</td>\n",
+       "      <td>By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...</td>\n",
+       "      <td>[by, sam, whelan, typhoon, have, prompt, port,...</td>\n",
+       "      <td>[additional_ripple_effect, avoid_path_typhoon,...</td>\n",
+       "      <td>44</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>5</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 16:20</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZWh0d...</td>\n",
+       "      <td>Jakarta Police Receive 2 More Reports on Coldp...</td>\n",
+       "      <td>TEMPO.CO, Jakarta - South Jakarta Metro Police...</td>\n",
+       "      <td>[jakarta, south, jakarta, metro, police, recei...</td>\n",
+       "      <td>[actress_accord, available_day_concert, click_...</td>\n",
+       "      <td>24</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>6</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>Severe winds have downed billboards and trees ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Roadway Closure / Disruption, Flooding, Severe...</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>19/4/17 9:10</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-6.91264</td>\n",
+       "      <td>107.65700</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>16.0</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiSWh0d...</td>\n",
+       "      <td>Indonesia hit by some of strongest winds recorded</td>\n",
+       "      <td>A man stands near damaged houses following a t...</td>\n",
+       "      <td>[man, stand, near, damage, house, follow, torn...</td>\n",
+       "      <td>[bbc_indonesia, climatologist_government_resea...</td>\n",
+       "      <td>28</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>14</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>Government sources are reporting a tornado has...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Tornado</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>17/9/18 19:55</td>\n",
+       "      <td>2018.0</td>\n",
+       "      <td>37.51000</td>\n",
+       "      <td>-77.61000</td>\n",
+       "      <td>...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>9.0</td>\n",
+       "      <td>38.0</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMigAFod...</td>\n",
+       "      <td>UPDATE: Number of homes without power down to ...</td>\n",
+       "      <td>More than 90,000 homes and businesses across t...</td>\n",
+       "      <td>[more, than, home, business, across, richmond,...</td>\n",
+       "      <td>[advise_seek_alternate, affect_richmond, alter...</td>\n",
+       "      <td>134</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 23 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   id                                           Headline  \\\n",
+       "0   1  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1   3  Shanghai port congestion impacts terminals in ...   \n",
+       "2   5  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "3   6  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "4  14  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "\n",
+       "                                             Details  Severity  \\\n",
+       "0  Media sources indicate that workers at the Gra...  Moderate   \n",
+       "1  The persisting port congestion at Shanghai’s Y...     Minor   \n",
+       "2  According to local police in Jakarta, two expl...   Extreme   \n",
+       "3  Severe winds have downed billboards and trees ...  Moderate   \n",
+       "4  Government sources are reporting a tornado has...     Minor   \n",
+       "\n",
+       "                                            Category         Region  \\\n",
+       "0                                Mine Workers Strike      Indonesia   \n",
+       "1                                    Port Congestion          China   \n",
+       "2                         Bombing, Police Operations      Indonesia   \n",
+       "3  Roadway Closure / Disruption, Flooding, Severe...      Indonesia   \n",
+       "4                                            Tornado  United States   \n",
+       "\n",
+       "        Datetime    Year       lat        lon  ...  if_labeled Month  Week  \\\n",
+       "0  28/5/17 17:08  2017.0  -4.05608  137.11302  ...       False   5.0  21.0   \n",
+       "1   27/4/17 9:16  2017.0  29.52000  121.33190  ...       False   4.0  17.0   \n",
+       "2  24/5/17 16:20  2017.0       NaN        NaN  ...        True   5.0  21.0   \n",
+       "3   19/4/17 9:10  2017.0  -6.91264  107.65700  ...        True   4.0  16.0   \n",
+       "4  17/9/18 19:55  2018.0  37.51000  -77.61000  ...        True   9.0  38.0   \n",
+       "\n",
+       "                                    Headline_Details  \\\n",
+       "0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1  Shanghai port congestion impacts terminals in ...   \n",
+       "2  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "3  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "4  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "\n",
+       "                                                 url  \\\n",
+       "0  https://news.google.com/rss/articles/CBMiZ2h0d...   \n",
+       "1  https://news.google.com/rss/articles/CBMiVWh0d...   \n",
+       "2  https://news.google.com/rss/articles/CBMiZWh0d...   \n",
+       "3  https://news.google.com/rss/articles/CBMiSWh0d...   \n",
+       "4  https://news.google.com/rss/articles/CBMigAFod...   \n",
+       "\n",
+       "                                               title  \\\n",
+       "0  Freeport Indonesia mine workers extend strike ...   \n",
+       "1  Typhoon Muifa to shut China ports for second t...   \n",
+       "2  Jakarta Police Receive 2 More Reports on Coldp...   \n",
+       "3  Indonesia hit by some of strongest winds recorded   \n",
+       "4  UPDATE: Number of homes without power down to ...   \n",
+       "\n",
+       "                                             content  \\\n",
+       "0  Trucks are seen on a road in the Grasberg copp...   \n",
+       "1  By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...   \n",
+       "2  TEMPO.CO, Jakarta - South Jakarta Metro Police...   \n",
+       "3  A man stands near damaged houses following a t...   \n",
+       "4  More than 90,000 homes and businesses across t...   \n",
+       "\n",
+       "                                     cleaned_content  \\\n",
+       "0  [truck, be, see, on, road, in, grasberg, coppe...   \n",
+       "1  [by, sam, whelan, typhoon, have, prompt, port,...   \n",
+       "2  [jakarta, south, jakarta, metro, police, recei...   \n",
+       "3  [man, stand, near, damage, house, follow, torn...   \n",
+       "4  [more, than, home, business, across, richmond,...   \n",
+       "\n",
+       "                                      binary_content word_count  \n",
+       "0  [adkerson_jakarta_try, agreement_freeport_indo...         53  \n",
+       "1  [additional_ripple_effect, avoid_path_typhoon,...         44  \n",
+       "2  [actress_accord, available_day_concert, click_...         24  \n",
+       "3  [bbc_indonesia, climatologist_government_resea...         28  \n",
+       "4  [advise_seek_alternate, affect_richmond, alter...        134  \n",
+       "\n",
+       "[5 rows x 23 columns]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "09113e88-66cc-414c-a953-da04db83c4ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(3681, 23)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "037e74fc-bbcd-43e3-8346-799920cca8d8",
+   "metadata": {},
+   "source": [
+    "# Vectorisation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d67cef3a-59fb-4dd8-adc8-2cf288b90728",
+   "metadata": {},
+   "source": [
+    "NLP vectorization refers to the process of converting text data into numerical vectors that machine learning algorithms can understand and process. \n",
+    "\n",
+    "Bag-of-Words (BoW) is used here that represents text as a collection of unique words along with their frequencies. Each word is assigned an index, and the vector contains the count of each word present in the document."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c95b7b8a-9767-469d-812d-c9a9d9fee0e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_copy = df.copy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "dfb2001e-04c1-49dc-b423-a64ea47af5a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# choose only the extreme and severe cases for modelling\n",
+    "cleaned = df_copy[df_copy['Severity'].isin(['Extreme', 'Severe'])]\n",
+    "cleaned.reset_index(drop=True, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "de71c523-a59e-44b2-aa96-5f17d872c9c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "headline = cleaned.binary_content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "5b1e34e1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array(['number_container', 'accord_detective_llamas', 'anyone_talk_crime',\n",
+       "       'arizmendi_girlfriend_become', 'auto_theft_robbery',\n",
+       "       'clothing_makeup_shoe', 'clue_loot', 'decode_container_stack',\n",
+       "       'detective_chavez', 'detective_put', 'electric_bicycle',\n",
+       "       'empire_farther_south', 'freight_train_repeat',\n",
+       "       'google_placard_lock', 'hard_drive_tablet', 'homicide_drug_gang',\n",
+       "       'inside_container_secure', 'llama_straight_tell',\n",
+       "       'llama_work_connie', 'metal_lock_size', 'mile_east',\n",
+       "       'motel_room_storage', 'plentiful_tv_beer',\n",
+       "       'southern_california_couple', 'succumb_bolt_cutter', 'sure_sign',\n",
+       "       'upgraded_lock'], dtype=object)"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "headline[5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "677055b4-978e-4253-90f4-3f903662e225",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# vectorise the words\n",
+    "doc_dict = gensim.corpora.Dictionary(headline)\n",
+    "docs_vecs = [doc_dict.doc2bow(doc) for doc in headline]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "a54d1768-b069-4936-a156-deaf0b506d93",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of unique tokens: 30464\n",
+      "Number of articles: 300\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('Number of unique tokens: %d' % len(doc_dict)) \n",
+    "print('Number of articles: %d' % len(docs_vecs)) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "9147fa86-1503-4252-bd9b-92fea1e6a926",
+   "metadata": {
+    "scrolled": true,
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[('heavy_rain', 15),\n",
+      " ('global_supply_chain', 15),\n",
+      " ('national_hurricane_center', 13),\n",
+      " ('heavy_rainfall', 12),\n",
+      " ('port_los', 12),\n",
+      " ('hong_kong', 12),\n",
+      " ('united_state', 11),\n",
+      " ('critical_destination_port', 11),\n",
+      " ('global_port_tracker', 11),\n",
+      " ('global_shipping_disruption', 11),\n",
+      " ('sign_confidence_consumer', 11),\n",
+      " ('upgrade_import_forecast', 11),\n",
+      " ('national_weather_service', 10),\n",
+      " ('social_medium', 10),\n",
+      " ('moment_exception_request', 10),\n",
+      " ('sorry_site', 10),\n",
+      " ('technical_difficulty_please', 10),\n",
+      " ('trade_statement', 9),\n",
+      " ('tropical_storm', 9),\n",
+      " ('help_business', 9),\n",
+      " ('meet_firm', 9),\n",
+      " ('website_see_service', 9),\n",
+      " ('supply_chain', 8),\n",
+      " ('strong_wind', 8),\n",
+      " ('coastal_area', 7),\n",
+      " ('geological_survey', 7),\n",
+      " ('asian_country', 7),\n",
+      " ('day_trade_asia', 7),\n",
+      " ('global_demand', 7),\n",
+      " ('global_economy', 7),\n",
+      " ('high_yard_density', 7),\n",
+      " ('inch_rain', 7),\n",
+      " ('coast_port', 7),\n",
+      " ('empty_container', 7),\n",
+      " ('union_worker', 7),\n",
+      " ('many_area', 6),\n",
+      " ('customer_demand', 6),\n",
+      " ('economic_growth', 6),\n",
+      " ('free_day', 6),\n",
+      " ('full_network', 6),\n",
+      " ('import_volume', 6),\n",
+      " ('major_economy', 6),\n",
+      " ('negative_impact', 6),\n",
+      " ('vertical_insight', 6),\n",
+      " ('supply_chain_issue', 6),\n",
+      " ('economic_recovery', 6),\n",
+      " ('death_toll', 6),\n",
+      " ('new_york_city', 6),\n",
+      " ('america_trade_statement', 6),\n",
+      " ('average_day', 6),\n",
+      " ('late_issue', 5),\n",
+      " ('paul_brashier_vice', 5),\n",
+      " ('president_drayage', 5),\n",
+      " ('strike_action', 5),\n",
+      " ('large_number', 5),\n",
+      " ('current_situation', 5),\n",
+      " ('high_inflation', 5),\n",
+      " ('severe_weather_event', 5),\n",
+      " ('accurate_quote_market', 5),\n",
+      " ('adapt_supply_chain', 5),\n",
+      " ('america_space', 5),\n",
+      " ('apapa_tin_tema', 5),\n",
+      " ('asia_day', 5),\n",
+      " ('china_area_chb', 5),\n",
+      " ('china_area_warehouse', 5),\n",
+      " ('company_face', 5),\n",
+      " ('critical_resource', 5),\n",
+      " ('date_change', 5),\n",
+      " ('energy_price', 5),\n",
+      " ('energy_price_fall', 5),\n",
+      " ('export_volume', 5),\n",
+      " ('fourth_quarter', 5),\n",
+      " ('full_truck', 5),\n",
+      " ('future_consumer_demand', 5),\n",
+      " ('general_administration_custom', 5),\n",
+      " ('high_flexibility_transit', 5),\n",
+      " ('high_interest_rate', 5),\n",
+      " ('high_inventory', 5),\n",
+      " ('increase_capacity', 5),\n",
+      " ('india_freight_cost', 5),\n",
+      " ('indirect_service', 5),\n",
+      " ('january_trade_maersk', 5),\n",
+      " ('load_truck_box', 5),\n",
+      " ('low_confidence', 5),\n",
+      " ('low_figure', 5),\n",
+      " ('main_product_continue', 5),\n",
+      " ('main_route_area', 5),\n",
+      " ('many_company', 5),\n",
+      " ('market_average', 5),\n",
+      " ('matadi_cape_town', 5),\n",
+      " ('mile_service', 5),\n",
+      " ('monetary_fund', 5),\n",
+      " ('new_air', 5),\n",
+      " ('new_law_limit', 5),\n",
+      " ('new_sea_rail', 5),\n",
+      " ('ocean_market', 5),\n",
+      " ('ocean_network', 5),\n",
+      " ('ok_day', 5),\n",
+      " ('operational_disruption', 5),\n",
+      " ('relevant_rate', 5)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Calculate word frequencies\n",
+    "word_frequencies = {doc_dict[word_id]: freq for word_id, freq in doc_dict.cfs.items()}\n",
+    "sorted_words = sorted(word_frequencies.items(), key=lambda x: x[1], reverse=True)\n",
+    "\n",
+    "pprint(sorted_words[:100])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5ed78239-2ce1-4784-a8f4-4c7438c8627b",
+   "metadata": {},
+   "source": [
+    "# LDA Modelling"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aacc2eb2-bce9-462f-b048-cc26baa2383d",
+   "metadata": {},
+   "source": [
+    "We initially selected a fixed topic number for model pipelien development and benchmark model setup. Then we used the full dataset for fine-tuning and evaluation."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9db83273-461d-4f70-b23f-ec967579d94f",
+   "metadata": {},
+   "source": [
+    "## Benchmark Model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "e6d577bd-9936-4d45-be90-345af2eb4827",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Build LDA benchmark model\n",
+    "lda_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n",
+    "                                       id2word=doc_dict,\n",
+    "                                       num_topics=4, \n",
+    "                                       random_state=42,\n",
+    "                                       chunksize=100,\n",
+    "                                       passes=10,\n",
+    "                                       per_word_topics=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "c4f1521f-5f43-40d2-a3a3-a8ac2ca6fec2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[(0,\n",
+      "  '0.001*\"technical_difficulty_please\" + 0.001*\"moment_exception_request\" + '\n",
+      "  '0.001*\"sorry_site\" + 0.001*\"heavy_rain\" + 0.000*\"heavy_rainfall\" + '\n",
+      "  '0.000*\"port_los\" + 0.000*\"national_weather_service\" + 0.000*\"coast_port\" + '\n",
+      "  '0.000*\"united_state\" + 0.000*\"empty_container\"'),\n",
+      " (1,\n",
+      "  '0.001*\"upgrade_import_forecast\" + 0.001*\"sign_confidence_consumer\" + '\n",
+      "  '0.001*\"global_shipping_disruption\" + 0.001*\"global_port_tracker\" + '\n",
+      "  '0.001*\"meet_firm\" + 0.001*\"website_see_service\" + 0.001*\"help_business\" + '\n",
+      "  '0.000*\"national_hurricane_center\" + 0.000*\"passenger_service\" + '\n",
+      "  '0.000*\"hong_kong\"'),\n",
+      " (2,\n",
+      "  '0.001*\"global_supply_chain\" + 0.000*\"negative_impact\" + '\n",
+      "  '0.000*\"critical_destination_port\" + 0.000*\"trade_statement\" + '\n",
+      "  '0.000*\"warm_winter_china\" + 0.000*\"import_volume\" + 0.000*\"global_demand\" + '\n",
+      "  '0.000*\"day_trade_asia\" + 0.000*\"slow_react\" + 0.000*\"full_network\"'),\n",
+      " (3,\n",
+      "  '0.000*\"critical_destination_port\" + 0.000*\"social_medium\" + '\n",
+      "  '0.000*\"meteorological_agency\" + 0.000*\"hong_kong\" + 0.000*\"many_area\" + '\n",
+      "  '0.000*\"trade_statement\" + 0.000*\"prime_minister\" + '\n",
+      "  '0.000*\"america_trade_statement\" + 0.000*\"oakland_day\" + '\n",
+      "  '0.000*\"ready_rescue_operation\"')]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Print the Keyword in the 10 topics\n",
+    "pprint(lda_model.print_topics())\n",
+    "doc_lda = lda_model[docs_vecs]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "fd57b1f4-a6cd-41e8-964f-d8a1d30aa3c9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Coherence Score LDAModel:  0.37181231277776183\n",
+      "CPU times: user 216 ms, sys: 147 ms, total: 364 ms\n",
+      "Wall time: 8min 58s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "# Compute Benchmark Coherence Score\n",
+    "coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n",
+    "coherence_lda = coherence_model_lda.get_coherence()\n",
+    "print('\\nCoherence Score LDAModel: ', coherence_lda)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "152e5a3a-7afe-4fb8-a02f-d7492ad80936",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Perplexity for LDAModel:  -10.57378514568444\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Compute Benchmark Perplexity\n",
+    "perplex= lda_model.log_perplexity(docs_vecs, total_docs=None) #For LDAModel\n",
+    "  # a measure of how good the model is. lower the better.\n",
+    "\n",
+    "print('\\nPerplexity for LDAModel: ', perplex)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "7dd3a60a-5c6f-4249-9868-30528a5b0ac8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n",
+      "/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/site-packages/joblib/externals/loky/backend/fork_exec.py:38: DeprecationWarning: This process (pid=56923) is multi-threaded, use of fork() may lead to deadlocks in the child.\n",
+      "  pid = os.fork()\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "import pyLDAvis\n",
+    "import pyLDAvis.gensim_models as gensimvis\n",
+    "\n",
+    "# feed the LDA model into the pyLDAvis instance\n",
+    "pyLDAvis.enable_notebook()\n",
+    "visual= gensimvis.prepare(lda_model, docs_vecs, doc_dict)\n",
+    "\n",
+    "#Save the output to the html file\n",
+    "pyLDAvis.save_html(visual, \"data/topic_viz_benchmark_severe.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "3a5612f7-6358-49c8-aba9-8aa54e275c6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Topic Keywords</th>\n",
+       "      <th>Topic ID</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.001*\"technical_difficulty_please\" + 0.001*\"moment_exception_request\" + 0.001*\"sorry_site\" + 0.001*\"heavy_rain\" + 0.000*\"heavy_rainfall\" + 0.000*\"port_los\"</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.001*\"upgrade_import_forecast\" + 0.001*\"sign_confidence_consumer\" + 0.001*\"global_shipping_disruption\" + 0.001*\"global_port_tracker\" + 0.001*\"meet_firm\" + 0.001*\"website_see_service\"</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0.001*\"global_supply_chain\" + 0.000*\"negative_impact\" + 0.000*\"critical_destination_port\" + 0.000*\"trade_statement\" + 0.000*\"warm_winter_china\" + 0.000*\"import_volume\"</td>\n",
+       "      <td>2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0.000*\"critical_destination_port\" + 0.000*\"social_medium\" + 0.000*\"meteorological_agency\" + 0.000*\"hong_kong\" + 0.000*\"many_area\" + 0.000*\"trade_statement\"</td>\n",
+       "      <td>3</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                                                                                                            Topic Keywords  \\\n",
+       "0                             0.001*\"technical_difficulty_please\" + 0.001*\"moment_exception_request\" + 0.001*\"sorry_site\" + 0.001*\"heavy_rain\" + 0.000*\"heavy_rainfall\" + 0.000*\"port_los\"   \n",
+       "1  0.001*\"upgrade_import_forecast\" + 0.001*\"sign_confidence_consumer\" + 0.001*\"global_shipping_disruption\" + 0.001*\"global_port_tracker\" + 0.001*\"meet_firm\" + 0.001*\"website_see_service\"   \n",
+       "2                  0.001*\"global_supply_chain\" + 0.000*\"negative_impact\" + 0.000*\"critical_destination_port\" + 0.000*\"trade_statement\" + 0.000*\"warm_winter_china\" + 0.000*\"import_volume\"   \n",
+       "3                              0.000*\"critical_destination_port\" + 0.000*\"social_medium\" + 0.000*\"meteorological_agency\" + 0.000*\"hong_kong\" + 0.000*\"many_area\" + 0.000*\"trade_statement\"   \n",
+       "\n",
+       "   Topic ID  \n",
+       "0         0  \n",
+       "1         1  \n",
+       "2         2  \n",
+       "3         3  "
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pd.set_option('max_colwidth', 200)\n",
+    "# Get the topics and their top keywords into a dataframe\n",
+    "topics = lda_model.show_topics(num_words=6) \n",
+    "\n",
+    "topic_keywords = pd.DataFrame()\n",
+    "for topic_id, topic in topics:\n",
+    "    topic_keywords.at[topic_id, 'Topic Keywords'] = topic\n",
+    "\n",
+    "topic_keywords['Topic ID'] = topic_keywords.index\n",
+    "topic_keywords"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "26da4eea-06a0-4ff7-ae14-2f40fa0db04b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# break "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1895598f-3e5f-4acd-83a6-4491cc90f695",
+   "metadata": {},
+   "source": [
+    "# Hyper-Perameter Tuning and Evaluation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "47136c89-ff7b-4ac9-840f-04122fe62160",
+   "metadata": {},
+   "source": [
+    "Run the cells below only for re-modelling with new datasets, the whole tuning and evaluation process may take hours to run."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "c79ca5c4-e078-43ce-a430-8c1ed93dcd64",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hyper-perameter tuning (alpha and beta)\n",
+    "def compute_coherence_values(corpus, dictionary, k, a, b):\n",
+    "    \n",
+    "    lda_model = gensim.models.LdaMulticore(corpus=corpus,\n",
+    "                                           id2word=dictionary,\n",
+    "                                           num_topics=k, \n",
+    "                                           random_state=42,\n",
+    "                                           chunksize=100,\n",
+    "                                           passes=10,\n",
+    "                                           alpha=a,\n",
+    "                                           eta=b)\n",
+    "    \n",
+    "    coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n",
+    "    coherence = coherence_model_lda.get_coherence()\n",
+    "    perplex = lda_model.log_perplexity(docs_vecs, total_docs=None) \n",
+    "    \n",
+    "    return coherence, perplex"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "1c3c8478-9336-40f2-bb30-a37db4243b67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# setup\n",
+    "import numpy as np\n",
+    "\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "model_list = []\n",
+    "coherence_values = []\n",
+    "perplexity_values = []\n",
+    "model_topics = []\n",
+    "alpha_result = []\n",
+    "beta_result = []\n",
+    "\n",
+    "# topic ranges\n",
+    "num_topics = range(4, 13)\n",
+    "\n",
+    "# Alpha parameter\n",
+    "alpha = list(np.arange(0.31, 1, 0.3))\n",
+    "alpha.append('symmetric')\n",
+    "alpha.append('asymmetric')\n",
+    "\n",
+    "# Beta parameter\n",
+    "beta = list(np.arange(0.31, 1, 0.3))\n",
+    "beta.append('symmetric')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c7e6bc53-0b57-4858-879a-644eca54ddbc",
+   "metadata": {},
+   "source": [
+    "Rational behind the alpha and eta: https://stats.stackexchange.com/questions/37405/natural-interpretation-for-lda-hyperparameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "02877b81-32df-4168-8e62-4cbca2be100b",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Topic range:  range(4, 13)\n",
+      "Alpha:  [0.31, 0.61, 0.9099999999999999, 'symmetric', 'asymmetric']\n",
+      "Beta:  [0.31, 0.61, 0.9099999999999999, 'symmetric']\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Topic range: \",num_topics)\n",
+    "print(\"Alpha: \",alpha)\n",
+    "print(\"Beta: \", beta)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "3c1f703c-4778-467f-a12e-0c18eeb274c5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2024-06-30 15:56:56.953954\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "#Topics: 4, CV Score: 0.3720156705867761, PV Score: -10.531015192970104, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.5104309491692648, PV Score: -10.501295581191243, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.4577416605401658, PV Score: -10.460472706228693, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.47905186758060786, PV Score: -10.45017220690116, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.5170931021465908, PV Score: -10.435198097218018, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.5418406648591022, PV Score: -10.41813577221336, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6289688892634311, PV Score: -10.40650838076674, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.615861099169618, PV Score: -10.400571085548444, Alpha: 0.31, Beta: 0.31\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Traceback (most recent call last):\n",
+      "  File \"<string>\", line 1, in <module>\n",
+      "  File \"/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/multiprocessing/spawn.py\", line 122, in spawn_main\n",
+      "    exitcode = _main(fd, parent_sentinel)\n",
+      "               ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "  File \"/Users/inflaton/anaconda3/envs/maritime/lib/python3.12/multiprocessing/spawn.py\", line 132, in _main\n",
+      "    self = reduction.pickle.load(from_parent)\n",
+      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+      "_pickle.UnpicklingError: pickle data was truncated\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "#Topics: 12, CV Score: 0.5821904373179804, PV Score: -10.405657307982493, Alpha: 0.31, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.39197924000152073, PV Score: -10.431108130993747, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.5158552240339984, PV Score: -10.415847323918724, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.4398095546006567, PV Score: -10.395709424729047, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.4759542844436549, PV Score: -10.390971943426882, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.5228046057671669, PV Score: -10.390099173623508, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.534380007483663, PV Score: -10.383173065174224, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.601346262577239, PV Score: -10.38283548973593, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.6182871521157967, PV Score: -10.381370037404881, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.6301666636692548, PV Score: -10.388156880830003, Alpha: 0.31, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.3906162608542371, PV Score: -10.401723128237816, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.5939884088380677, PV Score: -10.387492626618993, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.4508756651213514, PV Score: -10.37800267053485, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.5023247846073803, PV Score: -10.378610286427143, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.5338912827400732, PV Score: -10.382219717893106, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.5350215871400255, PV Score: -10.378352577214379, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.5843943719668163, PV Score: -10.379151839595481, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.6022597255391141, PV Score: -10.381816469366692, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.6549152327008515, PV Score: -10.3871327637851, Alpha: 0.31, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.38133564373441475, PV Score: -10.57910573063879, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.504405181495512, PV Score: -10.591716275575331, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.4434166489578631, PV Score: -10.577016836067129, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.4650127509674876, PV Score: -10.59307567259223, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.48085842979187243, PV Score: -10.595749214509778, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.5229987979769645, PV Score: -10.572445904678828, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.6011951298180502, PV Score: -10.580254153107184, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.5816514196292673, PV Score: -10.572981018751022, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.54353358800824, PV Score: -10.58723522762571, Alpha: 0.31, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.3747888014696905, PV Score: -10.553273801572038, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.41949099152429126, PV Score: -10.529284579653147, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.42792555699640306, PV Score: -10.493600231954503, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.4414682530110495, PV Score: -10.49167181275119, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.40257655373677936, PV Score: -10.480723050906821, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.46588662192698593, PV Score: -10.462647038151925, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6342207771608466, PV Score: -10.459408243710644, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.5957477827868781, PV Score: -10.457942472137361, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.5069147689054222, PV Score: -10.464993667751155, Alpha: 0.61, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.3819785352237406, PV Score: -10.457655853911954, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.521805619062306, PV Score: -10.44289355248452, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.4718959465808612, PV Score: -10.42865218169654, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.4666775368816065, PV Score: -10.435293881017774, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.4161034826078284, PV Score: -10.43039536604531, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.5356741464147949, PV Score: -10.430215877864402, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.6500234328720327, PV Score: -10.433181454609361, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.5724584031569051, PV Score: -10.436418913727316, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.47510144373255264, PV Score: -10.447173694241018, Alpha: 0.61, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.38858871959573177, PV Score: -10.422449139190153, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.5168433414143676, PV Score: -10.415463384416736, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.4623710938296017, PV Score: -10.410332603528257, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.47872425272635516, PV Score: -10.414848707310925, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.4916000111100134, PV Score: -10.42081190532211, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.5266551042307853, PV Score: -10.423109042698925, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.6308106489770785, PV Score: -10.429297527391322, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.5824504074419617, PV Score: -10.435341814119317, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.5655237958130822, PV Score: -10.446175294783924, Alpha: 0.61, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.38066789853933136, PV Score: -10.600860392033141, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.4230110894509031, PV Score: -10.624957958665691, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.46006178600015324, PV Score: -10.612290512343902, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.4303435103117806, PV Score: -10.629508186597825, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.43866952185453756, PV Score: -10.634503260588486, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.4702721241778767, PV Score: -10.621032221974806, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.6087872469897164, PV Score: -10.634684373455967, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.5297501274565263, PV Score: -10.641694238787153, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.5432630625800524, PV Score: -10.654685933857545, Alpha: 0.61, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.3980327518942032, PV Score: -10.571774729929455, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.47100281301765873, PV Score: -10.551206821119235, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.45585476488442933, PV Score: -10.5221789742425, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.42741889953329854, PV Score: -10.520073750838211, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.4445519399417415, PV Score: -10.513500785821066, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.5316093514063003, PV Score: -10.502280711961715, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6182445479757168, PV Score: -10.506699966303506, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.5706965784614643, PV Score: -10.509385494327319, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.5244982477634946, PV Score: -10.518817895245245, Alpha: 0.9099999999999999, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.34370448027723666, PV Score: -10.473583266573756, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.42177545406640793, PV Score: -10.466430482135557, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.48870590420062365, PV Score: -10.45681790473505, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.4818942174006588, PV Score: -10.462492853055563, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.49058139371286213, PV Score: -10.464079562560514, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.5284476302536344, PV Score: -10.469358208054853, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.5992842730225252, PV Score: -10.474371922798735, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.5550594565914921, PV Score: -10.482537823118147, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.5130408610330546, PV Score: -10.496326340379174, Alpha: 0.9099999999999999, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.3680605610706724, PV Score: -10.437968210441094, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.39834507895056426, PV Score: -10.434366957326267, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.5122840918935663, PV Score: -10.435732920054527, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.5242817034609356, PV Score: -10.444687489964636, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.525222502346446, PV Score: -10.450911057789357, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.5855489762461916, PV Score: -10.459363210197557, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.6033482162866756, PV Score: -10.46709651310468, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.534181138841984, PV Score: -10.478818718345869, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.5044175544841009, PV Score: -10.49287461972145, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.4030162857954225, PV Score: -10.61948175515634, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.47100281301765873, PV Score: -10.643778224222777, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.44431460450915367, PV Score: -10.640983049810698, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.44194061169816934, PV Score: -10.6627668410305, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.43220156375405006, PV Score: -10.66934582156036, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.5246794752098919, PV Score: -10.666334218302142, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.5839917253272302, PV Score: -10.683233193788631, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.5812551603138659, PV Score: -10.68512322593847, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.5267098171572085, PV Score: -10.712358225009568, Alpha: 0.9099999999999999, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.3720156705867761, PV Score: -10.526282588811258, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.49933029874610535, PV Score: -10.48844257371801, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.47205899846058585, PV Score: -10.439636548970395, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.48269249518744994, PV Score: -10.421634188787365, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.538335613683992, PV Score: -10.399271817666595, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.5761042276388716, PV Score: -10.381594889094856, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6148075051567828, PV Score: -10.361131747776065, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.5588504038246359, PV Score: -10.345500948521796, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.5448010772010606, PV Score: -10.345617505586791, Alpha: symmetric, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.39936348065904637, PV Score: -10.426149669454736, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.5090823271966699, PV Score: -10.404214549545916, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.4278461684376491, PV Score: -10.376087516260217, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.5044713577089992, PV Score: -10.365065223661983, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.5585090181447581, PV Score: -10.357648533539216, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.5308131994078027, PV Score: -10.343761327293674, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.5933548111304747, PV Score: -10.333543849654015, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.5157930553152481, PV Score: -10.326695408594487, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.6500731393929028, PV Score: -10.325261572586404, Alpha: symmetric, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.3724208362940652, PV Score: -10.397307737784152, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.6047343451735429, PV Score: -10.37632434171058, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 6, CV Score: 0.40146108571917843, PV Score: -10.358709138199426, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 7, CV Score: 0.5478418889572643, PV Score: -10.352775652604763, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 8, CV Score: 0.4965510774725434, PV Score: -10.35076287539615, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 9, CV Score: 0.5184439894201485, PV Score: -10.338520514214467, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 10, CV Score: 0.6022496614184635, PV Score: -10.33271452286526, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 11, CV Score: 0.4966577290320975, PV Score: -10.328283322344763, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 12, CV Score: 0.6261571238431066, PV Score: -10.32532208498972, Alpha: symmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 4, CV Score: 0.37181231277776183, PV Score: -10.5737851457114, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 5, CV Score: 0.49982175351658986, PV Score: -10.57991503850104, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 6, CV Score: 0.4623746453802937, PV Score: -10.558772260964222, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 7, CV Score: 0.4884610769859329, PV Score: -10.570459578130873, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 8, CV Score: 0.5131836632736474, PV Score: -10.559869552593637, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 9, CV Score: 0.5414297105648862, PV Score: -10.529852182357658, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 10, CV Score: 0.6785044360557178, PV Score: -10.535124275209565, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 11, CV Score: 0.5657397622843224, PV Score: -10.52133404122993, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 12, CV Score: 0.582327359714131, PV Score: -10.526610298202412, Alpha: symmetric, Beta: symmetric\n",
+      "#Topics: 4, CV Score: 0.43912942092956697, PV Score: -10.522848808310528, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 5, CV Score: 0.40751184962270537, PV Score: -10.48541066901554, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 6, CV Score: 0.4542494682857962, PV Score: -10.442773054340485, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 7, CV Score: 0.47041200319916493, PV Score: -10.420479103000723, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 8, CV Score: 0.49233261299212405, PV Score: -10.403763191353837, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 9, CV Score: 0.45907092550739237, PV Score: -10.3786293878649, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 10, CV Score: 0.6217993088001614, PV Score: -10.36148775185113, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 11, CV Score: 0.5185376069390376, PV Score: -10.349232810275204, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 12, CV Score: 0.5034307005049391, PV Score: -10.34076712188869, Alpha: asymmetric, Beta: 0.31\n",
+      "#Topics: 4, CV Score: 0.515628236666662, PV Score: -10.422196504217826, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 5, CV Score: 0.423162935257951, PV Score: -10.398368311505568, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 6, CV Score: 0.45944499325883426, PV Score: -10.374638816393565, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 7, CV Score: 0.4820831491535998, PV Score: -10.36267991668515, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 8, CV Score: 0.4036989725279696, PV Score: -10.354531424153524, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 9, CV Score: 0.45919112117643734, PV Score: -10.342620369274787, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 10, CV Score: 0.6451794871459123, PV Score: -10.330084574702694, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 11, CV Score: 0.572154524993651, PV Score: -10.324677755836108, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 12, CV Score: 0.6194824233122164, PV Score: -10.323963042788398, Alpha: asymmetric, Beta: 0.61\n",
+      "#Topics: 4, CV Score: 0.519154732232678, PV Score: -10.389407917509194, Alpha: asymmetric, Beta: 0.9099999999999999\n",
+      "#Topics: 5, CV Score: 0.4151559388052012, PV Score: -10.373488513541524, Alpha: asymmetric, Beta: 0.9099999999999999\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "File \u001b[0;32m<timed exec>:10\u001b[0m\n",
+      "Cell \u001b[0;32mIn[22], line 14\u001b[0m, in \u001b[0;36mcompute_coherence_values\u001b[0;34m(corpus, dictionary, k, a, b)\u001b[0m\n\u001b[1;32m      4\u001b[0m lda_model \u001b[38;5;241m=\u001b[39m gensim\u001b[38;5;241m.\u001b[39mmodels\u001b[38;5;241m.\u001b[39mLdaMulticore(corpus\u001b[38;5;241m=\u001b[39mcorpus,\n\u001b[1;32m      5\u001b[0m                                        id2word\u001b[38;5;241m=\u001b[39mdictionary,\n\u001b[1;32m      6\u001b[0m                                        num_topics\u001b[38;5;241m=\u001b[39mk, \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     10\u001b[0m                                        alpha\u001b[38;5;241m=\u001b[39ma,\n\u001b[1;32m     11\u001b[0m                                        eta\u001b[38;5;241m=\u001b[39mb)\n\u001b[1;32m     13\u001b[0m coherence_model_lda \u001b[38;5;241m=\u001b[39m CoherenceModel(model\u001b[38;5;241m=\u001b[39mlda_model, texts\u001b[38;5;241m=\u001b[39mheadline, dictionary\u001b[38;5;241m=\u001b[39mdoc_dict, coherence\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mc_v\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 14\u001b[0m coherence \u001b[38;5;241m=\u001b[39m \u001b[43mcoherence_model_lda\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_coherence\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     15\u001b[0m perplex \u001b[38;5;241m=\u001b[39m lda_model\u001b[38;5;241m.\u001b[39mlog_perplexity(docs_vecs, total_docs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m) \n\u001b[1;32m     17\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m coherence, perplex\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/coherencemodel.py:614\u001b[0m, in \u001b[0;36mCoherenceModel.get_coherence\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    605\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_coherence\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    606\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Get coherence value based on pipeline parameters.\u001b[39;00m\n\u001b[1;32m    607\u001b[0m \n\u001b[1;32m    608\u001b[0m \u001b[38;5;124;03m    Returns\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    612\u001b[0m \n\u001b[1;32m    613\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 614\u001b[0m     confirmed_measures \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_coherence_per_topic\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    615\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maggregate_measures(confirmed_measures)\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/coherencemodel.py:574\u001b[0m, in \u001b[0;36mCoherenceModel.get_coherence_per_topic\u001b[0;34m(self, segmented_topics, with_std, with_support)\u001b[0m\n\u001b[1;32m    572\u001b[0m     segmented_topics \u001b[38;5;241m=\u001b[39m measure\u001b[38;5;241m.\u001b[39mseg(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtopics)\n\u001b[1;32m    573\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_accumulator \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 574\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mestimate_probabilities\u001b[49m\u001b[43m(\u001b[49m\u001b[43msegmented_topics\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    576\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(with_std\u001b[38;5;241m=\u001b[39mwith_std, with_support\u001b[38;5;241m=\u001b[39mwith_support)\n\u001b[1;32m    577\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcoherence \u001b[38;5;129;01min\u001b[39;00m BOOLEAN_DOCUMENT_BASED \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcoherence \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mc_w2v\u001b[39m\u001b[38;5;124m'\u001b[39m:\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/coherencemodel.py:546\u001b[0m, in \u001b[0;36mCoherenceModel.estimate_probabilities\u001b[0;34m(self, segmented_topics)\u001b[0m\n\u001b[1;32m    543\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcoherence \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mc_w2v\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m    544\u001b[0m         kwargs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkeyed_vectors\n\u001b[0;32m--> 546\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_accumulator \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmeasure\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprob\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    548\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_accumulator\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/topic_coherence/probability_estimation.py:156\u001b[0m, in \u001b[0;36mp_boolean_sliding_window\u001b[0;34m(texts, segmented_topics, dictionary, window_size, processes)\u001b[0m\n\u001b[1;32m    154\u001b[0m     accumulator \u001b[38;5;241m=\u001b[39m ParallelWordOccurrenceAccumulator(processes, top_ids, dictionary)\n\u001b[1;32m    155\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124musing \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m to estimate probabilities from sliding windows\u001b[39m\u001b[38;5;124m\"\u001b[39m, accumulator)\n\u001b[0;32m--> 156\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43maccumulator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maccumulate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtexts\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwindow_size\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/topic_coherence/text_analysis.py:437\u001b[0m, in \u001b[0;36mParallelWordOccurrenceAccumulator.accumulate\u001b[0;34m(self, texts, window_size)\u001b[0m\n\u001b[1;32m    436\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21maccumulate\u001b[39m(\u001b[38;5;28mself\u001b[39m, texts, window_size):\n\u001b[0;32m--> 437\u001b[0m     workers, input_q, output_q \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart_workers\u001b[49m\u001b[43m(\u001b[49m\u001b[43mwindow_size\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    438\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    439\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mqueue_all_texts(input_q, texts, window_size)\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/topic_coherence/text_analysis.py:471\u001b[0m, in \u001b[0;36mParallelWordOccurrenceAccumulator.start_workers\u001b[0;34m(self, window_size)\u001b[0m\n\u001b[1;32m    469\u001b[0m     accumulator \u001b[38;5;241m=\u001b[39m PatchedWordOccurrenceAccumulator(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrelevant_ids, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdictionary)\n\u001b[1;32m    470\u001b[0m     worker \u001b[38;5;241m=\u001b[39m AccumulatingWorker(input_q, output_q, accumulator, window_size)\n\u001b[0;32m--> 471\u001b[0m     \u001b[43mworker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    472\u001b[0m     workers\u001b[38;5;241m.\u001b[39mappend(worker)\n\u001b[1;32m    474\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m workers, input_q, output_q\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/process.py:121\u001b[0m, in \u001b[0;36mBaseProcess.start\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _current_process\u001b[38;5;241m.\u001b[39m_config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdaemon\u001b[39m\u001b[38;5;124m'\u001b[39m), \\\n\u001b[1;32m    119\u001b[0m        \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdaemonic processes are not allowed to have children\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    120\u001b[0m _cleanup()\n\u001b[0;32m--> 121\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_popen \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_Popen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    122\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sentinel \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_popen\u001b[38;5;241m.\u001b[39msentinel\n\u001b[1;32m    123\u001b[0m \u001b[38;5;66;03m# Avoid a refcycle if the target function holds an indirect\u001b[39;00m\n\u001b[1;32m    124\u001b[0m \u001b[38;5;66;03m# reference to the process object (see bpo-30775)\u001b[39;00m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/context.py:224\u001b[0m, in \u001b[0;36mProcess._Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m    222\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m    223\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_Popen\u001b[39m(process_obj):\n\u001b[0;32m--> 224\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_context\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mProcess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_Popen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/context.py:289\u001b[0m, in \u001b[0;36mSpawnProcess._Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m    286\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m    287\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_Popen\u001b[39m(process_obj):\n\u001b[1;32m    288\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpopen_spawn_posix\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Popen\n\u001b[0;32m--> 289\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mPopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_spawn_posix.py:32\u001b[0m, in \u001b[0;36mPopen.__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, process_obj):\n\u001b[1;32m     31\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fds \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 32\u001b[0m     \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_fork.py:19\u001b[0m, in \u001b[0;36mPopen.__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturncode \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfinalizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_launch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_spawn_posix.py:62\u001b[0m, in \u001b[0;36mPopen._launch\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msentinel \u001b[38;5;241m=\u001b[39m parent_r\n\u001b[1;32m     61\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(parent_w, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwb\u001b[39m\u001b[38;5;124m'\u001b[39m, closefd\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m---> 62\u001b[0m         \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetbuffer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     64\u001b[0m     fds_to_close \u001b[38;5;241m=\u001b[39m []\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "import datetime\n",
+    "import numpy as np\n",
+    "from gensim.models import CoherenceModel\n",
+    "\n",
+    "print(datetime.datetime.now())\n",
+    "\n",
+    "for a in alpha:\n",
+    "    for b in beta:\n",
+    "        for num in num_topics:\n",
+    "            cv, pv = compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=num, a=a, b=b) \n",
+    "\n",
+    "            model_topics.append(num)    \n",
+    "            coherence_values.append(cv)  \n",
+    "            perplexity_values.append(pv)\n",
+    "            alpha_result.append(a)\n",
+    "            beta_result.append(b)\n",
+    "            print(\"#Topics: \" + str(num) + \", CV Score: \" + str(coherence_values[-1]) + \", PV Score: \" + str(perplexity_values[-1]) + \", Alpha: \" + str(alpha_result[-1]) + \", Beta: \" + str(beta_result[-1]))\n",
+    "    \n",
+    "print(datetime.datetime.now())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "364ff6d5-e3da-4dde-a2c8-5375fc5d711f",
+   "metadata": {},
+   "source": [
+    "The table below reveals the top 20 fine tuned models with best combinations of coherence score and perplexity score. It was sorted by the coherence score in descending order as a higher coherence score indicates a better model, and sorted the perplexity score in ascending order as a lower perplexity score indicates a better model. While coherence score evaluates the quality of the topics, the perplexity score evaluates the overall performance of the model in predicting new documents. Usually, the coherence score is a better metric to use if the goal is to obtain topics that are semantically coherent and interpretable. Perplexity score, on the other hand, is a better metric to use if the goal is to build a model that generalises well to new data, in other words, how confident the model is in predicting the new data (Sánchez-Aguayo, et al., 2022). Ultimately, we aim to get a balance between the perplexity value and coherence score when determining our final model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "78a60032-a4d7-44d4-841c-a1bd3740d5dd",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Find the top 20 combinations based on Coherence Score and Perplexity Score\n",
+    "result = pd.DataFrame(\n",
+    "    {'Topics': model_topics,\n",
+    "     'Coherence Score': coherence_values,\n",
+    "     'Perplexity Score': perplexity_values,\n",
+    "     'Alpha': alpha_result,\n",
+    "     'Beta': beta_result\n",
+    "    })\n",
+    "result.sort_values(by=['Coherence Score', 'Perplexity Score'], ascending=[False, True]).head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3461df57-c069-4ad2-80d7-8890dec9438e",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "result.to_csv('data/lda_fine_tuning_result_severe.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "800e5a4b-7302-42e8-97b0-5b598c1c80ae",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Show graph Topics vs Coherence Score\n",
+    "result.groupby('Alpha').plot(x='Topics', y='Coherence Score', legend = True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26996b89-0e7a-4f2d-8cf7-c4a716569bc2",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Show graph Topics vs Perplexity Score\n",
+    "\n",
+    "plt.plot(model_topics, coherence_values)\n",
+    "plt.xlabel(\"Num Topics\")\n",
+    "plt.ylabel(\"Coherence Score\")\n",
+    "plt.legend((\"Coherence Score\"), loc='best')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "91d2f4c1-de77-44b6-b41b-fcc9a07233e8",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Show graph Topics vs Perplexity Score\n",
+    "\n",
+    "plt.plot(model_topics, perplexity_values)\n",
+    "plt.xlabel(\"Num Topics\")\n",
+    "plt.ylabel(\"Perplexity score\")\n",
+    "plt.legend((\"perplexity_values\"), loc='best')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cdc3ddd2-f743-4e5b-b6c6-2656e0b77aec",
+   "metadata": {},
+   "source": [
+    "## Final Model"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8a4196d2-0f8a-4b0b-a6dd-ead9441af44e",
+   "metadata": {},
+   "source": [
+    "Topic 4, 8, 9 and 10 were selected for further evaluation using the visual graphs, considering that the best combination does not always yield the best result since a model with higher number of topics tends to have a better measurable result but may not fit the data the most. \n",
+    "\n",
+    "However, take note that even the random_state was preset and all other parameters were fixed, there are still randomness found that the model may produce inconsistant output each time. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "df1c00ad-ba54-4686-ac75-ef1033066dce",
+   "metadata": {},
+   "source": [
+    "unfortunately, the alter of the number of topics has no much effect on the results, and the news are not clustered into relevant topics properly. also, most topics are stacked together, indicating high similarity and ambiguity among them due to the multi-aspect nature of the news contents. As a result, LDA may not be a suitable solution for this kind of news content. same result goes for moderate and minor."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "490734ed-077c-4fb0-930c-0b42f4f63c94",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# realised that there may be some overlaps for 8 topics, thus 4-6 topics are optimal\n",
+    "k = 8\n",
+    "a = 'asymmetric'\n",
+    "# a = 0.91\n",
+    "# b = 0.61\n",
+    "b = 'symmetric'\n",
+    "\n",
+    "\n",
+    "\n",
+    "final_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n",
+    "                                           id2word=doc_dict,\n",
+    "                                           num_topics=k, \n",
+    "                                           random_state=42,\n",
+    "                                           chunksize=100,\n",
+    "                                           passes=10,\n",
+    "                                           alpha=a,\n",
+    "                                           eta=b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "afe8abf0-2d12-414e-92be-a655865addb1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=k, a=a, b=b) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8430a827-6dbb-4737-8ccc-78ed17a01234",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "#Set up the environment to display the graphical outputs\n",
+    "# feed the LDA model into the pyLDAvis instance\n",
+    "pyLDAvis.enable_notebook()\n",
+    "visual= gensimvis.prepare(final_model, docs_vecs, doc_dict)\n",
+    "\n",
+    "#Save the output to the html file\n",
+    "pyLDAvis.save_html(visual, \"data/topic_viz8_severe_training.html\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5e30d71a-a3c7-40c7-94c0-7eea1bedc887",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "final_model.print_topics(num_words=30)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4fc5e753-e0e7-4520-9e7d-10e26e4d580d",
+   "metadata": {},
+   "source": [
+    "This allows ease access to the trained model for future prediction work."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "84eb2746-173a-4283-bca5-681f77548698",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "#Save a model to disk, or reload a pre-trained model\n",
+    "# naming convention: final_model_topic_alpha_eta\n",
+    "final_model.save(\"final_model_8_asym_sym\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "466c3952-69d1-4d19-b49c-d4b1e1844572",
+   "metadata": {},
+   "source": [
+    "get dominant topics and topic percentage contribution.\n",
+    "Made use of gensim lda's own function: https://radimrehurek.com/gensim/models/ldamodel.html"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cd88034c-2fb8-4f1f-a4e8-85d09b4fc1dc",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "def format_topics_sentences(ldamodel, corpus, data):\n",
+    "    # Preallocate memory for the DataFrame\n",
+    "    num_docs = len(corpus)\n",
+    "    sent_topics = {'Dominant_Topic': [0] * num_docs, 'Perc_Contribution': [0.0] * num_docs, 'Topic_Distribution': [()] * num_docs}\n",
+    "    \n",
+    "    # Get main topic in each document\n",
+    "    for i, row in enumerate(ldamodel[corpus]):\n",
+    "        row = sorted(row, key=lambda x: (x[1]), reverse=True)\n",
+    "        if row:\n",
+    "            # Get the Dominant topic, Perc Contribution and Keywords for each document\n",
+    "            dominant_topic, perc_contribution = row[0]\n",
+    "            topic_distribution = row\n",
+    "            sent_topics['Dominant_Topic'][i] = int(dominant_topic)\n",
+    "            sent_topics['Perc_Contribution'][i] = round(perc_contribution, 4)\n",
+    "            sent_topics['Topic_Distribution'][i] = topic_distribution\n",
+    "\n",
+    "    # Create the DataFrame\n",
+    "    sent_topics_df = pd.DataFrame(sent_topics)\n",
+    "    sent_topics_df['Text'] = data\n",
+    "\n",
+    "    return sent_topics_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24d3ff60-035e-4133-9ffd-88cce5cdccb1",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_topic_sents_keywords = format_topics_sentences(ldamodel=final_model, corpus=docs_vecs, data=cleaned.Headline_Details)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c88b088b",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Format\n",
+    "df_dominant_topic = df_topic_sents_keywords.reset_index()\n",
+    "df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Topic_Distribution', 'Text']\n",
+    "\n",
+    "# Show\n",
+    "df_dominant_topic.head(10)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "560da382-aa86-4df1-8b85-56b057a27cd4",
+   "metadata": {},
+   "source": [
+    "# Result Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4fe6b40b-6922-4de3-8d9e-dac7474b6303",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_dominant_topic[\"Dominant_Topic\"].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b9917340-31cf-48af-871f-b481128fdf22",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "# Get value counts of each topic\n",
+    "topic_counts = df_dominant_topic[\"Dominant_Topic\"].value_counts()\n",
+    "\n",
+    "# Create a bar plot\n",
+    "plt.figure(figsize=(8, 6))\n",
+    "topic_counts.plot(kind=\"bar\", color=\"skyblue\")\n",
+    "\n",
+    "# Add labels to the bars\n",
+    "for i, count in enumerate(topic_counts):\n",
+    "    plt.text(i, count, str(count), ha=\"center\", va=\"bottom\")\n",
+    "\n",
+    "# Add labels and title\n",
+    "plt.xlabel(\"Topics\")\n",
+    "plt.ylabel(\"Number of News\")\n",
+    "plt.title(\"Topic Distribution\")\n",
+    "\n",
+    "# Show the plot\n",
+    "plt.xticks(rotation=45)  # Rotate x-axis labels for better readability\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fffa1e57-f975-4469-a42b-19d76c60fb66",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_dominant_topic.sort_values(by='Topic_Perc_Contrib', ascending=False).head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8510f506-141f-4382-b668-251df1afc95f",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Sample 100 rows, can change the random_state for different samples\n",
+    "sampled_data = df_dominant_topic.sample(n=100, random_state=42)  \n",
+    "sampled_df = pd.DataFrame(sampled_data).reset_index()\n",
+    "sampled_df.to_csv('data/sample_severe.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16388596-a1d6-4509-acac-6dd57220554a",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/10_LLM_evaluation.ipynb

@@ -0,0 +1,355 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a64f4c1c",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "52701550-0bda-4fa4-959f-2b42eb42e140",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8cb2dd41-79e1-45bf-a0eb-ff9abc0baa8b",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df = pd.read_csv(\"cleaned_data_with_categories.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c4e79890-de47-4774-9445-142cff46cf86",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "df3cdbd0-a8b4-4b6e-b5bf-5fdbb7ce93a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_sorted = df.groupby(\"Category\").count().sort_values(by='id', ascending = False)\n",
+    "df_sorted"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d3ddee2c-946c-4ee5-b0ba-31ac6d201261",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "# Function to determine the value for the new column\n",
+    "def categorize(value):\n",
+    "    if '/' in str(value) or ',' in str(value):\n",
+    "        return 'Miscellaneous Events'\n",
+    "    else:\n",
+    "        return value"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b470266e-00c1-4d4c-8738-28b4fde2dcc6",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df['GPT Generated Result'] = df['Category'].apply(categorize)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d5d7300-a663-4394-979e-cf7c5d4f88d5",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df_sorted1 = df.groupby(\"GPT Generated Result\").count().sort_values(by='id', ascending = False)\n",
+    "df_sorted1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6dbe0ac2-4ac6-4d75-9fa7-c856b9370269",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "df.to_csv('result.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5dfe7870-c29c-4942-8301-f5e8b1bd9994",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc6007d3-c6bd-4bc3-8ccc-099440354ce8",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "eva = pd.read_csv('evaluation_result.csv')\n",
+    "eva"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a98d47b7-7ab1-4947-a10a-a36fa303dcc1",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "eva['Result_GPT_True_Count'] = eva['Result_GPT'].astype(int)  # Convert boolean values to integers\n",
+    "result = eva.groupby(\"GPT Generated Result\")['Result_GPT_True_Count'].sum()\n",
+    "\n",
+    "result_gpt = result.sort_values(ascending=False)\n",
+    "result_gpt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ec0defea-c4ad-4f97-9704-23ef83f73ff7",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "test = eva.groupby(\"Category_GoldenResult\").count().sort_values(by='id', ascending = False)\n",
+    "test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6bc57f6b-c554-4634-8a98-45d82546d6f8",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n",
+      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
+   "source": [
+    "eva['Result_Golden_True_Count'] = eva['Result_Golden'].astype(int)  # Convert boolean values to integers\n",
+    "result = eva.groupby(\"Category_GoldenResult\")['Result_Golden_True_Count'].sum()\n",
+    "\n",
+    "# If you want to sort the result by the count in descending order:\n",
+    "result_golden = result.sort_values(ascending=False)\n",
+    "\n",
+    "result_golden"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/12_Pipeline.ipynb

@@ -0,0 +1,598 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n",
+      "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "workding_dir = str(Path.cwd().parent)\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### The file aims for the demonstration of the methdology pipeline, please used our new designed database (new excel attached )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### The following code is when user put a new news article link into the model, the model will extract the headline , Date and Content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Headline: Singapore Airlines stops using Iranian airspace as 'precautionary measure' amid Middle East tensions\n",
+      "Publication Date: 2024-04-14 08:58:00\n",
+      "Content: Advertisement Singapore Singapore Airlines stops using Iranian airspace as 'precautionary measure' amid Middle East tensions Singapore Airlines says it is closely monitoring the situation in the Middle East. (File photo: REUTERS/Edgar Su) New: You can now listen to articles. This audio is generated by an AI tool. 14 Apr 2024 08:58AM (Updated: 14 Apr 2024 07:26PM) Bookmark Bookmark Share WhatsApp Telegram Facebook Twitter Email LinkedIn SINGAPORE: Singapore Airlines (SIA) said on Sunday (Apr 14) \n"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "from bs4 import BeautifulSoup\n",
+    "from datetime import datetime\n",
+    "\n",
+    "\n",
+    "def get_article_details(article_url):\n",
+    "    response = requests.get(article_url)\n",
+    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
+    "\n",
+    "    # Extract headline\n",
+    "    headline_tag = soup.find(\"h1\")\n",
+    "    headline = (\n",
+    "        headline_tag.get_text(strip=True) if headline_tag else \"No headline found\"\n",
+    "    )\n",
+    "\n",
+    "    # Attempt to extract publication date with error handling\n",
+    "    date_container = soup.find(\"div\", class_=\"article-publish\")\n",
+    "    if date_container:\n",
+    "        # Extract the text and handle cleaning it up\n",
+    "        date_text = date_container.get_text(strip=True)\n",
+    "        # Extract the first date assuming it's the publication date (before \"Updated:\")\n",
+    "        publication_date_text = date_text.split(\"(Updated:\")[0].strip()\n",
+    "        try:\n",
+    "            publication_date = datetime.strptime(\n",
+    "                publication_date_text, \"%d %b %Y %I:%M%p\"\n",
+    "            ).strftime(\"%Y-%m-%d %H:%M:%S\")\n",
+    "        except ValueError:\n",
+    "            publication_date = \"No publication date found\"\n",
+    "    else:\n",
+    "        publication_date = \"No publication date found\"\n",
+    "\n",
+    "    # Extract main content of the article\n",
+    "    article_body = soup.find(\"article\")\n",
+    "    if not article_body:\n",
+    "        article_body = soup\n",
+    "    article_text = (\n",
+    "        article_body.get_text(separator=\" \", strip=True)\n",
+    "        if article_body\n",
+    "        else \"No article content found\"\n",
+    "    )\n",
+    "\n",
+    "    return headline, publication_date, article_text\n",
+    "\n",
+    "\n",
+    "# Example usage\n",
+    "article_url = \"https://www.channelnewsasia.com/singapore/singapore-airlines-stops-using-iran-airspace-israel-hamas-war-middle-east-escalation-4264011\"\n",
+    "headline, publication_date, article_content = get_article_details(article_url)\n",
+    "print(\"Headline:\", headline)\n",
+    "print(\"Publication Date:\", publication_date)\n",
+    "print(\"Content:\", article_content[:500])  # Print the first 500 characters to check"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Here is the code for Summarization of the aritlce (PLEASE USE UR OWN OPENAI KEY HAHA)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Summary: Singapore Airlines has stopped using Iranian airspace as a precautionary measure due to the escalating tensions in the Middle East. This move was mirrored by other airlines, including Lufthansa and Austrian Airlines, who also suspended flights to and from Tehran. The situation has also led to economic consequences, as flights\n"
+     ]
+    }
+   ],
+   "source": [
+    "import openai\n",
+    "\n",
+    "\n",
+    "def summarize_article(article_content):\n",
+    "    try:\n",
+    "        # Generating the prompt for GPT-3\n",
+    "        prompt_text = (\n",
+    "            \"Summarize the following article in about 70 words, focusing on \"\n",
+    "            \"what happened, where it happened, and the consequences (economic loss, environmental impact, etc.):\\n\\n\"\n",
+    "            f\"{article_content}\"\n",
+    "        )\n",
+    "\n",
+    "        # Call to OpenAI's Completion API\n",
+    "        response = openai.Completion.create(\n",
+    "            engine=\"gpt-3.5-turbo-instruct\",\n",
+    "            prompt=prompt_text,\n",
+    "            temperature=0.5,\n",
+    "            max_tokens=60,  # Adjust as needed to fit the summary length\n",
+    "            top_p=1,\n",
+    "            frequency_penalty=0,\n",
+    "            presence_penalty=0,\n",
+    "        )\n",
+    "\n",
+    "        # Extracting the text from the response\n",
+    "        summary = response.choices[0].text.strip()\n",
+    "        return summary\n",
+    "    except Exception as e:\n",
+    "        print(f\"An error occurred: {e}\")\n",
+    "        return \"\"\n",
+    "\n",
+    "\n",
+    "summary = summarize_article(article_content)\n",
+    "print(\"Summary:\", summary)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# unique_categories = df['Category'].unique()\n",
+    "# print(unique_categories)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import openai\n",
+    "\n",
+    "\n",
+    "def classify_article(article_content):\n",
+    "    prompt = f\"\"\"Read the following article and classify its content into one of these categories: 'Aviation Advisory',\n",
+    "'Bombing',\n",
+    "'Cargo Disruption',\n",
+    "'Warehouse Theft',\n",
+    "'Chemical Spill',\n",
+    "'Injury',\n",
+    "'Earthquake',\n",
+    "'Flooding',\n",
+    "'Ground Transportation Advisory',\n",
+    "'Hazmat Response',\n",
+    "'Ice Storm',\n",
+    "'Individuals in Focus',\n",
+    "'Industrial Action',\n",
+    "'Maritime Accident',\n",
+    "'Maritime Accident.Ground Transportation Advisory',\n",
+    "'Maritime Advisory',\n",
+    "'Mine Workers Strike',\n",
+    "'Miscellaneous Events',\n",
+    "'Miscellaneous Strikes',\n",
+    "'Network Disruption',\n",
+    "'Non-industrial Fire',\n",
+    "'Police Operations',\n",
+    "'Port Closure',\n",
+    "'Port Congestion',\n",
+    "'Port Disruption',\n",
+    "'Power Outage',\n",
+    "'Production Halt',\n",
+    "'Protest',\n",
+    "'Public Safety ,\n",
+    "'Public Transportation Disruption',\n",
+    "'Roadway Closuren',\n",
+    "'Severe Winds',\n",
+    "'Storm',\n",
+    "'Tornado',\n",
+    "'Train Delays,\n",
+    "'Travel Warning',\n",
+    "'Tropical Cyclone Storm',\n",
+    "'Typhoon',\n",
+    "'Vehicle Accident',\n",
+    "'Weather Advisory',\n",
+    "'Workplace Accident'\n",
+    "Summary: {article_content}\n",
+    "Category:\"\"\"\n",
+    "\n",
+    "    response = openai.Completion.create(\n",
+    "        engine=\"gpt-3.5-turbo-instruct\",  # Adjust according to the latest available and appropriate model\n",
+    "        prompt=prompt,\n",
+    "        temperature=0.7,\n",
+    "        max_tokens=60,  # Adjust based on your needs\n",
+    "        top_p=1.0,\n",
+    "        frequency_penalty=0,\n",
+    "        presence_penalty=0,\n",
+    "        stop=[\"\\n\"],  # Stop generating further when a newline character is encountered\n",
+    "    )\n",
+    "    category = response.choices[0].text.strip()\n",
+    "    return category"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "from bs4 import BeautifulSoup\n",
+    "\n",
+    "\n",
+    "def fetch_article_content(url):\n",
+    "    response = requests.get(url)\n",
+    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
+    "    article_text = \" \".join([p.text for p in soup.find_all(\"p\")])\n",
+    "    return article_text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Category: Aviation Advisory\n"
+     ]
+    }
+   ],
+   "source": [
+    "import openai\n",
+    "\n",
+    "\n",
+    "def classify_article(url):\n",
+    "    # Fetch article content\n",
+    "    article_content = fetch_article_content(url)\n",
+    "\n",
+    "    # Construct the classification prompt\n",
+    "    prompt = f\"\"\"Read the following article and classify its content into one of these categories:\n",
+    "    'Aviation Advisory', 'Bombing',\n",
+    "'Cargo Disruption',\n",
+    "'Chemical Spill',\n",
+    "'Death',\n",
+    "'Earthquake',\n",
+    "'Flooding',\n",
+    "'Ground Transportation Advisory',\n",
+    "'Hazmat Response',\n",
+    "'Ice Storm',\n",
+    "'Individuals in Focus',\n",
+    "'Industrial Action',\n",
+    "'Maritime Accident',\n",
+    "'Maritime Accident.Ground Transportation Advisory',\n",
+    "'Maritime Advisory',\n",
+    "'Mine Workers Strike',\n",
+    "'Miscellaneous Events',\n",
+    "'Miscellaneous Strikes',\n",
+    "'Network Disruption',\n",
+    "'Non-industrial Fire',\n",
+    "'Police Operations',\n",
+    "'Port Closure',\n",
+    "'Port Congestion',\n",
+    "'Port Disruption',\n",
+    "'Power Outage',\n",
+    "'Production Halt',\n",
+    "'Protest / Riot',\n",
+    "'Public Safety / Security',\n",
+    "'Public Transportation Disruption',\n",
+    "'Roadway Closure / Disruption',\n",
+    "'Severe Winds',\n",
+    "'Storm',\n",
+    "'Tornado',\n",
+    "'Train Delays / Disruption',\n",
+    "'Travel Warning',\n",
+    "'Tropical Cyclone / Storm',\n",
+    "'Typhoon',\n",
+    "'Vehicle Accident',\n",
+    "'Weather Advisory',\n",
+    "'Workplace Accident', ...\n",
+    "    \n",
+    "    Article:\n",
+    "    {article_content}\n",
+    "    \n",
+    "    Category:\"\"\"\n",
+    "\n",
+    "    # Classify using OpenAI GPT-3\n",
+    "    response = openai.Completion.create(\n",
+    "        engine=\"gpt-3.5-turbo-instruct\",  # Ensure using a correct and non-deprecated model\n",
+    "        prompt=prompt,\n",
+    "        temperature=0.7,\n",
+    "        max_tokens=60,\n",
+    "        top_p=1.0,\n",
+    "        frequency_penalty=0,\n",
+    "        presence_penalty=0,\n",
+    "        stop=[\"\\n\"],  # Stop generating further when a newline character is encountered\n",
+    "    )\n",
+    "    category = response.choices[0].text.strip()\n",
+    "    return category\n",
+    "\n",
+    "\n",
+    "# Example usage\n",
+    "url = \"https://www.channelnewsasia.com/singapore/singapore-airlines-stops-using-iran-airspace-israel-hamas-war-middle-east-escalation-4264011\"\n",
+    "category = classify_article(url)\n",
+    "print(\"Category:\", category)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Now you the classfication result, which is quite accurate :))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Singapore Airlines stops using Iranian airspace as 'precautionary measure' amid Middle East tensions\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(headline)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Update our database!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Database updated successfully with ID 1.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from datetime import datetime\n",
+    "\n",
+    "\n",
+    "def update_database(file_path, url):\n",
+    "    # Fetch details from the article\n",
+    "    headline, publication_date, article_content = get_article_details(article_url)\n",
+    "    summary = summarize_article(article_content)\n",
+    "    category = classify_article(url)\n",
+    "\n",
+    "    new_data = {\n",
+    "        \"Headline\": headline,\n",
+    "        \"Summary\": summary,\n",
+    "        \"Category\": category,\n",
+    "        \"Datetime\": publication_date,\n",
+    "        \"URL\": article_url,\n",
+    "    }\n",
+    "\n",
+    "    # Load the existing data from the CSV file\n",
+    "    try:\n",
+    "        df = pd.read_csv(file_path)\n",
+    "    except FileNotFoundError:\n",
+    "        # If the file does not exist, create a new DataFrame\n",
+    "        df = pd.DataFrame(\n",
+    "            columns=[\"id\", \"Headline\", \"Summary\", \"Category\", \"Datetime\", \"URL\"]\n",
+    "        )\n",
+    "        new_id = 1  # Start with ID 1 if no file exists\n",
+    "    else:\n",
+    "        # If IDs exist, increment from the last used ID\n",
+    "        new_id = df[\"id\"].max() + 1 if not df.empty else 1\n",
+    "\n",
+    "    # Prepare the new data entry\n",
+    "    new_entry = pd.DataFrame(\n",
+    "        {\n",
+    "            \"id\": [new_id],\n",
+    "            \"Headline\": [new_data[\"Headline\"]],\n",
+    "            \"Summary\": [new_data[\"Summary\"]],\n",
+    "            \"Category\": [new_data[\"Category\"]],\n",
+    "            \"Datetime\": [new_data[\"Datetime\"]],\n",
+    "            \"URL\": [new_data[\"URL\"]],\n",
+    "        }\n",
+    "    )\n",
+    "\n",
+    "    # Append the new data entry to the DataFrame using concat\n",
+    "    df = pd.concat([df, new_entry], ignore_index=True)\n",
+    "\n",
+    "    # Save the updated DataFrame back to CSV\n",
+    "    df.to_csv(file_path, index=False)\n",
+    "    print(f\"Database updated successfully with ID {new_id}.\")\n",
+    "\n",
+    "\n",
+    "# Example usage\n",
+    "url = \"https://www.channelnewsasia.com/singapore/singapore-airlines-stops-using-iran-airspace-israel-hamas-war-middle-east-escalation-4264011\"\n",
+    "file_path = \"cleaned_data1.csv\"\n",
+    "update_database(file_path, url)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   id                                           Headline  \\\n",
+      "0   1  Singapore Airlines stops using Iranian airspac...   \n",
+      "\n",
+      "                                             Summary           Category  \\\n",
+      "0  Singapore Airlines has stopped using Iranian a...  Aviation Advisory   \n",
+      "\n",
+      "             Datetime                                                URL  \n",
+      "0 2024-04-14 08:58:00  https://www.channelnewsasia.com/singapore/sing...  \n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "\n",
+    "def rank_related_articles(file_path, category):\n",
+    "    # Load the existing data from the CSV file\n",
+    "    try:\n",
+    "        df = pd.read_csv(file_path)\n",
+    "    except FileNotFoundError:\n",
+    "        print(\"Database file not found.\")\n",
+    "        return\n",
+    "\n",
+    "    # Filter articles by the specified category\n",
+    "    filtered_df = df[df[\"Category\"] == category]\n",
+    "\n",
+    "    # Convert 'Datetime' from string to datetime objects for accurate sorting\n",
+    "    filtered_df[\"Datetime\"] = pd.to_datetime(filtered_df[\"Datetime\"])\n",
+    "\n",
+    "    # Sort articles by 'Datetime' in descending order to get the most recent articles first\n",
+    "    sorted_df = filtered_df.sort_values(by=\"Datetime\", ascending=False)\n",
+    "\n",
+    "    # Display the sorted DataFrame\n",
+    "    print(sorted_df[[\"id\", \"Headline\", \"Summary\", \"Category\", \"Datetime\", \"URL\"]])\n",
+    "    return sorted_df\n",
+    "\n",
+    "\n",
+    "# Example usage\n",
+    "file_path = \"cleaned_data1.csv\"\n",
+    "category = \"Aviation Advisory\"\n",
+    "ranked_articles = rank_related_articles(file_path, category)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+----+------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------+---------------------+----------------------------------------------------------------------------------------------------------------------------------------+\n",
+      "| id |                                               Headline                                               |                                                                                                                                                             Summary                                                                                                                                                              |     Category      |      Datetime       |                                                                  URL                                                                   |\n",
+      "+----+------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------+---------------------+----------------------------------------------------------------------------------------------------------------------------------------+\n",
+      "| 1  | Singapore Airlines stops using Iranian airspace as 'precautionary measure' amid Middle East tensions | Singapore Airlines has stopped using Iranian airspace as a precautionary measure amid the escalating tensions in the Middle East. This decision was made after Iran launched over 200 drones and missiles at Israel, following an Israeli strike on an Iranian building in Syria. Other airlines, such as Lufthansa and Austrian | Aviation Advisory | 2024-04-14 08:58:00 | https://www.channelnewsasia.com/singapore/singapore-airlines-stops-using-iran-airspace-israel-hamas-war-middle-east-escalation-4264011 |\n",
+      "+----+------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------+---------------------+----------------------------------------------------------------------------------------------------------------------------------------+\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from tabulate import tabulate\n",
+    "\n",
+    "\n",
+    "def print_ranked_articles_tabulate(file_path, category):\n",
+    "    try:\n",
+    "        df = pd.read_csv(file_path)\n",
+    "        df[\"Datetime\"] = pd.to_datetime(df[\"Datetime\"])\n",
+    "        filtered_df = df[df[\"Category\"] == category]\n",
+    "        sorted_df = filtered_df.sort_values(by=\"Datetime\", ascending=False)\n",
+    "\n",
+    "        # Print DataFrame using tabulate\n",
+    "        print(tabulate(sorted_df, headers=\"keys\", tablefmt=\"pretty\", showindex=False))\n",
+    "    except FileNotFoundError:\n",
+    "        print(\"Database file not found.\")\n",
+    "\n",
+    "\n",
+    "# Example usage\n",
+    "file_path = \"cleaned_data1.csv\"\n",
+    "category = \"Aviation Advisory\"\n",
+    "print_ranked_articles_tabulate(file_path, category)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!rm cleaned_data1.csv"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -1 +1,25 @@
-huggingface_hub==0.22.2
+huggingface_hub==0.22.2
+pandas
+jupyter 
+ipywidgets
+nltk
+openpyxl
+scikit-learn
+matplotlib
+python-dotenv
+openai==0.28
+spacy
+wordcloud
+contractions
+gensim
+scipy==1.12.0
+seaborn
+pyarrow
+pyLDAvis
+tabulate
+black
+black[jupyter]
+lxml_html_clean
+newspaper3k
+gnews
+