add code

app.py

@@ -0,0 +1,576 @@
+# Streamlit
+import streamlit as st 
+import os
+import pandas as pd
+import pickle
+import json
+# Preprocessing
+import re
+import phonlp
+import underthesea
+import re
+
+# Visualize
+import numpy as np
+
+
+# Model
+import tensorflow as tf
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+from tensorflow.keras.models import load_model
+from transformers import AutoModel, AutoTokenizer
+import torch
+import torch.nn as nn
+from sklearn.model_selection import StratifiedKFold
+
+# Evaluate
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
+
+# Set up the Streamlit page
+st.set_page_config(layout='wide')
+
+# Define variables
+PREPROCESSED_DATA = "data/val_data_162k.json"
+device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+N_SPLITS = 5
+skf = StratifiedKFold(n_splits=N_SPLITS)
+
+# Define class names
+class_names = ['Cong nghe', 'Doi song', 'Giai tri', 'Giao duc', 'Khoa hoc', 'Kinh te',
+               'Nha dat', 'Phap luat', 'The gioi', 'The thao', 'Van hoa', 'Xa hoi', 'Xe co']
+
+# Define the NewsClassifier class for BERT-based models
+class NewsClassifier(nn.Module):
+    def __init__(self, n_classes, model_name):
+        super(NewsClassifier, self).__init__()
+        # Load a pre-trained BERT model
+        self.bert = AutoModel.from_pretrained(model_name)
+        # Dropout layer to prevent overfitting
+        self.drop = nn.Dropout(p=0.3)
+        # Fully-connected layer to convert BERT's hidden state to the number of classes to predict
+        self.fc = nn.Linear(self.bert.config.hidden_size, n_classes)
+        # Initialize weights and biases of the fully-connected layer using the normal distribution method
+        nn.init.normal_(self.fc.weight, std=0.02)
+        nn.init.normal_(self.fc.bias, 0)
+
+    def forward(self, input_ids, attention_mask):
+        # Get the output from the BERT model
+        last_hidden_state, output = self.bert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            return_dict=False
+        )
+        # Apply dropout
+        x = self.drop(output)
+        # Pass through the fully-connected layer to get predictions
+        x = self.fc(x)
+        return x
+
+@st.cache_data
+def load_models(model_type):
+    models = None
+    model = None
+
+    if model_type == 'phobertbase':
+        models = []
+        for fold in range(skf.n_splits):
+            model = NewsClassifier(n_classes=13, model_name='vinai/phobert-base-v2')
+            model.to(device)
+            model.load_state_dict(torch.load(f'models/phobert_256_fold{fold+1}.pth', map_location=device))
+            model.eval()
+            models.append(model)
+        tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-base-v2")
+        max_len = 256
+    elif model_type == 'longformer':
+        models = []
+        for fold in range(skf.n_splits):
+            model = NewsClassifier(n_classes=13, model_name='bluenguyen/longformer-phobert-base-4096')
+            model.to(device)
+            model.load_state_dict(torch.load(f'models/phobert_fold{fold+1}.pth', map_location=device))
+            model.eval()
+            models.append(model)
+        tokenizer = AutoTokenizer.from_pretrained("bluenguyen/longformer-phobert-base-4096")
+        max_len = 512
+    elif model_type == 'bilstm_phobertbase':
+        model = load_model("models/bilstm_phobertbase.h5", compile=False)
+        tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-base-v2")
+        max_len = 256
+    else:
+        raise ValueError("Invalid model type specified.")
+    
+    if models is not None:
+        return models, tokenizer, max_len
+    else:
+        return model, tokenizer, max_len
+
+
+
+# Function to preprocess text using PhonLP and Underthesea
+def preprocess_text(text):
+    nlp_model = phonlp.load(save_dir="./phonlp")
+    text = re.sub(r'[^\w\s.]', '', text)
+    sentences = underthesea.sent_tokenize(text)
+    preprocessed_words = []
+    for sentence in sentences:
+        try:
+            word_tokens = underthesea.word_tokenize(sentence, format="text")
+            tags = nlp_model.annotate(word_tokens, batch_size=64)
+            filtered_words = [word.lower() for word, tag in zip(tags[0][0], tags[1][0]) if tag[0] not in ['M', 'X', 'CH'] 
+                              and word not in ["'", ","]]
+            preprocessed_words.extend(filtered_words)
+        except Exception as e:
+            pass
+    return ' '.join(preprocessed_words)
+
+# Function to tokenize text using BERT tokenizer
+def tokenize_text(text, tokenizer, max_len=256):
+    tokenized = tokenizer.encode_plus(
+        text,
+        max_length=max_len,
+        truncation=True,
+        add_special_tokens=True,
+        padding='max_length',
+        return_attention_mask=True,
+        return_token_type_ids=False,
+        return_tensors='pt',
+    )
+    return tokenized['input_ids'], tokenized['attention_mask']
+
+# Function to get BERT features
+def get_bert_features(input_ids, attention_mask, phobert):
+    with torch.no_grad():
+        last_hidden_states = phobert(input_ids=input_ids, attention_mask=attention_mask)
+    features = last_hidden_states[0][:, 0, :].numpy()
+    return features
+
+# Function to predict label using BiLSTM model
+def predict_label(text, tokenizer, phobert, model, class_names, max_len):
+    processed_text = preprocess_text(text)
+    input_ids, attention_mask = tokenize_text(processed_text, tokenizer, max_len)
+    input_ids = torch.tensor(input_ids).to(torch.long).to(device)
+    attention_mask = torch.tensor(attention_mask).to(torch.long).to(device)
+
+    with torch.no_grad():
+        features = get_bert_features(input_ids, attention_mask, phobert)
+        features = np.expand_dims(features, axis=1)
+        prediction = model.predict(features)
+    
+    predicted_label_index = np.argmax(prediction, axis=1)[0]
+    predicted_label = class_names[predicted_label_index]
+
+    confidence_scores = {class_names[i]: float(prediction[0][i]) for i in range(len(prediction[0]))}
+    confidence_df = pd.DataFrame([confidence_scores])
+    confidence_df = confidence_df.melt(var_name='Label', value_name='Confidence')
+
+    return predicted_label, confidence_df
+
+# Function to infer predictions using ensemble of BERT-based models
+def infer(text, tokenizer, models, class_names, max_len):
+    tokenized = tokenizer.encode_plus(
+        text,
+        max_length=max_len,
+        truncation=True,
+        add_special_tokens=True,
+        padding='max_length',
+        return_attention_mask=True,
+        return_token_type_ids=False,
+        return_tensors='pt',
+    )
+    input_ids = tokenized['input_ids'].to(device)
+    attention_mask = tokenized['attention_mask'].to(device)
+
+    with torch.no_grad():
+        all_outputs = []
+        for model in models:
+            model.eval()
+            output = model(input_ids, attention_mask)
+            all_outputs.append(output)
+
+        all_outputs = torch.stack(all_outputs)
+        mean_output = all_outputs.mean(0)
+        _, predicted = torch.max(mean_output, dim=1)
+
+    confidence_scores = mean_output.softmax(dim=1).cpu().numpy()
+    confidence_df = pd.DataFrame([confidence_scores[0]], columns=class_names)
+    confidence_df = confidence_df.melt(var_name='Label', value_name='Confidence')
+    predicted_label = class_names[predicted.item()]
+
+    return confidence_df, predicted_label
+
+# Function to load BERT model and tokenizer
+def load_bert():
+    phobert = AutoModel.from_pretrained("vinai/phobert-base-v2")
+    tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-base-v2", use_fast=False)
+    return phobert, tokenizer
+
+# Function to plot HTML data
+def plot_data(train_html_path, test_html_path, val_html_path):
+    if not (os.path.exists(train_html_path) and os.path.exists(test_html_path) and os.path.exists(val_html_path)):
+        st.error("HTML files not found.")
+        return
+
+    with open(train_html_path, "r", encoding="utf-8") as f_train:
+        train_content = f_train.read()
+        st.components.v1.html(train_content, height=600, scrolling=True)
+
+    with open(test_html_path, "r", encoding="utf-8") as f_test:
+        test_content = f_test.read()
+        st.components.v1.html(test_content, height=600, scrolling=True)
+
+    with open(val_html_path, "r", encoding="utf-8") as f_val:
+        val_content = f_val.read()
+        st.components.v1.html(val_content, height=600, scrolling=True)
+
+
+
+
+
+def main():
+    
+    #st.title("News Classifier App")
+    activities = ["Introduction", "Text Preprocessing", "Feature Extraction", "Train and Evaluate Models", "Prediction"]
+    choice = st.sidebar.selectbox("Choose Activity", activities)
+
+    # Preprocessing data
+    if choice == "Text Preprocessing":
+        st.info("Text Preprocessing")
+        preprocessing_task = ["No Options", "Data Overview", "Process Text Demo", "Load Preprocessed Data"]
+        task_choice = st.selectbox("Choose Task", preprocessing_task)
+        if task_choice == "Data Overview":
+            st.markdown("This dataset consists of Vietnamese news articles collected from various Vietnamese online news portals such as Thanh Nien, VNExpress, BaoMoi, etc. The dataset was originally sourced from a MongoDB dump containing over 20 million articles.")
+            st.markdown("From this large dataset, our team extracted approximately 162,000 articles categorized into 13 distinct categorie and split into training, test and validation sets after preprocessing the data with 70%, 15% and 15% respectively.")
+            st.markdown("Link to dataset: https://github.com/binhvq/news-corpus")
+            st.image("images/sample_data.png", caption="Sample original data", use_column_width=True)
+            summary_df = pd.read_csv("assets/summary_data.csv")
+            st.dataframe(summary_df)
+            train_images = "images/article_by_categories_train_data.html"
+            test_images = "images/article_by_categories_test_data.html"
+            val_images = "images/article_by_categories_val_data.html"
+            plot_data(train_images, test_images, val_images)
+            st.image("images/token_length_distribution.png",caption="Distribution of Token Count per Sentence", use_column_width=True)
+        elif task_choice == "Process Text Demo":
+            st.markdown("**Preprocessing Steps:**")
+            st.markdown("- Standardize Vietnamese words, convert to lower case")
+            st.markdown("- Utilize techniques such as regular expressions to remove unwanted elements: html, links, emails, numbers,...")
+            st.markdown("- Employ a POS tagging tool to determine the grammatical category of each word in the sentence and filter out important components")
+            
+            news_text = st.text_area("Enter Text","Type Here")
+            if st.button("Execute"):
+                st.subheader("Original Text")
+                st.info(news_text)
+                preprocessed_news = preprocess_text(news_text)
+                st.subheader("Preprocessed Text")
+                st.success(preprocessed_news)
+        elif task_choice == "Load Preprocessed Data":
+            df = pd.read_json(PREPROCESSED_DATA, encoding='utf-8', lines=True)
+            st.dataframe(df.head(20), use_container_width=True)
+
+    # Feature Extration
+    if choice == "Feature Extraction":
+        st.info("Feature Extraction")
+        
+        feature_extraction_task = ["No Options", "PhoBert"]
+        task_choice = st.selectbox("Choose Model",feature_extraction_task)
+        if task_choice == "PhoBert":
+            st.markdown("**Feature Extraction Steps:**")
+            st.markdown("- Tokenize using PhoBert's Tokenizer. Note that when tokenizing we will add two special tokens, [CLS] and [SEP] at the beginning and end of the sentence.")
+            st.markdown("- Insert the tokenized text sentence into the model with the attention mask. Attention mask helps the model only focus on words in the sentence and ignore words with additional padding. Added words are marked = 0")
+            st.markdown("- Take the output and take the first output vector (which is in the special token position [CLS]) as a feature for the sentence to train or predict (depending on the phase).")
+            phobert, tokenizer = load_bert()
+            text = st.text_area("Enter Text","Type Here")
+            if st.button("Execute"):
+                st.subheader("Sentence to ids")
+                padded, attention_mask = tokenize_text([text], tokenizer, max_len=256)
+                st.write("Padded Sequence:", padded)
+                st.write("Attention Mask:", attention_mask)
+
+                st.subheader("Vector Embedding of Sentence")
+                v_features = get_vector_embedding(padded, attention_mask, phobert)
+                st.write("Vector Embedding:", v_features)
+            
+
+    if choice == "Prediction":
+         st.info("Predict with new text")
+         
+         all_dl_models = ["No Options", "BiLSTM + phobertbase", "longformer-phobertbase", "phobertbase"]
+         model_choice = st.selectbox("Choose Model", all_dl_models)
+         
+         if model_choice == "BiLSTM + phobertbase":
+                model, tokenizer, max_len, phobert = load_models(model_type="bilstm_phobertbase")
+                news_text = st.text_area("Enter Text", "Type Here")
+                if st.button("Classify"):
+                    st.header("Original Text")
+                    st.info(news_text)
+                    st.header("Predict")
+                    processed_news = preprocess_text(news_text)
+                    predicted_label, confidence_df = predict_label(processed_news, tokenizer, phobert, model, class_names, max_len)
+                    st.subheader("Confidence Per Label")
+                    st.dataframe(confidence_df, use_container_width=True)
+                    st.subheader("Predicted Label")
+                    st.success(predicted_label)
+                
+         if model_choice == "longformer-phobertbase":
+                models, tokenizer, max_len = load_models(model_type="longformer")
+                news_text = st.text_area("Enter Text", "Type Here")
+                if st.button("Classify"):
+                    st.header("Original Text")
+                    st.info(news_text)
+                    st.header("Predict")
+                    df_confidence, predicted_label = infer(news_text, tokenizer, models, class_names, max_len)
+                    st.subheader("Confidence Per Label")
+                    st.dataframe(df_confidence, use_container_width=True)
+                    st.subheader("Predicted Label")
+                    st.success(predicted_label)
+         if model_choice == "phobertbase":
+                models, tokenizer, max_len = load_models(model_type="phobertbase")
+                news_text = st.text_area("Enter Text", "Type Here")
+                if st.button("Classify"):
+                    st.header("Original Text")
+                    st.info(news_text)
+                    st.header("Predict")
+                    df_confidence, predicted_label = infer(news_text, tokenizer, models, class_names, max_len)
+                    st.subheader("Confidence Per Label")
+                    st.dataframe(df_confidence, use_container_width=True)
+                    st.subheader("Predicted Label")
+                    st.success(predicted_label)
+    if choice == "Train and Evaluate Models":
+         st.info("Train and Evaluate Models")
+         training_task = ["No Options", "Model Definitions", "Hyperparameters", "Result of Evaluation"]
+         training_choice = st.selectbox("Choose Options", training_task)
+         if training_choice == "Model Definitions":
+             st.subheader("Longformer-phobertbase Model and Phobertbase Model")
+             # Display model architecture
+             st.code("""
+                    class NewsClassifier(nn.Module):
+                        def __init__(self, n_classes, model_name):
+                            super(NewsClassifier, self).__init__()
+                            # Load a pre-trained BERT model
+                            self.bert = AutoModel.from_pretrained(model_name)
+                            # Dropout layer to prevent overfitting
+                            self.drop = nn.Dropout(p=0.3)
+                            # Fully-connected layer to convert BERT's hidden state to the number of classes to predict
+                            self.fc = nn.Linear(self.bert.config.hidden_size, n_classes)
+                            # Initialize weights and biases of the fully-connected layer using the normal distribution method
+                            nn.init.normal_(self.fc.weight, std=0.02)
+                            nn.init.normal_(self.fc.bias, 0)
+
+                        def forward(self, input_ids, attention_mask):
+                            # Get the output from the BERT model
+                            last_hidden_state, output = self.bert(
+                                input_ids=input_ids,
+                                attention_mask=attention_mask,
+                                return_dict=False
+                            )
+                            # Apply dropout
+                            x = self.drop(output)
+                            # Pass through the fully-connected layer to get predictions
+                            x = self.fc(x)
+                            return x
+                    """, language='python')
+             # Explanation for each layer
+             st.markdown("""
+                - **Dropout Layer**: The dropout layer with a dropout probability of 0.3 helps prevent overfitting during training.
+                - **Fully-connected Layer**: The fully-connected layer (`self.fc`) converts the output of the BERT model to a set of class predictions corresponding to the number of classes. This is achieved by a linear transformation using the BERT hidden size as the input dimension and the number of classes (`n_classes`) as the output dimension.
+                - **Weight Initialization**: The weights and biases of the fully-connected layer are initialized using a normal distribution to facilitate better training.
+                - **Forward Method**: In the forward method, the BERT model is called with the input IDs and attention mask. The output is passed through the dropout layer and then through the fully-connected layer to produce the final predictions.
+                """)
+
+             st.subheader("BiLSTM Model with Phobert feature extraction")
+             # Display model architecture
+             st.image("images/bilstm_phobertbase_summary.png")
+
+             # Explanation for each layer
+             st.markdown("""
+              **Input Layer (input_1):** This layer accepts the input data and prepares it for further processing by the model. 
+              It receives input in the shape (None, 1, 768), where `None` represents the batch size, `1` represents the sequence length (or time steps), and `768` represents the feature dimension. 
+
+              **Bidirectional LSTM Layer (bidirectional):** This layer processes the input sequence bidirectionally, combining information from both past and future states to enhance learning. 
+              It takes input in the shape (None, 1, 768) and outputs (None, 1, 448), reducing the feature dimension to `448`.
+
+              **Dropout Layer (dropout):** Dropout is applied to regularize the model by randomly setting a fraction of input units to zero during training, preventing overfitting. 
+              It takes input in the shape (None, 1, 448) and outputs (None, 1, 448), maintaining the same shape as the input.
+
+              **Second Bidirectional LSTM Layer (bidirectional_1):** Another BiLSTM layer further refines the sequence representation by processing it bidirectionally again. 
+              It takes input in the shape (None, 1, 448) and outputs (None, 1, 288), reducing the feature dimension to `288`.
+
+              **Second Dropout Layer (dropout_1):** Another dropout layer is applied to further regularize the model after the second BiLSTM layer. 
+              It takes input in the shape (None, 288) and outputs (None, 288), maintaining the same shape as the input.
+
+              **Dense Layer (dense):** This fully connected layer applies a non-linear transformation to the extracted features, aiding in capturing complex patterns in the data. 
+              It takes input in the shape (None, 288) and outputs (None, 160), reducing the dimensionality of the data to `160`.
+
+              **Output Dense Layer (dense_1):** The final dense layer with softmax activation produces probabilities across multiple classes, making predictions based on the learned features. 
+              It takes input in the shape (None, 160) and outputs (None, 13), corresponding to the number of classes in the classification task.
+              """)
+         if training_choice == "Hyperparameters":
+             dl_model = ["No Options", "BiLSTM + phobertbase", "longformer-phobertbase and phobertbase"]
+             model_choice = st.selectbox("Choose Model", dl_model)
+             if st.button("Show Result"):
+                if model_choice == "BiLSTM + phobertbase":
+                  st.header("Optuna Hyperparameter Optimization")
+                  st.markdown("""
+                  We used `Optuna` for hyperparameter optimization due to its efficiency and advanced search algorithms. It automates the optimization process, reducing manual effort and improving model performance.
+                  
+                  The study is set to `maximize` the target metric. `TPESampler` is used for efficient and adaptive search, while `HyperbandPruner` stops unpromising trials early to save resources and speed up the optimization process.
+                  """)
+                  
+                  # Explanation of Optuna terms
+                  st.subheader("Understanding Optuna Terms")
+                  st.markdown("""
+                  **Pruner Trials:** These are trials that Optuna has pruned during the optimization process to reduce resource consumption. Pruning helps discard trials that are unlikely to yield better results or are taking too long to converge.
+                  
+                  **Complete Trials:** These trials are successfully completed by Optuna and have provided valid results. Optuna uses these trials to evaluate and select the best hyperparameters based on the defined optimization objective.
+                  
+                  **Failed Trials:** Trials that have encountered errors or failed to complete due to technical issues or improper configurations. These trials do not contribute valid results to the optimization process.
+                  """)
+                  
+                  # Load and display trial information
+                  trials = pd.read_csv("assets/study_bilstm_256_trials.csv")
+                  st.subheader("Number of Completed Trials out of 100 trials")
+                  st.dataframe(trials.style.format(precision=6), height=600, hide_index=True, use_container_width=True)
+                  
+                  # Load best hyperparameters and display
+                  with open("hyperparameters/BiLSTM_phobertbase.json", 'r', encoding='utf-8') as file:
+                      bilstm_phobertbase_best_param = json.load(file)
+                  bilstm_phobertbase_best_param_df = pd.DataFrame([bilstm_phobertbase_best_param])
+                  st.subheader("Best Hyperparameters")
+                  st.dataframe(bilstm_phobertbase_best_param_df.style.format(precision=6), hide_index=True, use_container_width=True)
+                  
+                  # Display optimization history plot with title
+                  st.subheader("Optimization History Plot")
+                  with open("images/study_bilstm_phobertbase_optimize_history.html", "r", encoding="utf-8") as f:
+                      content = f.read()
+                      st.components.v1.html(content, height=600, scrolling=True)
+                        
+                if model_choice == "longformer-phobertbase and phobertbase":
+                        
+                        with open("./hyperparameters/phobertbase.json", 'r', encoding='utf-8') as file:
+                            param = json.load(file)
+                        param_df = pd.DataFrame([param])
+                        st.subheader("Best Hyperparamters")
+                        st.dataframe(param_df.style.format(precision=6), hide_index=True, use_container_width=True)
+         if training_choice == "Result of Evaluation":
+            st.markdown("To evaluate the performance of our models, we used several key metrics:")
+            st.markdown("1. **Accuracy**: The proportion of correctly classified instances among the total instances.")
+            st.markdown("2. **Precision**: The proportion of true positives among all positive predictions, indicating the accuracy of the positive predictions.")
+            st.markdown("3. **Recall**: The proportion of true positives among all actual positives, reflecting the model's ability to capture all relevant instances.")
+            st.markdown("4. **F1-score**: The harmonic mean of precision and recall, providing a balance between the two metrics.")
+            st.markdown("5. **Confusion Matrix**: A table that displays the true positives, true negatives, false positives, and false negatives, used to evaluate the overall performance and error types of the model.")
+            task = ["No Options", "Overall", "Evaluate per Label"]
+            task_choice = st.selectbox("Choose Options", task)
+            if task_choice == "Overall":
+              result = pd.read_csv("assets/model_results.csv")
+              st.dataframe(result, height=150, hide_index=True, use_container_width=True)
+            if task_choice == "Evaluate per Label":
+              st.subheader("Confusion Matrix Comparison")
+              col1, col2, col3 = st.columns(3)  
+              
+              with col1:
+                  st.image("images/confusion_matrix_bilstm_phobertbase.png", caption="BiLSTM with PhoBert feature extraction", use_column_width=True)
+              
+              with col2:
+                  st.image("images/confusion_matrix_phobertbase.png", caption="phobertbase", use_column_width=True)
+              
+              with col3:
+                  st.image("images/confusion_matrix_longformer.png", caption="longformer-phobertbase", use_column_width=True)
+
+              st.subheader("Classification Report Comparison")
+              col4, col5, col6 = st.columns(3)  
+              
+              with col4:
+                  st.markdown("**BiLSTM with PhoBert feature extraction**")
+                  bilstm_report = pd.read_csv("assets/classification_report_bilstm_phobertbase.csv")
+                  st.dataframe(bilstm_report, height=600, hide_index=True, use_container_width=True)
+              
+              with col5:
+                  st.markdown("**phobertbase**")
+                  phobertbase_report = pd.read_csv("assets/classification_report_phobertbase.csv")
+                  st.dataframe(phobertbase_report, height=600, hide_index=True, use_container_width=True)
+              
+              with col6:
+                  st.markdown("**longformer-phobertbase**")
+                  longformer_report = pd.read_csv("assets/classification_report_longformer.csv")
+                  st.dataframe(longformer_report, height=600, hide_index=True, use_container_width=True)
+    if choice == "Introduction":
+      st.markdown(
+        """
+        <style>
+            .title {
+                font-size: 35px;
+                font-weight: bold;
+                text-align: center;
+                color: #2c3e50;
+                margin-top: 0px;
+            }
+            .university {
+                font-size: 30px;
+                font-weight: bold;
+                text-align: center;
+                color: #34495e;
+                margin-top: 0px;
+            }
+            .faculty {
+                font-size: 30px;
+                font-weight: bold;
+                text-align: center;
+                color: #34495e;
+                margin-bottom: 20px;
+            }
+            .subtitle {
+                font-size: 24px;
+                font-weight: bold;
+                text-align: center;
+                color: #34495e;
+                margin-bottom: 10px;
+            }
+            .student-info, .instructor-info {
+                font-size: 18px;
+                text-align: center;
+                color: #7f8c8d;
+                margin: 10px 20px;
+            }
+            .note {
+                font-size: 16px;
+                color: #95a5a6;
+                margin-top: 20px;
+                font-style: italic;
+                text-align: left;
+                margin: 20px;
+            }
+            
+        </style>
+        """,
+        unsafe_allow_html=True
+    )
+
+      st.markdown('<div class="university">HCMC University of Technology and Education</div>', unsafe_allow_html=True)
+      st.markdown('<div class="faculty">Faculty of Information Technology</div>', unsafe_allow_html=True)
+
+      # Use Streamlit's st.image to display the logos
+      left_co, cent_co,last_co, t, f, s, s = st.columns(7)
+      with t:
+          st.image("images/logo.png")
+
+      st.markdown('<div class="subtitle">Graduation Thesis</div>', unsafe_allow_html=True)
+      st.markdown('<div class="title">Vietnamese News and Articles Classification using PhoBERT</div>', unsafe_allow_html=True)
+
+      st.markdown(
+        """
+        <div class="student-info">
+            <p>Nguyen Thi Dieu Hien - 20133040</p>
+            <p>Bui Tan Dat - 20133033</p>
+        </div>
+
+        <div class="instructor-info">
+            <p>Instructor: PhD. Nguyen Thanh Son</p>
+        </div>
+
+        <div class="note">
+            Note: This is an interactive web application to demonstrate various tasks related to news classification using deep learning models. Choose an activity from the sidebar to get started.
+        </div>
+        """,
+        unsafe_allow_html=True
+    )
+
+if __name__ == '__main__':
+	main()

feature_extraction.py

@@ -0,0 +1,116 @@
+import pickle
+from transformers import AutoModel, AutoTokenizer
+from tqdm import tqdm
+import pandas as pd
+import torch
+import numpy as np
+from pyspark.sql import SparkSession
+import time
+
+# Paths to JSON data files
+TRAIN_DATA = "data/train_data_162k.json"
+TEST_DATA = "data/test_data_162k.json"
+VAL_DATA = "data/val_data_162k.json"
+
+# Function to load BERT model and tokenizer
+def load_bert():
+    v_phobert = AutoModel.from_pretrained("vinai/phobert-base-v2")
+    v_tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-base-v2", use_fast=False)
+    return v_phobert, v_tokenizer
+
+# Load BERT model and tokenizer
+phobert, tokenizer = load_bert()
+print("Load model done!")
+
+# Initialize SparkSession
+spark = SparkSession.builder \
+    .appName("Feature Extraction") \
+    .master("local[*]") \
+    .config("spark.executor.memory", "50g") \
+    .config("spark.executor.instances", "4") \
+    .config("spark.executor.cores", "12") \
+    .config("spark.memory.offHeap.enabled", True) \
+    .config("spark.driver.memory", "50g") \
+    .config("spark.memory.offHeap.size", "16g") \
+    .config("spark.ui.showConsoleProgress", False) \
+    .config("spark.driver.maxResultSize", "16g") \
+    .config("spark.log.level", "ERROR") \
+    .getOrCreate()
+
+# Load JSON data into Spark DataFrames
+train_data = spark.read.json(TRAIN_DATA)
+test_data = spark.read.json(TEST_DATA)
+val_data = spark.read.json(VAL_DATA)
+print("Load data done!")
+
+# Function to extract BERT features from text
+def make_bert_features(v_text):
+    v_tokenized = []
+    max_len = 256  # Maximum sequence length
+
+    # Use tqdm to display progress
+    for i_text in v_text:
+        # Tokenize using BERT tokenizer
+        line = tokenizer.encode(i_text, truncation=True)
+        v_tokenized.append(line)
+
+    # Pad sequences to ensure consistent length
+    padded = []
+    for i in v_tokenized:
+        if len(i) < max_len:
+            padded.append(i + [1] * (max_len - len(i)))  # Padding with 1s
+        else:
+            padded.append(i[:max_len])  # Truncate if sequence is too long
+
+    padded = np.array(padded)
+
+    # Create attention mask
+    attention_mask = np.where(padded == 1, 0, 1)
+
+    # Convert to PyTorch tensors
+    padded = torch.tensor(padded).to(torch.long)
+    attention_mask = torch.tensor(attention_mask)
+
+    # Obtain features from BERT
+    with torch.no_grad():
+        last_hidden_states = phobert(input_ids=padded, attention_mask=attention_mask)
+
+    v_features = last_hidden_states[0][:, 0, :].numpy()
+    print(v_features.shape)
+    return v_features
+
+# Extract BERT features for train, test, and validation datasets
+train_features = train_data.select("processed_content").rdd.map(make_bert_features)
+test_features = test_data.select("processed_content").rdd.map(make_bert_features)
+val_features = val_data.select("processed_content").rdd.map(make_bert_features)
+
+# Convert category column to lists
+category_list_train = train_data.select("category").rdd.flatMap(lambda x: x).collect()
+category_list_test = test_data.select("category").rdd.flatMap(lambda x: x).collect()
+category_list_val = val_data.select("category").rdd.flatMap(lambda x: x).collect()
+
+# Convert to one-hot encoding using pd.get_dummies()
+y_train = pd.get_dummies(category_list_train)
+y_test = pd.get_dummies(category_list_test)
+y_val = pd.get_dummies(category_list_val)
+
+# Save data to file using pickle
+start_time = time.time()
+print("Saving to file")
+data_dict = {
+    'X_train': train_features.collect(),
+    'X_test': test_features.collect(),
+    'X_val': val_features.collect(),
+    'y_train': y_train,
+    'y_test': y_test,
+    'y_val': y_val
+}
+
+# Save dictionary to pickle file
+with open('data/features_162k_phobertbase_v2.pkl', 'wb') as f:
+    pickle.dump(data_dict, f)
+
+end_time = time.time()
+duration = end_time - start_time
+print(f'Total feature extraction time: {duration:.2f} seconds')
+print("Done!")

finetune_PhoBert.py

@@ -0,0 +1,273 @@
+import torch
+import numpy as np
+import pandas as pd
+
+from sklearn.model_selection import StratifiedKFold
+from sklearn.metrics import classification_report, confusion_matrix
+
+import torch.nn as nn
+from torch.optim import AdamW
+from torch.utils.data import Dataset, DataLoader
+
+from transformers import get_linear_schedule_with_warmup, AutoTokenizer, AutoModel, logging
+
+import warnings
+import time
+import pickle
+warnings.filterwarnings("ignore")
+
+logging.set_verbosity_error()
+
+# Function to set seed for reproducibility
+def seed_everything(seed_value):
+    np.random.seed(seed_value)  # Set seed for numpy random numbers
+    torch.manual_seed(seed_value)  # Set seed for PyTorch random numbers
+
+    if torch.cuda.is_available():  # If CUDA is available, set CUDA seed
+        torch.cuda.manual_seed(seed_value)
+        torch.cuda.manual_seed_all(seed_value)
+        torch.backends.cudnn.deterministic = True  # Ensure deterministic behavior
+        torch.backends.cudnn.benchmark = True  # Improve performance by allowing cudnn benchmarking
+
+seed_everything(86)  # Set seed value for reproducibility
+
+model_name = "bluenguyen/longformer-phobert-base-4096"  # Pretrained model name
+max_len = 512  # Maximum sequence length for tokenizer (512, but can use 256 if phobertbase)
+n_classes = 13  # Number of output classes
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)  # Load tokenizer
+
+device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')  # Set device to GPU if available, otherwise CPU
+EPOCHS = 5  # Number of training epochs
+N_SPLITS = 5  # Number of folds for cross-validation
+
+TRAIN_PATH = "data/train_data_162k.json"  
+TEST_PATH = "data/test_data_162k.json"  
+VAL_PATH = "data/val_data_162k.json"  
+
+# Function to read data from JSON file
+def get_data(path):
+    df = pd.read_json(path, lines=True)
+    return df
+
+# Read the data from JSON files
+train_df = get_data(TRAIN_PATH)
+test_df = get_data(TEST_PATH)
+valid_df = get_data(VAL_PATH)
+
+# Combine train and validation data
+train_df = pd.concat([train_df, valid_df], ignore_index=True)
+
+# Apply StratifiedKFold
+skf = StratifiedKFold(n_splits=N_SPLITS)
+for fold, (_, val_) in enumerate(skf.split(X=train_df, y=train_df.category)):
+    train_df.loc[val_, "kfold"] = fold
+
+class NewsDataset(Dataset):
+    def __init__(self, df, tokenizer, max_len):
+        self.df = df
+        self.max_len = max_len
+        self.tokenizer = tokenizer
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, index):
+        """
+        To customize dataset, inherit from Dataset class and implement
+        __len__ & __getitem__
+        __getitem__ should return
+            data:
+                input_ids
+                attention_masks
+                text
+                targets
+        """
+        row = self.df.iloc[index]
+        text, label = self.get_input_data(row)
+
+        # Encode_plus will:
+        # (1) split text into token
+        # (2) Add the '[CLS]' and '[SEP]' token to the start and end
+        # (3) Truncate/Pad sentence to max length
+        # (4) Map token to their IDS
+        # (5) Create attention mask
+        # (6) Return a dictionary of outputs
+        encoding = self.tokenizer.encode_plus(
+            text,
+            truncation=True,
+            add_special_tokens=True,
+            max_length=self.max_len,
+            padding='max_length',
+            return_attention_mask=True,
+            return_token_type_ids=False,
+            return_tensors='pt',
+        )
+
+        return {
+            'text': text,
+            'input_ids': encoding['input_ids'].flatten(),
+            'attention_masks': encoding['attention_mask'].flatten(),
+            'targets': torch.tensor(label, dtype=torch.long),
+        }
+
+
+    def labelencoder(self, text):
+        label_map = {
+            'Cong nghe': 0, 'Doi song': 1, 'Giai tri': 2, 'Giao duc': 3, 'Khoa hoc': 4,
+            'Kinh te': 5, 'Nha dat': 6, 'Phap luat': 7, 'The gioi': 8, 'The thao': 9,
+            'Van hoa': 10, 'Xa hoi': 11, 'Xe co': 12
+        }
+        return label_map.get(text, -1)
+
+    def get_input_data(self, row):
+        text = row['processed_content']
+        label = self.labelencoder(row['category'])
+        return text, label
+
+class NewsClassifier(nn.Module):
+    def __init__(self, n_classes, model_name):
+        super(NewsClassifier, self).__init__()
+        # Load a pre-trained BERT model
+        self.bert = AutoModel.from_pretrained(model_name)
+        # Dropout layer to prevent overfitting
+        self.drop = nn.Dropout(p=0.3)
+        # Fully-connected layer to convert BERT's hidden state to the number of classes to predict
+        self.fc = nn.Linear(self.bert.config.hidden_size, n_classes)
+        # Initialize weights and biases of the fully-connected layer using the normal distribution method
+        nn.init.normal_(self.fc.weight, std=0.02)
+        nn.init.normal_(self.fc.bias, 0)
+
+    def forward(self, input_ids, attention_mask):
+        # Get the output from the BERT model
+        last_hidden_state, output = self.bert(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            return_dict=False
+        )
+        # Apply dropout
+        x = self.drop(output)
+        # Pass through the fully-connected layer to get predictions
+        x = self.fc(x)
+        return x
+
+def prepare_loaders(df, fold):
+    df_train = df[df.kfold != fold].reset_index(drop=True)
+    df_valid = df[df.kfold == fold].reset_index(drop=True)
+
+    train_dataset = NewsDataset(df_train, tokenizer, max_len)
+    valid_dataset = NewsDataset(df_valid, tokenizer, max_len)
+
+    train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=2)
+    valid_loader = DataLoader(valid_dataset, batch_size=16, shuffle=True, num_workers=2)
+
+    return train_loader, valid_loader
+
+# Function to train the model for one epoch
+def train(model, criterion, optimizer, train_loader, lr_scheduler):
+    model.train()  # Set the model to training mode
+    losses = []  # List to store losses during training
+    correct = 0  # Variable to store number of correct predictions
+
+    # Iterate over batches in the training data loader
+    for batch_idx, data in enumerate(train_loader):
+        input_ids = data['input_ids'].to(device)  # Move input_ids to GPU/CPU
+        attention_mask = data['attention_masks'].to(device)  # Move attention_mask to GPU/CPU
+        targets = data['targets'].to(device)  # Move targets to GPU/CPU
+
+        optimizer.zero_grad()  # Clear gradients from previous iteration
+        outputs = model(  # Forward pass through the model
+            input_ids=input_ids,
+            attention_mask=attention_mask
+        )
+
+        loss = criterion(outputs, targets)  # Calculate the loss
+        _, pred = torch.max(outputs, dim=1)  # Get the predicted labels
+
+        correct += torch.sum(pred == targets)  # Count correct predictions
+        losses.append(loss.item())  # Append the current loss value to losses list
+        loss.backward()  # Backpropagation: compute gradients
+        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # Clip gradients to prevent exploding gradients
+        optimizer.step()  # Update model parameters
+        lr_scheduler.step()  # Update learning rate scheduler
+
+        # Print training progress every 1000 batches
+        if batch_idx % 1000 == 0:
+            print(f'Batch {batch_idx}/{len(train_loader)} - Loss: {loss.item():.4f}, Accuracy: {correct.double() / ((batch_idx + 1) * train_loader.batch_size):.4f}')
+
+    train_accuracy = correct.double() / len(train_loader.dataset)  # Calculate training accuracy
+    avg_loss = np.mean(losses)  # Calculate average loss
+    print(f'Train Accuracy: {train_accuracy:.4f} Loss: {avg_loss:.4f}')
+
+# Function to evaluate the model
+def eval(model, criterion, valid_loader, test_loader=None):
+    model.eval()  # Set the model to evaluation mode
+    losses = []  # List to store losses during evaluation
+    correct = 0  # Variable to store number of correct predictions
+
+    with torch.no_grad():  # Disable gradient calculation for evaluation
+        data_loader = test_loader if test_loader else valid_loader  # Choose between test and validation data loader
+        for batch_idx, data in enumerate(data_loader):
+            input_ids = data['input_ids'].to(device)  # Move input_ids to GPU/CPU
+            attention_mask = data['attention_masks'].to(device)  # Move attention_mask to GPU/CPU
+            targets = data['targets'].to(device)  # Move targets to GPU/CPU
+
+            outputs = model(  # Forward pass through the model
+                input_ids=input_ids,
+                attention_mask=attention_mask
+            )
+
+            loss = criterion(outputs, targets)  # Calculate the loss
+            _, pred = torch.max(outputs, dim=1)  # Get the predicted labels
+
+            correct += torch.sum(pred == targets)  # Count correct predictions
+            losses.append(loss.item())  # Append the current loss value to losses list
+
+    dataset_size = len(test_loader.dataset) if test_loader else len(valid_loader.dataset)  # Determine dataset size
+    accuracy = correct.double() / dataset_size  # Calculate accuracy
+    avg_loss = np.mean(losses)  # Calculate average loss
+
+    # Print evaluation results (either test or validation)
+    if test_loader:
+        print(f'Test Accuracy: {accuracy:.4f} Loss: {avg_loss:.4f}')
+    else:
+        print(f'Valid Accuracy: {accuracy:.4f} Loss: {avg_loss:.4f}')
+
+    return accuracy  # Return accuracy for further analysis or logging
+
+total_start_time = time.time()
+
+# Main training loop
+for fold in range(skf.n_splits):
+    print(f'----------- Fold: {fold + 1} ------------------')
+    train_loader, valid_loader = prepare_loaders(train_df, fold=fold)
+    model = NewsClassifier(n_classes=13).to(device)
+    criterion = nn.CrossEntropyLoss()
+    optimizer = AdamW(model.parameters(), lr=2e-5)
+
+    lr_scheduler = get_linear_schedule_with_warmup(
+        optimizer,
+        num_warmup_steps=0,
+        num_training_steps=len(train_loader) * EPOCHS
+    )
+    best_acc = 0
+
+    for epoch in range(EPOCHS):
+        print(f'Epoch {epoch + 1}/{EPOCHS}')
+        print('-' * 30)
+
+        train(model, criterion, optimizer, train_loader, lr_scheduler)
+        val_acc = eval(model, criterion, valid_loader)
+
+        if val_acc > best_acc:
+            torch.save(model.state_dict(), f'phobert_fold{fold + 1}.pth')
+            best_acc = val_acc
+        print(f'Best Accuracy for Fold {fold + 1}: {best_acc:.4f}')
+        print()
+    print(f'Finished Fold {fold + 1} with Best Accuracy: {best_acc:.4f}')
+    print('--------------------------------------')
+
+
+total_end_time = time.time()
+
+total_duration = total_end_time - total_start_time
+print(f'Total training time: {total_duration:.2f} seconds')

optimize_bilstm.py

@@ -0,0 +1,109 @@
+import optuna
+import numpy as np
+import pandas as pd
+import json
+import tensorflow as tf
+from tensorflow.compat.v1 import ConfigProto
+from tensorflow.compat.v1 import InteractiveSession
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Input, Bidirectional, LSTM, Dropout, Dense
+from tensorflow.keras.optimizers import Adam
+from optuna.integration import TFKerasPruningCallback
+import pickle
+from optuna.visualization import plot_optimization_history
+import optuna.visualization as ov
+from optuna.trial import TrialState
+
+config = ConfigProto()
+config.gpu_options.allow_growth = True
+session = InteractiveSession(config=config)
+
+"""### **Load data**"""
+
+# Load dữ liệu từ file pickle
+with open('data/features_162k_phobertbase.pkl', 'rb') as f:
+    data_dict = pickle.load(f)
+
+# Trích xuất các đặc trưng và nhãn từ dictionary
+X_train = np.array(data_dict['X_train'])
+X_val = np.array(data_dict['X_val'])
+X_test = np.array(data_dict['X_test'])
+y_train = data_dict['y_train']
+y_val = data_dict['y_val']
+y_test = data_dict['y_test']
+
+y_train = y_train.values.astype(int)
+y_test = y_test.values.astype(int)
+y_val = y_val.values.astype(int)
+
+"""##**Build Model**"""
+
+# Define the BiLSTM model architecture
+def build_bilstm_model(lstm_units_1, lstm_units_2, dense_units, dropout_rate, learning_rate):
+    model = Sequential()
+    model.add(Input(shape=(X_train.shape[1], X_train.shape[2])))
+    # Lớp LSTM 1 với dropout
+    model.add(Bidirectional(LSTM(lstm_units_1, return_sequences=True)))
+    model.add(Dropout(dropout_rate))
+    # Lớp LSTM 2 với dropout
+    model.add(Bidirectional(LSTM(lstm_units_2, return_sequences=False)))
+    model.add(Dropout(dropout_rate))
+    # Lớp Dense với dropout và kích hoạt ReLU
+    model.add(Dense(dense_units, activation='relu'))
+    model.add(Dropout(dropout_rate))
+    # Lớp Dense cuối cùng với kích hoạt softmax
+    model.add(Dense(y_train.shape[1], activation='softmax'))
+    # Sử dụng tối ưu hóa Adam với learning rate được truyền vào
+    optimizer = Adam(learning_rate=learning_rate)
+    # Biên soạn mô hình
+    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
+
+    return model
+
+"""##**Create objective**"""
+
+# Define the objective function for optimization
+def objective_bilstm(trial):
+    lstm_units_1 = trial.suggest_int('lstm_units_1', 64, 512, step=32)
+    lstm_units_2 = trial.suggest_int('lstm_units_2', lstm_units_1//2, lstm_units_1, step=32)
+    dense_units = trial.suggest_int('dense_units', 64, 512, step=32)
+    dropout_rate = trial.suggest_float('dropout_rate', 0.2, 0.5, step=0.1)
+    learning_rate = trial.suggest_float('learning_rate', 1e-5, 1e-2, log=True)
+    epochs = trial.suggest_int('epochs', 10, 30, step=10)
+    batch_size = trial.suggest_int('batch_size', 64, 256, step=32)
+
+    print(f"Trying hyperparameters: lstm_units_1={lstm_units_1}, lstm_units_2={lstm_units_2}, dense_units={dense_units}, "
+          f"dropout_rate={dropout_rate}, learning_rate={learning_rate}, batch_size={batch_size}")
+
+    model = build_bilstm_model(lstm_units_1, lstm_units_2, dense_units, dropout_rate, learning_rate)
+
+    model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,
+              validation_data=(X_val, y_val), callbacks=[TFKerasPruningCallback(trial, "val_loss")], verbose=1)
+
+    _, accuracy = model.evaluate(X_test, y_test, verbose=0)
+
+    return accuracy
+
+"""##**Study to find hyperparameters**"""
+
+# Create an Optuna study for optimization
+study_bilstm = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(), pruner=optuna.pruners.HyperbandPruner())
+study_bilstm.optimize(lambda trial: objective_bilstm(trial), n_trials=100)
+
+# Save completed trials to a CSV file
+complete_trials = study_bilstm.trials_dataframe()[study_bilstm.trials_dataframe()['state'] == 'COMPLETE']
+complete_trials.to_csv("assets/study_bilstm_256_trials.csv", index=False)
+
+# Extract the best hyperparameters
+best_hyperparameters_bilstm = study_bilstm.best_trial.params
+
+# Save the best hyperparameters to a JSON file
+with open('hyperparameters/BiLSTM_phobertbase.json', 'w') as file:
+    json.dump(best_hyperparameters_bilstm, file)
+
+plot_optimization_history(study_bilstm)
+
+html_file_path = "images/study_bilstm_phobertbase_optimize_history.html"
+# Plot and save the optimization history plot as an HTML file
+ov.plot_optimization_history(study_bilstm).write_html(html_file_path)
+plot_optimization_history(study_bilstm)

preprocess_data.py

@@ -0,0 +1,74 @@
+import phonlp
+import underthesea
+from pyspark.sql import SparkSession
+from pyspark.sql.functions import udf, StringType
+import re
+
+# Paths to original and processed data files
+ORIGINAL_DATA = "./data/news_v2/news_v2.json"
+PROCESSED_DATA = "./data/processed_data/final_data.json"
+
+# Load NLP model
+nlp_model = phonlp.load(save_dir="./phonlp")
+
+# Initialize SparkSession
+spark = SparkSession.builder \
+    .appName("Preprocessing") \
+    .master("local[*]") \
+    .config("spark.executor.memory", "8g") \
+    .config("spark.executor.instances", "64") \
+    .config("spark.executor.cores", "1") \
+    .config("spark.memory.offHeap.enabled", True) \
+    .config("spark.driver.memory", "50g") \
+    .config("spark.memory.offHeap.size", "16g") \
+    .config("spark.ui.showConsoleProgress", False) \
+    .config("spark.driver.maxResultSize", "8g") \
+    .config("spark.log.level", "ERROR") \
+    .getOrCreate()
+
+print("Loading data....")
+df = spark.read.json(ORIGINAL_DATA)
+
+# Function to preprocess text
+def preprocess_text(text):
+    text = re.sub(r'[^\w\s.]', '', text)  # Remove special characters
+    # Tokenize text into sentences
+    sentences = underthesea.sent_tokenize(text)
+    
+    # List to store preprocessed words
+    preprocessed_words = []
+    
+    # Iterate through each sentence
+    for sentence in sentences:
+        try:
+            word_tokens = underthesea.word_tokenize(sentence, format="text")
+            # Tokenize words and perform POS tagging
+            tags = nlp_model.annotate(word_tokens, batch_size=64)
+            
+            # Filter words based on POS tags
+            filtered_words = [word.lower() for word, tag in zip(tags[0][0], tags[1][0]) if tag[0] not in ['M', 'X', 'CH'] 
+                              and word not in ["'", ","]]
+            
+            # Append filtered words to the result list
+            preprocessed_words.extend(filtered_words)
+        except Exception as e:
+            pass
+    
+    # Convert list of words to string and return
+    return ' '.join(preprocessed_words)
+
+# Register preprocess_text function as a Spark UDF
+preprocess_udf = udf(lambda text: preprocess_text(text), StringType())
+
+# Add "processed_content" column to DataFrame by applying preprocess_text function to "content" column
+df_processed = df.withColumn("processed_content", preprocess_udf(df["content"]))
+
+# Select "processed_content" and "category" columns from DataFrame
+selected_columns = ["processed_content", "category"]
+df_selected = df_processed.select(selected_columns)
+
+# Number of partitions
+num_partitions = 1024 
+
+# Write DataFrame with specified number of partitions
+df_selected.repartition(num_partitions).coalesce(1).write.json(PROCESSED_DATA)

requirements.txt

@@ -0,0 +1,516 @@
+absl-py==1.4.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+alabaster==0.7.16
+albumentations==1.3.1
+altair==4.2.2
+annotated-types==0.7.0
+anyio==3.7.1
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+array_record==0.5.1
+arviz==0.15.1
+astropy==5.3.4
+astunparse==1.6.3
+async-timeout==4.0.3
+atpublic==4.1.0
+attrs==23.2.0
+audioread==3.0.1
+autograd==1.6.2
+Babel==2.15.0
+backcall==0.2.0
+beautifulsoup4==4.12.3
+bidict==0.23.1
+bigframes==1.10.0
+bleach==6.1.0
+blinker==1.4
+blis==0.7.11
+blosc2==2.0.0
+bokeh==3.3.4
+bqplot==0.12.43
+branca==0.7.2
+build==1.2.1
+CacheControl==0.14.0
+cachetools==5.3.3
+catalogue==2.0.10
+certifi==2024.6.2
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+chex==0.1.86
+click==8.1.7
+click-plugins==1.1.1
+cligj==0.7.2
+cloudpathlib==0.18.1
+cloudpickle==2.2.1
+cmake==3.27.9
+cmdstanpy==1.2.4
+colorcet==3.1.0
+colorlover==0.3.0
+colour==0.1.5
+community==1.0.0b1
+confection==0.1.5
+cons==0.4.6
+contextlib2==21.6.0
+contourpy==1.2.1
+cryptography==42.0.8
+cuda-python==12.2.1
+cudf-cu12==24.4.1
+cufflinks==0.17.3
+cupy-cuda12x==12.2.0
+cvxopt==1.3.2
+cvxpy==1.3.4
+cycler==0.12.1
+cymem==2.0.8
+Cython==3.0.10
+dask==2023.8.1
+datascience==0.17.6
+db-dtypes==1.2.0
+dbus-python==1.2.18
+debugpy==1.6.6
+decorator==4.4.2
+defusedxml==0.7.1
+distributed==2023.8.1
+distro==1.7.0
+dlib==19.24.4
+dm-tree==0.1.8
+docstring_parser==0.16
+docutils==0.18.1
+dopamine_rl==4.0.9
+duckdb==0.10.3
+earthengine-api==0.1.409
+easydict==1.13
+ecos==2.0.14
+editdistance==0.6.2
+eerepr==0.0.4
+en-core-web-sm==3.7.1
+entrypoints==0.4
+et-xmlfile==1.1.0
+etils==1.7.0
+etuples==0.3.9
+exceptiongroup==1.2.1
+fastai==2.7.15
+fastcore==1.5.48
+fastdownload==0.0.7
+fastjsonschema==2.20.0
+fastprogress==1.0.3
+fastrlock==0.8.2
+filelock==3.15.4
+fiona==1.9.6
+firebase-admin==5.3.0
+Flask==2.2.5
+flatbuffers==24.3.25
+flax==0.8.4
+folium==0.14.0
+fonttools==4.53.0
+frozendict==2.4.4
+frozenlist==1.4.1
+fsspec==2023.6.0
+future==0.18.3
+gast==0.6.0
+gcsfs==2023.6.0
+GDAL==3.6.4
+gdown==5.1.0
+geemap==0.32.1
+gensim==4.3.2
+geocoder==1.38.1
+geographiclib==2.0
+geopandas==0.13.2
+geopy==2.3.0
+gin-config==0.5.0
+gitdb==4.0.11
+GitPython==3.1.43
+glob2==0.7
+google==2.0.3
+google-ai-generativelanguage==0.6.4
+google-api-core==2.16.2
+google-api-python-client==2.84.0
+google-auth==2.27.0
+google-auth-httplib2==0.1.1
+google-auth-oauthlib==1.2.0
+google-cloud-aiplatform==1.57.0
+google-cloud-bigquery==3.21.0
+google-cloud-bigquery-connection==1.12.1
+google-cloud-bigquery-storage==2.25.0
+google-cloud-bigtable==2.24.0
+google-cloud-core==2.3.3
+google-cloud-datastore==2.15.2
+google-cloud-firestore==2.11.1
+google-cloud-functions==1.13.3
+google-cloud-iam==2.15.0
+google-cloud-language==2.13.3
+google-cloud-resource-manager==1.12.3
+google-cloud-storage==2.8.0
+google-cloud-translate==3.11.3
+google-colab==1.0.0
+google-crc32c==1.5.0
+google-generativeai==0.5.4
+google-pasta==0.2.0
+google-resumable-media==2.7.1
+googleapis-common-protos==1.63.2
+googledrivedownloader==0.4
+graphviz==0.20.3
+greenlet==3.0.3
+grpc-google-iam-v1==0.13.1
+grpcio==1.64.1
+grpcio-status==1.48.2
+gspread==6.0.2
+gspread-dataframe==3.3.1
+gym==0.25.2
+gym-notices==0.0.8
+h5netcdf==1.3.0
+h5py==3.9.0
+holidays==0.51
+holoviews==1.17.1
+html5lib==1.1
+httpimport==1.3.1
+httplib2==0.22.0
+huggingface-hub==0.23.4
+humanize==4.7.0
+hyperopt==0.2.7
+ibis-framework==8.0.0
+idna==3.7
+imageio==2.31.6
+imageio-ffmpeg==0.5.1
+imagesize==1.4.1
+imbalanced-learn==0.10.1
+imgaug==0.4.0
+immutabledict==4.2.0
+importlib_metadata==8.0.0
+importlib_resources==6.4.0
+imutils==0.5.4
+inflect==7.0.0
+iniconfig==2.0.0
+intel-openmp==2023.2.4
+ipyevents==2.0.2
+ipyfilechooser==0.6.0
+ipykernel==5.5.6
+ipyleaflet==0.18.2
+ipyparallel==8.8.0
+ipython==7.34.0
+ipython-genutils==0.2.0
+ipython-sql==0.5.0
+ipytree==0.2.2
+ipywidgets==7.7.1
+itsdangerous==2.2.0
+jax==0.4.26
+jaxlib==0.4.26+cuda12.cudnn89
+jeepney==0.7.1
+jellyfish==1.0.4
+jieba==0.42.1
+Jinja2==3.1.4
+joblib==1.4.2
+jsonpickle==3.2.2
+jsonschema==4.19.2
+jsonschema-specifications==2023.12.1
+jupyter-client==6.1.12
+jupyter-console==6.1.0
+jupyter_core==5.7.2
+jupyter-server==1.24.0
+jupyterlab_pygments==0.3.0
+jupyterlab_widgets==3.0.11
+kaggle==1.6.14
+kagglehub==0.2.5
+keras==2.15.0
+keyring==23.5.0
+kiwisolver==1.4.5
+langcodes==3.4.0
+language_data==1.2.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+lazy_loader==0.4
+libclang==18.1.1
+librosa==0.10.2.post1
+lightgbm==4.1.0
+linkify-it-py==2.0.3
+llvmlite==0.41.1
+locket==1.0.0
+logical-unification==0.4.6
+lxml==4.9.4
+malloy==2023.1067
+marisa-trie==1.2.0
+Markdown==3.6
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.7.1
+matplotlib-inline==0.1.7
+matplotlib-venn==0.11.10
+mdit-py-plugins==0.4.1
+mdurl==0.1.2
+miniKanren==1.0.3
+missingno==0.5.2
+mistune==0.8.4
+mizani==0.9.3
+mkl==2023.2.0
+ml-dtypes==0.2.0
+mlxtend==0.22.0
+more-itertools==10.1.0
+moviepy==1.0.3
+mpmath==1.3.0
+msgpack==1.0.8
+multidict==6.0.5
+multipledispatch==1.0.0
+multitasking==0.0.11
+murmurhash==1.0.10
+music21==9.1.0
+natsort==8.4.0
+nbclassic==1.1.0
+nbclient==0.10.0
+nbconvert==6.5.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.3
+nibabel==4.0.2
+nltk==3.8.1
+notebook==6.5.5
+notebook_shim==0.2.4
+numba==0.58.1
+numexpr==2.10.1
+numpy==1.25.2
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+nvtx==0.2.10
+oauth2client==4.1.3
+oauthlib==3.2.2
+opencv-contrib-python==4.8.0.76
+opencv-python==4.8.0.76
+opencv-python-headless==4.10.0.84
+openpyxl==3.1.5
+opt-einsum==3.3.0
+optax==0.2.2
+orbax-checkpoint==0.4.4
+osqp==0.6.2.post8
+packaging==24.1
+pandas==2.0.3
+pandas-datareader==0.10.0
+pandas-gbq==0.19.2
+pandas-stubs==2.0.3.230814
+pandocfilters==1.5.1
+panel==1.3.8
+param==2.1.1
+parso==0.8.4
+parsy==2.1
+partd==1.4.2
+pathlib==1.0.1
+patsy==0.5.6
+peewee==3.17.5
+pexpect==4.9.0
+phonlp==0.3.4
+pickleshare==0.7.5
+Pillow==9.4.0
+pip==23.1.2
+pip-tools==6.13.0
+platformdirs==4.2.2
+plotly==5.15.0
+plotnine==0.12.4
+pluggy==1.5.0
+polars==0.20.2
+pooch==1.8.2
+portpicker==1.5.2
+prefetch-generator==1.0.3
+preshed==3.0.9
+prettytable==3.10.0
+proglog==0.1.10
+progressbar2==4.2.0
+prometheus_client==0.20.0
+promise==2.3
+prompt_toolkit==3.0.47
+prophet==1.1.5
+proto-plus==1.24.0
+protobuf==3.20.3
+psutil==5.9.5
+psycopg2==2.9.9
+ptyprocess==0.7.0
+py-cpuinfo==9.0.0
+py4j==0.10.9.7
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pycocotools==2.0.8
+pycparser==2.22
+pydantic==2.7.4
+pydantic_core==2.18.4
+pydata-google-auth==1.8.2
+pydeck==0.9.1
+pydot==1.4.2
+pydot-ng==2.0.0
+pydotplus==2.0.2
+PyDrive==1.3.1
+PyDrive2==1.6.3
+pyerfa==2.0.1.4
+pygame==2.6.0
+Pygments==2.16.1
+PyGObject==3.42.1
+PyJWT==2.3.0
+pymc==5.10.4
+pymystem3==0.2.0
+pynvjitlink-cu12==0.2.4
+PyOpenGL==3.1.7
+pyOpenSSL==24.1.0
+pyparsing==3.1.2
+pyperclip==1.9.0
+pyproj==3.6.1
+pyproject_hooks==1.1.0
+pyshp==2.3.1
+PySocks==1.7.1
+pytensor==2.18.6
+pytest==7.4.4
+python-apt==0.0.0
+python-box==7.2.0
+python-crfsuite==0.9.10
+python-dateutil==2.8.2
+python-louvain==0.16
+python-slugify==8.0.4
+python-utils==3.8.2
+pytz==2023.4
+pyviz_comms==3.0.2
+PyWavelets==1.6.0
+PyYAML==6.0.1
+pyzmq==24.0.1
+qdldl==0.1.7.post4
+qudida==0.0.4
+ratelim==0.1.6
+referencing==0.35.1
+regex==2024.5.15
+requests==2.31.0
+requests-oauthlib==1.3.1
+requirements-parser==0.9.0
+rich==13.7.1
+rmm-cu12==24.4.0
+rpds-py==0.18.1
+rpy2==3.4.2
+rsa==4.9
+safetensors==0.4.3
+scikit-image==0.19.3
+scikit-learn==1.2.2
+scipy==1.11.4
+scooby==0.10.0
+scs==3.2.5
+seaborn==0.13.1
+SecretStorage==3.3.1
+Send2Trash==1.8.3
+sentencepiece==0.1.99
+session-info==1.0.0
+setuptools==67.7.2
+shapely==2.0.4
+shellingham==1.5.4
+simple_parsing==0.1.5
+six==1.16.0
+sklearn-pandas==2.2.0
+smart-open==7.0.4
+smmap==5.0.1
+sniffio==1.3.1
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy==3.7.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+Sphinx==5.0.2
+sphinxcontrib-applehelp==1.0.8
+sphinxcontrib-devhelp==1.0.6
+sphinxcontrib-htmlhelp==2.0.5
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.7
+sphinxcontrib-serializinghtml==1.1.10
+SQLAlchemy==2.0.31
+sqlglot==20.11.0
+sqlparse==0.5.0
+srsly==2.4.8
+stanio==0.5.0
+statsmodels==0.14.2
+stdlib-list==0.10.0
+streamlit==1.36.0
+StrEnum==0.4.15
+sympy==1.12.1
+tables==3.8.0
+tabulate==0.9.0
+tbb==2021.13.0
+tblib==3.0.0
+tenacity==8.4.2
+tensorboard==2.15.2
+tensorboard-data-server==0.7.2
+tensorflow==2.15.0
+tensorflow-datasets==4.9.6
+tensorflow-estimator==2.15.0
+tensorflow-gcs-config==2.15.0
+tensorflow-hub==0.16.1
+tensorflow-io-gcs-filesystem==0.37.0
+tensorflow-metadata==1.15.0
+tensorflow-probability==0.23.0
+tensorstore==0.1.45
+termcolor==2.4.0
+terminado==0.18.1
+text-unidecode==1.3
+textblob==0.17.1
+tf_keras==2.15.1
+tf-slim==1.1.0
+thinc==8.2.5
+threadpoolctl==3.5.0
+tifffile==2024.6.18
+tinycss2==1.3.0
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch==2.3.0+cu121
+torchaudio==2.3.0+cu121
+torchsummary==1.5.1
+torchtext==0.18.0
+torchvision==0.18.0+cu121
+tornado==6.3.3
+tqdm==4.66.4
+traitlets==5.7.1
+traittypes==0.2.1
+transformers==4.41.2
+triton==2.3.0
+tweepy==4.14.0
+typer==0.12.3
+types-pytz==2024.1.0.20240417
+types-setuptools==70.1.0.20240627
+typing_extensions==4.12.2
+tzdata==2024.1
+tzlocal==5.2
+uc-micro-py==1.0.3
+underthesea==6.8.4
+underthesea_core==1.0.4
+uritemplate==4.1.1
+urllib3==2.0.7
+vega-datasets==0.9.0
+wadllib==1.3.6
+wasabi==1.1.3
+watchdog==4.0.1
+wcwidth==0.2.13
+weasel==0.4.1
+webcolors==24.6.0
+webencodings==0.5.1
+websocket-client==1.8.0
+Werkzeug==3.0.3
+wheel==0.43.0
+widgetsnbextension==3.6.6
+wordcloud==1.9.3
+wrapt==1.14.1
+xarray==2023.7.0
+xarray-einstats==0.7.0
+xgboost==2.0.3
+xlrd==2.0.1
+xyzservices==2024.6.0
+yarl==1.9.4
+yellowbrick==1.5
+yfinance==0.2.40
+zict==3.0.0
+zipp==3.19.2