Updated preprocessing and inferenced on data_3

data_3/news_articles.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53855240e9036a7d6c204e72bd0fa9d37a10f8e1bd2b2fdf34b962569ef271c6
+size 10969548

inference_main.py

@@ -3,53 +3,79 @@ import pandas as pd
 from preprocessing import preprocess_text, load_tokenizer, prepare_data
 from data_loader import create_data_loader
 from inference import load_model, evaluate_model
+import os
 
-version = 7
+version = 9
 
 
 def run_evaluation(model_path, tokenizer_path, device):
-    cleaned_path = f'./output/version_{version}/cleaned_inference_data_{version}.csv'
+    cleaned_path = f"./output/version_{version}/cleaned_inference_data_{version}.csv"
     # Load data
-    try:
+    if os.path.exists(cleaned_path):
         df = pd.read_csv(cleaned_path)
         df.dropna(inplace=True)
         print("Cleaned data found.")
-    except:
+    else:
         print("No cleaned data found. Cleaning data now...")
-        # Load the datasets
-        true_news = pd.read_csv('data_1/True.csv')
-        fake_news = pd.read_csv('data_1/Fake.csv')
+        # # Load the datasets
+        # true_news = pd.read_csv("data_1/True.csv")
+        # fake_news = pd.read_csv("data_1/Fake.csv")
+
+        # # Add labels
+        # true_news["label"] = 1
+        # fake_news["label"] = 0
+
+        # # Combine the datasets
+        # df = pd.concat([true_news, fake_news], ignore_index=True)
+
+        # # Drop unnecessary columns
+        # df.drop(columns=["subject", "date"], inplace=True)
+
+        df = pd.read_csv("./data_3/news_articles.csv")
+        df.drop(
+            columns=[
+                "author",
+                "published",
+                "site_url",
+                "main_img_url",
+                "type",
+                "text_without_stopwords",
+                "title_without_stopwords",
+                "hasImage",
+            ],
+            inplace=True,
+        )
+        # Map Real to 1 and Fake to 0
+        df["label"] = df["label"].map({"Real": 1, "Fake": 0})
+        df = df[df["label"].isin([1, 0])]
+
+        # Drop rows where the language is not 'english'
+        df = df[df["language"] == "english"]
+        df.drop(columns=["language"], inplace=True)
+
+        # Convert "no title" to empty string
+        df["title"] = df["title"].apply(lambda x: "" if x == "no title" else x)
 
-        # Add labels
-        true_news['label'] = 1
-        fake_news['label'] = 0
-
-        # Combine the datasets
-        df = pd.concat([true_news, fake_news], ignore_index=True)
-
-        # Drop unnecessary columns
-        df.drop(columns=['subject', 'date'], inplace=True)
-
-        df['title'] = df['title'].apply(preprocess_text)
-        df['text'] = df['text'].apply(preprocess_text)
+        df.dropna(inplace=True)
+        df["title"] = df["title"].apply(preprocess_text)
+        df["text"] = df["text"].apply(preprocess_text)
 
         df.to_csv(cleaned_path, index=False)
         df.dropna(inplace=True)
         print("Cleaned data saved.")
 
-    labels = df['label'].values
+    labels = df["label"].values
 
     # Load tokenizer and model
     tokenizer = load_tokenizer(tokenizer_path)
     model = load_model(model_path, len(tokenizer.word_index) + 1)
 
     # Prepare data
-    titles = prepare_data(df['title'], tokenizer)
-    texts = prepare_data(df['text'], tokenizer)
+    titles = prepare_data(df["title"], tokenizer)
+    texts = prepare_data(df["text"], tokenizer)
 
     # Create DataLoader
-    data_loader = create_data_loader(
-        titles, texts, batch_size=32, shuffle=False)
+    data_loader = create_data_loader(titles, texts, batch_size=32, shuffle=False)
 
     # Evaluate
     accuracy, f1, auc_roc = evaluate_model(model, data_loader, device, labels)
@@ -57,11 +83,10 @@ def run_evaluation(model_path, tokenizer_path, device):
 
 
 if __name__ == "__main__":
-    model_path = f'./output/version_{version}/best_model_{version}.pth'
-    tokenizer_path = f'./output/version_{version}/tokenizer_{version}.pickle'
+    model_path = f"./output/version_{version}/best_model_{version}.pth"
+    tokenizer_path = f"./output/version_{version}/tokenizer_{version}.pickle"
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     print(f"Device: {device}")
 
     accuracy, f1, auc_roc = run_evaluation(model_path, tokenizer_path, device)
-    print(
-        f'Accuracy: {accuracy:.4f}, F1 Score: {f1:.4f}, AUC-ROC: {auc_roc:.4f}')
+    print(f"Accuracy: {accuracy:.4f}, F1 Score: {f1:.4f}, AUC-ROC: {auc_roc:.4f}")

output/version_8/best_model_8.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:addb4e8467fca9234e566636574ad2bd0544e91f629c5cf27ec88b173eb6df69
+size 101405944

output/version_8/cleaned_news_data_8.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0cae611f708ed033cb431b4ff525901cdfbc27e81eeacc872087a4efd6e8310
+size 154593478

output/version_8/confusion_matrix_data_8.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37c139bea7aa1bc4d747f5ad9cfbb93fafa113689865bfb329d480a818367134
+size 127312

output/version_8/tokenizer_8.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:420dc8ff87e3271e6a2f4e12f503f0a746a17bf3284bcf8e450808894109d0e4
+size 8809775

output/version_8/training_metrics_8.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06c222696c315a66c1bfa0433472784fc560e11384ddc9055d724a5a13921f1f
+size 2296

output/version_9/best_model_9.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50c1dc4473380483255e98c19d58f34d2aab132ab17be2c7cc31d6eb88551dc8
+size 101405944

output/version_9/cleaned_inference_data_9.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29cd7b40d7e925e4613e986b5e68420c0ca252544aa3fa6a435723b11d2a0a01
+size 3873531

output/version_9/cleaned_news_data_9.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0cae611f708ed033cb431b4ff525901cdfbc27e81eeacc872087a4efd6e8310
+size 154593478

output/version_9/confusion_matrix_data_9.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c07e7992962a370e4e35680f5dc5ba6ee49b1c8c9ce056fa292f296ceb424a5
+size 127312

output/version_9/tokenizer_9.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16950eb0e8dc9cb8d1eadeda4f15387ce0fed66b5b3349229aa5e371a918602d
+size 8809775

output/version_9/training_metrics_9.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd2fc4ee2c7804cbb0a2f6a74bab16af6ecacb7b05ce7efcedbdaae7757352eb
+size 843

preprocessing.py

@@ -4,24 +4,23 @@ from keras.preprocessing.text import Tokenizer
 from keras_preprocessing.sequence import pad_sequences
 import pickle
 
-spacy.prefer_gpu()
-print("GPU is available:", spacy.prefer_gpu())
 
 # Load spaCy's English model
-nlp = spacy.load('en_core_web_sm')
+nlp = spacy.load("en_core_web_sm")
 
 
 def preprocess_text(text):
-    # Remove patterns like "COUNTRY or STATE NAME (Reuters) -"
-    text = re.sub(r'\b[A-Z]{2,}(?:\s[A-Z]{2,})*\s\(Reuters\)\s-', '', text)
+    # Remove patterns like "COUNTRY or STATE NAME (Reuters) -" or just "(Reuters)"
+    text = re.sub(
+        r"(\b[A-Z]{2,}(?:\s[A-Z]{2,})*\s\(Reuters\)\s-|\(Reuters\))", "", text
+    )
 
     # Remove patterns like "Featured image via author name / image place"
-    text = re.sub(r'Featured image via .+ / .+', '', text)
+    text = re.sub(r"Featured image via .+?\.($|\s)", "", text)
 
     # Process text with spaCy
     doc = nlp(text)
 
-    # Improved lemmatization
     lemmatized_text = []
     for token in doc:
         # Preserve named entities in their original form
@@ -31,11 +30,11 @@ def preprocess_text(text):
         elif token.is_alpha and not token.is_stop:
             lemmatized_text.append(token.lemma_.lower())
 
-    return ' '.join(lemmatized_text)
+    return " ".join(lemmatized_text)
 
 
 def load_tokenizer(tokenizer_path):
-    with open(tokenizer_path, 'rb') as handle:
+    with open(tokenizer_path, "rb") as handle:
         tokenizer = pickle.load(handle)
     return tokenizer
 

train_main.py

@@ -12,11 +12,11 @@ import pickle
 import train as tr
 from torch.utils.data import Dataset, DataLoader
 from data_loader import NewsDataset
+import os
 
-version = 7
+version = 9
 
 if __name__ == "__main__":
-
     # fake_path = './data_1/Fake.csv'
     # true_path = './data_1/True.csv'
     # cleaned_path = './cleaned_news_data.csv'
@@ -47,8 +47,8 @@ if __name__ == "__main__":
     #     df.to_csv('cleaned_news_data.csv', index=False)
     #     df.dropna(inplace=True)
 
-    data_path = './data_2/WELFake_Dataset.csv'
-    cleaned_path = f'./output/version_{version}/cleaned_news_data_{version}.csv'
+    data_path = "./data_2/WELFake_Dataset.csv"
+    cleaned_path = f"./output/version_{version}/cleaned_news_data_{version}.csv"
     # Load data
     try:
         df = pd.read_csv(cleaned_path)
@@ -63,35 +63,38 @@ if __name__ == "__main__":
         df.dropna(inplace=True)
 
         # Swapping labels around since it originally is the opposite
-        df['label'] = df['label'].map({0: 1, 1: 0})
+        df["label"] = df["label"].map({0: 1, 1: 0})
 
-        df['title'] = df['title'].apply(preprocess_text)
-        df['text'] = df['text'].apply(preprocess_text)
+        df["title"] = df["title"].apply(preprocess_text)
+        df["text"] = df["text"].apply(preprocess_text)
 
+        # Create the directory if it does not exist
+        os.makedirs(os.path.dirname(cleaned_path), exist_ok=True)
         df.to_csv(cleaned_path, index=False)
         print("Cleaned data saved.")
 
     # Splitting the data
     train_val, test = train_test_split(df, test_size=0.2, random_state=42)
     train, val = train_test_split(
-        train_val, test_size=0.25, random_state=42)  # 0.25 * 0.8 = 0.2
+        train_val, test_size=0.25, random_state=42
+    )  # 0.25 * 0.8 = 0.2
 
     # Initialize the tokenizer
     tokenizer = Tokenizer()
 
     # Fit the tokenizer on the training data
-    tokenizer.fit_on_texts(train['title'] + train['text'])
+    tokenizer.fit_on_texts(train["title"] + train["text"])
 
-    with open(f'./output/version_{version}/tokenizer_{version}.pickle', 'wb') as handle:
+    with open(f"./output/version_{version}/tokenizer_{version}.pickle", "wb") as handle:
         pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
     # Tokenize the data
-    X_train_title = tokenizer.texts_to_sequences(train['title'])
-    X_train_text = tokenizer.texts_to_sequences(train['text'])
-    X_val_title = tokenizer.texts_to_sequences(val['title'])
-    X_val_text = tokenizer.texts_to_sequences(val['text'])
-    X_test_title = tokenizer.texts_to_sequences(test['title'])
-    X_test_text = tokenizer.texts_to_sequences(test['text'])
+    X_train_title = tokenizer.texts_to_sequences(train["title"])
+    X_train_text = tokenizer.texts_to_sequences(train["text"])
+    X_val_title = tokenizer.texts_to_sequences(val["title"])
+    X_val_text = tokenizer.texts_to_sequences(val["text"])
+    X_test_title = tokenizer.texts_to_sequences(test["title"])
+    X_test_text = tokenizer.texts_to_sequences(test["text"])
 
     # Padding sequences
     max_length = 500
@@ -108,19 +111,46 @@ if __name__ == "__main__":
     model = LSTMModel(len(tokenizer.word_index) + 1).to(device)
 
     # Convert data to PyTorch tensors
-    train_data = NewsDataset(torch.tensor(X_train_title), torch.tensor(
-        X_train_text), torch.tensor(train['label'].values))
-    val_data = NewsDataset(torch.tensor(X_val_title), torch.tensor(
-        X_val_text), torch.tensor(val['label'].values))
-    test_data = NewsDataset(torch.tensor(X_test_title), torch.tensor(
-        X_test_text), torch.tensor(test['label'].values))
-
-    train_loader = DataLoader(train_data, batch_size=32,
-                              shuffle=True, num_workers=6, pin_memory=True, persistent_workers=True)
-    val_loader = DataLoader(val_data, batch_size=32,
-                            shuffle=False, num_workers=6, pin_memory=True, persistent_workers=True)
-    test_loader = DataLoader(test_data, batch_size=32,
-                             shuffle=False, num_workers=6, pin_memory=True, persistent_workers=True)
+    train_data = NewsDataset(
+        torch.tensor(X_train_title),
+        torch.tensor(X_train_text),
+        torch.tensor(train["label"].values),
+    )
+    val_data = NewsDataset(
+        torch.tensor(X_val_title),
+        torch.tensor(X_val_text),
+        torch.tensor(val["label"].values),
+    )
+    test_data = NewsDataset(
+        torch.tensor(X_test_title),
+        torch.tensor(X_test_text),
+        torch.tensor(test["label"].values),
+    )
+
+    train_loader = DataLoader(
+        train_data,
+        batch_size=32,
+        shuffle=True,
+        num_workers=6,
+        pin_memory=True,
+        persistent_workers=True,
+    )
+    val_loader = DataLoader(
+        val_data,
+        batch_size=32,
+        shuffle=False,
+        num_workers=6,
+        pin_memory=True,
+        persistent_workers=True,
+    )
+    test_loader = DataLoader(
+        test_data,
+        batch_size=32,
+        shuffle=False,
+        num_workers=6,
+        pin_memory=True,
+        persistent_workers=True,
+    )
 
     criterion = nn.BCELoss()
     optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
@@ -132,17 +162,17 @@ if __name__ == "__main__":
         criterion=criterion,
         optimizer=optimizer,
         version=version,
-        epochs=50,
+        epochs=10,
         device=device,
         max_grad_norm=1.0,
         early_stopping_patience=3,
-        early_stopping_delta=0.001
+        early_stopping_delta=0.01,
     )
 
-    print(f'Best model was saved at epoch: {best_epoch}')
+    print(f"Best model was saved at epoch: {best_epoch}")
 
     # Load the best model before testing
-    best_model_path = f'./output/version_{version}/best_model_{version}.pth'
+    best_model_path = f"./output/version_{version}/best_model_{version}.pth"
     model.load_state_dict(torch.load(best_model_path, map_location=device))
 
     # Testing
@@ -155,8 +185,11 @@ if __name__ == "__main__":
         correct = 0
         total = 0
         for titles, texts, labels in test_loader:
-            titles, texts, labels = titles.to(device), texts.to(
-                device), labels.to(device).float()
+            titles, texts, labels = (
+                titles.to(device),
+                texts.to(device),
+                labels.to(device).float(),
+            )
             outputs = model(titles, texts).squeeze()
 
             predicted = (outputs > 0.5).float()
@@ -171,10 +204,11 @@ if __name__ == "__main__":
     auc_roc = roc_auc_score(true_labels, predicted_probs)
 
     print(
-        f'Test Accuracy: {test_accuracy:.2f}%, F1 Score: {f1:.4f}, AUC-ROC: {auc_roc:.4f}')
+        f"Test Accuracy: {test_accuracy:.2f}%, F1 Score: {f1:.4f}, AUC-ROC: {auc_roc:.4f}"
+    )
 
     # Create DataFrame and Save to CSV
-    confusion_data = pd.DataFrame(
-        {'True': true_labels, 'Predicted': predicted_labels})
+    confusion_data = pd.DataFrame({"True": true_labels, "Predicted": predicted_labels})
     confusion_data.to_csv(
-        f'./output/version_{version}/confusion_matrix_data_{version}.csv', index=False)
+        f"./output/version_{version}/confusion_matrix_data_{version}.csv", index=False
+    )