Added test model performance code

app.py

@@ -4,8 +4,8 @@ import numpy as np
 from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
 
 # Define global variables
-fine_tuned_model = "andyqin18/test-finetuned"
-sample_text_num = 10
+FINE_TUNED_MODEL = "andyqin18/test-finetuned"
+NUM_SAMPLE_TEXT = 10
 
 # Define analyze function
 def analyze(model_name: str, text: str, top_k=1) -> dict:
@@ -24,7 +24,7 @@ st.write("You can choose to use my fine-tuned model or pre-trained models.")
 
 # Model hub
 model_descrip = {
-    fine_tuned_model: "This is a customized BERT-base finetuned model that detects multiple toxicity for a text. \
+    FINE_TUNED_MODEL: "This is a customized BERT-base finetuned model that detects multiple toxicity for a text. \
         Labels: toxic, severe_toxic, obscene, threat, insult, identity_hate",
     "distilbert-base-uncased-finetuned-sst-2-english": "This model is a fine-tune checkpoint of DistilBERT-base-uncased, fine-tuned on SST-2. \
         Labels: POSITIVE; NEGATIVE ",
@@ -50,7 +50,7 @@ if st.button("Analyze"):
     else:
         with st.spinner("Hang on.... Analyzing..."):
             # If fine-tuned
-            if user_model == fine_tuned_model:
+            if user_model == FINE_TUNED_MODEL:
                 result = analyze(user_model, user_input, top_k=2)  # Top 2 labels with highest score
                 result_dict = {
                         "Text": [user_input],
@@ -84,7 +84,7 @@ if st.button("Analyze"):
                                 }
 
                 for text in sample_texts:
-                    result = analyze(fine_tuned_model, text[:50], top_k=2)
+                    result = analyze(FINE_TUNED_MODEL, text[:50], top_k=2)
                     init_table_dict["Text"].append(text[:50])
                     init_table_dict["Highest Toxicity Class"].append(result[0][0]['label'])
                     init_table_dict["Highest Score"].append(result[0][0]['score'])

milestone3/milestone3.py

@@ -1,41 +0,0 @@
-from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
-
-def analyze(model_name: str, text: str, top_k=1) -> dict:
-    '''
-    Output result of sentiment analysis of a text through a defined model
-    '''
-    model = AutoModelForSequenceClassification.from_pretrained(model_name)
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    classifier = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer, top_k=top_k)
-    return classifier(text)
-
-
-user_input = "Go fuck yourself"
-user_model = "andyqin18/test-finetuned"
-
-# result = analyze(user_model, user_input, top_k=2)
-
-# print(result[0][0]['label'])
-
-import pandas as pd
-import numpy as np
-df = pd.read_csv("milestone3/comp/test_comment.csv")
-test_texts = df["comment_text"].values
-sample_texts = np.random.choice(test_texts, size=10, replace=False)
-init_table_dict = {
-            "Text": [],
-            "Highest Toxicity Class": [],
-            "Highest Score": [],
-            "Second Highest Toxicity Class": [],
-            "Second Highest Score": []
-                }
-
-for text in sample_texts:
-    result = analyze(user_model, text, top_k=2)
-    init_table_dict["Text"].append(text[:50])
-    init_table_dict["Highest Toxicity Class"].append(result[0][0]['label'])
-    init_table_dict["Highest Score"].append(result[0][0]['score'])
-    init_table_dict["Second Highest Toxicity Class"].append(result[0][1]['label'])
-    init_table_dict["Second Highest Score"].append(result[0][1]['score'])
-
-print(init_table_dict)

test_model.py

@@ -0,0 +1,84 @@
+import numpy as np
+import pandas as pd
+from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
+import torch
+from tqdm import tqdm
+
+
+# Global var
+TEST_SIZE = 1000
+FINE_TUNED_MODEL = "andyqin18/test-finetuned"
+
+
+# Define analyze function
+def analyze(text: str):
+    '''
+    Input: Text string
+    Output: Prediction array (6x1) with threshold prob > 0.5
+    '''
+    encoding = tokenizer(text, return_tensors="pt")
+    encoding = {k: v.to(model.device) for k,v in encoding.items()}
+    outputs = model(**encoding)
+    logits = outputs.logits
+    sigmoid = torch.nn.Sigmoid()
+    probs = sigmoid(logits.squeeze().cpu())
+    predictions = np.zeros(probs.shape)
+    predictions[np.where(probs >= 0.5)] = 1
+    return predictions
+
+
+# Read dataset and randomly select testing texts and respective labels
+df = pd.read_csv("milestone3/comp/train.csv")
+labels = df.columns[2:]
+num_label = len(labels)
+train_texts = df["comment_text"].values
+train_labels = df[labels].values
+
+np.random.seed(1)
+small_test_texts = np.random.choice(train_texts, size=TEST_SIZE, replace=False)
+
+np.random.seed(1)
+small_test_labels_idx = np.random.choice(train_labels.shape[0], size=TEST_SIZE, replace=False)
+small_test_labels = train_labels[small_test_labels_idx, :]
+
+
+# Load model and tokenizer. Prepare for analysis loop
+model = AutoModelForSequenceClassification.from_pretrained(FINE_TUNED_MODEL)
+tokenizer = AutoTokenizer.from_pretrained(FINE_TUNED_MODEL)
+total_true = 0
+total_success = 0
+TP, FP, TN, FN = 0, 0, 0, 0
+
+
+# Analysis Loop
+for comment_idx in tqdm(range(TEST_SIZE), desc="Analyzing..."):
+    comment = small_test_texts[comment_idx]
+    target = small_test_labels[comment_idx]
+    result = analyze(comment[:500])
+
+    # Counting TP, FP, TN, FN
+    for i in range(num_label):
+        if result[i] == target[i]:
+            if result[i] == 1:
+                TP += 1
+            else:
+                TN += 1
+        else:
+            if result[i] == 1:
+                FP += 1
+            else:
+                FN += 1
+
+    # Counting success prediction of 1) each label, 2) label array 
+    num_true = (result == target).sum()
+    if num_true == len(labels):
+        total_success += 1
+    total_true += num_true
+
+# Calculate performance
+performance = {}
+performance["label_accuracy"] = total_true/(len(labels) * TEST_SIZE)
+performance["prediction_accuracy"] = total_success/TEST_SIZE
+performance["precision"] = TP / (TP + FP)
+performance["recall"] = TP / (TP + FN)
+print(performance)