Update app.py

app.py

@@ -4,48 +4,55 @@ import re
 
 sentiment_pipeline = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
 
+ner_pipeline = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-english", aggregation_strategy="simple")
+
 moderation_guidelines = """
 - Allow positive messages
 - Block cuss words
 - Allow negative comments about individuals but block negative comments against a community
-- Block words: Darren
+- Block personal names
 """
 
+default_cuss_words = {"damn", "hell", "shit", "fuck", "ass", "bastard", "bitch", "bollocks", "bugger", 
+"bullshit", "crap", "dammit", "douche", "dumbass", "faggot", "jackass", "jerk", 
+"motherfucker", "piss", "prick", "slut", "son of a bitch", "twat", "wanker"}
 
-default_cuss_words = {
-    "damn", "hell", "shit", "fuck", "ass", "bastard", "bitch", "bollocks", "bugger", 
-    "bullshit", "crap", "dammit", "douche", "dumbass", "faggot", "jackass", "jerk", 
-    "motherfucker", "piss", "prick", "slut", "son of a bitch", "twat", "wanker"
-}
-
-community_terms = {"religion", "race", "ethnicity", "group", "community", "gender", "china"}
+community_terms = {"religion", "race", "ethnicity", "group", "community", "gender"}
 
 def extract_blocked_words(guidelines):
-    """Extract blocked words from guidelines"""
+    """Extracts blocked words from moderation guidelines."""
     match = re.search(r"block words:\s*(.*)", guidelines.lower())
-    return set(match.group(1).split(",")) if match else set()
+    return {word.strip() for word in match.group(1).split(",") if word.strip()} if match else set()
 
 def moderate_message(message, guidelines):
-    """Moderates a message based on sentiment and dynamic guidelines."""
+    """Moderates a message based on sentiment and dynamic moderation rules."""
     
     sentiment = sentiment_pipeline(message)[0]['label']  
     
-  
     blocked_words = extract_blocked_words(guidelines)
-    
-    
     allow_positive = "allow positive" in guidelines.lower()
     block_cuss_words = "block cuss" in guidelines.lower()
     allow_negative_personal = "allow negative comments about individuals" in guidelines.lower()
     block_negative_community = "block negative comments against a community" in guidelines.lower()
+    block_personal_names = "block personal names" in guidelines.lower() 
+
+    words = set(re.findall(r'\w+', message.lower()))
 
-  
-    words = set(re.findall(r'\w+', message.lower()))  
-    if block_cuss_words and (words & default_cuss_words):
+    # 1. Block Cuss Words
+    if block_cuss_words and words & default_cuss_words:
         return "❌ Message Blocked: Contains inappropriate language."
 
+    # 2. Block Dynamically Defined Words
     if words & blocked_words:
         return "🚫 Message Blocked: Contains restricted words."
+
+    # 3. Block Personal Names Dynamically
+    if block_personal_names:
+        entities = ner_pipeline(message)
+        for entity in entities:
+            if entity['entity_group'] == 'PER': 
+                return "🚫 Message Blocked: Contains personal names."
+
     if sentiment == "POSITIVE" and allow_positive:
         return f"✅ Allowed (Positive): {message}"
 
@@ -59,14 +66,14 @@ def moderate_message(message, guidelines):
 
 with gr.Blocks() as demo:
     gr.Markdown("### 🛡️ AI-Powered Moderation System")
-
-    guidelines_input = gr.Textbox(value=moderation_guidelines, label="Moderation Guidelines (Admins Can Update)", lines=4)
-
+    guidelines_input = gr.Textbox(value=moderation_guidelines, label="Moderation Guidelines (Admins Can Update)", lines=6)
+    
     with gr.Row():
         msg_input = gr.Textbox(label="Enter Message")
         msg_output = gr.Textbox(label="Moderation Result", interactive=False)
     
     moderate_btn = gr.Button("Check Message")
+    
     moderate_btn.click(moderate_message, inputs=[msg_input, guidelines_input], outputs=[msg_output])
 
 demo.launch()