Fixed words with # and @

app.py

@@ -41,6 +41,7 @@ profanities = read_text('profanities', 'json')
 lookup_profanity = np.concatenate([np.hstack(list(profanities.values())), list(profanities.keys())])
 lookup_words = list(set(similar_words).union(set(lookup_profanity.tolist())))
 eng_words = list(set(words.words()) - set(lookup_profanity))
+punctuations = re.compile(r'^[^\w#@]+|[^\w#@]+$')
 
 # TODO check eng words that are tagalog profanities
 
@@ -48,29 +49,34 @@ def fuzzy_lookup(tweet):
 
     matched_profanity = []
 
-    for word in tweet.split():
+    # tweet = punctuations.sub('', tweet).lower()
 
-        word = word.strip(punctuation)
-        base_word = word
+    for word in tweet.split():
 
+        word = punctuations.sub('', word).lower()
+        base_word =  word
+        
         if word in eng_words:
             continue
         
         for addon in addon_words:
             if word.startswith(addon):
-                word[len(addon):]
+                word = word[len(addon):]
             if word.endswith(addon):
-                word[:-len(addon)]
+                word = word[:-len(addon)]
+
+        if word.startswith("@") or word.startswith("#"):
+            word = word[1:]
 
         scores = []
         matched_words = []
-        
-        processed_word = re.sub("[^a-zA-Z0-9@#]", "", word)
 
-        if len(processed_word) >= 4:
+        print(word)
+
+        if len(word) >= 4:
             # Get fuzzy ratio
             for lookup_word in lookup_words:
-                score = fuzz.ratio(processed_word, lookup_word)
+                score = fuzz.ratio(word, lookup_word)
                 if score >= 70:
                     scores.append(score)
                     matched_words.append(lookup_word)
@@ -159,4 +165,4 @@ demo = gr.Interface(
     title="Tagalog Profanity Classifier"
 )
 
-demo.launch(debug=True)
+demo.launch(debug=True)

profanities.json

@@ -9,7 +9,7 @@
     "punyeta": [],
     "puta": ["pota"],
     "putangina": ["pukingina", "kinangina", "putang"],
-    "tanga": [],
+    "tanga": ["t4ng4"],
     "tangina": ["tangna", "inamo", "tatanga"],
     "tarantado": ["t4r4nt4do", "t@r@nt@do"],
     "ulol": ["ul0l", "olol", "0lol"]