preprocessssssss

.gitignore

@@ -2,4 +2,5 @@ venv
 __pycache__
 *.csv
 models/
-.cache/
+.cache/
+!new_kamusalay.csv

main.py

@@ -3,8 +3,7 @@ from type.request.predict import PredictRequest
 from type.response.predict import PredictResponse
 from hugging_face import model, dataset
 from transformer import transformer
-
-import sys
+from preprocess import preprocess
 
 hate_speech_model = model.load_hugging_face_model('model_rf.pkl')
 hate_speech_dataset = dataset.load_dataset('data_clean.csv')
@@ -22,7 +21,8 @@ def healthz():
 
 @app.post("/predict")
 def predict(request: PredictRequest):
-  predict_text = [request.predict_text]
+  preprocessed_text = preprocess.preprocess(request.predict_text)
+  predict_text = [preprocessed_text]
 
   predict_text = tfidf.transform(predict_text)
   prediction = hate_speech_model.predict(predict_text)

preprocess/preprocess.py

@@ -0,0 +1,51 @@
+import re
+import pandas as pd
+from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
+import nltk
+from nltk.corpus import stopwords
+
+nltk.download('stopwords')
+alay_dict = pd.read_csv('./new_kamusalay.csv', encoding='latin-1', header=None)
+alay_dict = alay_dict.rename(columns={0: 'original', 1: 'replacement'})
+
+factory = StemmerFactory()
+stemmer = factory.create_stemmer()
+
+def lowercase(text):
+    return text.lower()
+
+def remove_unnecessary_char(text):
+    text = re.sub('\n',' ',text) # Remove every '\n'
+    text = re.sub('rt',' ',text) # Remove every retweet symbol
+    text = re.sub('user',' ',text) # Remove every username
+    text = re.sub('((www\.[^\s]+)|(https?://[^\s]+)|(http?://[^\s]+))',' ',text) # Remove every URL
+    text = re.sub('  +', ' ', text) # Remove extra spaces
+    return text
+    
+def remove_nonaplhanumeric(text):
+    text = re.sub('[^0-9a-zA-Z]+', ' ', text) 
+    return text
+
+alay_dict_map = dict(zip(alay_dict['original'], alay_dict['replacement']))
+def normalize_alay(text):
+    return ' '.join([alay_dict_map[word] if word in alay_dict_map else word for word in text.split(' ')])
+
+def remove_stopword(text):
+    list_stopwords = stopwords.words('indonesian')
+    text = ' '.join(['' if word in list_stopwords else word for word in text.split(' ')])
+
+    text = re.sub('  +', ' ', text) # Remove extra spaces
+    text = text.strip()
+    return text
+
+def stemming(text):
+    return stemmer.stem(text)
+
+def preprocess(text):
+    text = lowercase(text) # 1
+    text = remove_nonaplhanumeric(text) # 2
+    text = remove_unnecessary_char(text) # 2
+    text = normalize_alay(text) # 3
+    text = stemming(text) # 4
+    text = remove_stopword(text) # 5
+    return text

requirements.txt

@@ -49,3 +49,5 @@ watchfiles==0.21.0
 websockets==12.0
 xxhash==3.4.1
 yarl==1.9.4
+Sastrawi==1.0.1
+nltk==3.8.1