init commit

.gitignore

@@ -0,0 +1,3 @@
+venv
+__pycache__
+*.csv

hugging_face/dataset.py

@@ -0,0 +1,14 @@
+from datasets import load_dataset as hf_load_dataset
+from pandas import DataFrame, read_csv
+
+REPO_ID = "kompiangg/twitter_hate_speech_classification"
+
+def load_dataset(filename) -> DataFrame:
+  try:
+    df = read_csv(filename)
+  except:
+    datasets = hf_load_dataset(REPO_ID, data_files=filename, encoding='latin-1')
+    df = DataFrame(data=datasets['train'])
+    df.to_csv(filename, index=False)
+
+  return df

hugging_face/model.py

@@ -0,0 +1,7 @@
+from huggingface_hub import hf_hub_download
+import joblib
+
+REPO_ID = "kompiangg/svm-hate-speech-classification"
+
+def load_hugging_face_model(filename):
+  return joblib.load(hf_hub_download(REPO_ID, filename))

main.py

@@ -0,0 +1,29 @@
+from fastapi import FastAPI
+from type.request.predict import PredictRequest
+from type.response.predict import PredictResponse
+from hugging_face import model, dataset
+from transformer import transformer
+
+import sys
+
+hate_speech_model = model.load_hugging_face_model('model_svm.pkl')
+hate_speech_dataset = dataset.load_dataset('data_clean.csv')
+tfidf = transformer.create_tfidf(hate_speech_dataset, 'Tweet', 'U')
+
+app = FastAPI()
+
+@app.get("/healthz")
+def healthz():
+  return {"message": "All system running well :)"}
+
+@app.post("/predict")
+def predict(request: PredictRequest):
+  predict_text = [request.predict_text]
+
+  predict_text = tfidf.transform(predict_text)
+  prediction = hate_speech_model.predict(predict_text)
+  
+  return PredictResponse(
+    predict_text = request.predict_text, 
+    is_hate_speech = prediction[0] == 1
+    )

transformer/transformer.py

@@ -0,0 +1,5 @@
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+def create_tfidf(dataset, feature, label):
+  tfidf = TfidfVectorizer()
+  return tfidf.fit(dataset[feature].astype(label))

type/request/predict.py

@@ -0,0 +1,4 @@
+from pydantic import BaseModel
+
+class PredictRequest(BaseModel):
+  predict_text: str

type/response/predict.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+class PredictResponse(BaseModel):
+  predict_text: str
+  is_hate_speech: bool