Repository: GlobalMaksimum/sadedegel (language: python)

Issue:
Customer Reviews Classification Prebuilt Model
For the purpose of adding different use cases of our prebuilt function we could implement a customer review classification model. There is a decent dataset available [here](https://www.kaggle.com/savasy/multiclass-classification-data-for-turkish-tc32):

- It contians 430K lines, 32 categories
- Each category roughly has 13K comments
- Data is collected from Turkish web sites
- The data contains the comments of the products and product categories

For this, these steps going be taken:

- Prepare the dataset,
- Built and test model accuracy,
- Publish the prebuilt model,



Discussion / hints:

Fix patch:
diff --git a/sadedegel/prebuilt/README.md b/sadedegel/prebuilt/README.md
index 6d948cb4..09aece5c 100644
--- a/sadedegel/prebuilt/README.md
+++ b/sadedegel/prebuilt/README.md
@@ -124,3 +124,37 @@ Comparable [benchmark](https://ieeexplore.ieee.org/document/8554037/) models has
 * `0.66` **macro-F1** score (classical ML approach fed with bag-of-words)
 on the hold-out set.
 
+
+### Turkish Customer Reviews Classification
+
+Classifier assigns each Turkish customer review text into 32 classes by using sadedegel built-in pipeline.
+
+#### Loading and Predicting with the Model:
+
+```python
+from sadedegel.prebuilt import customer_reviews_classification
+
+# We load our prebuilt model:
+model = customer_reviews_classification.load()
+
+# Here we feed our text to get predictions:
+y_pred = model.predict(['odalar çok kirliydi'])
+
+# You can also get probabilities with:
+y_probs = model.predict_proba(['odalar çok kirliydi'])
+
+# You can check original test results on holdout set:
+customer_reviews_classification.evaluate()
+
+# You convert class id's back to labels by importing:
+from sadedegel.dataset.customer_review import CLASS_VALUES
+
+# And simply map them to get the converted string list:
+y_pred_label = [CLASS_VALUES[idx] for idx in y_pred]
+```
+
+#### Accuracy
+Current prebuilt customer review classification model has a macro-F1 score of `0.851` on holdout test set model never seen before.
+
+If you want to compare benchmark results:
+> The model on [Kaggle](https://www.kaggle.com/savasy/multiclass-classification-data-for-turkish-tc32) where we got dataset from has F1 score of `0.84`.
diff --git a/sadedegel/prebuilt/customer_reviews_classification.py b/sadedegel/prebuilt/customer_reviews_classification.py
new file mode 100644
index 00000000..2ebf07ef
--- /dev/null
+++ b/sadedegel/prebuilt/customer_reviews_classification.py
@@ -0,0 +1,92 @@
+from pathlib import Path
+from os.path import dirname
+
+from sklearn.linear_model import LogisticRegression
+from sklearn.utils import shuffle
+from sklearn.metrics import f1_score, roc_auc_score
+from sklearn.pipeline import Pipeline
+
+from rich.console import Console
+
+from joblib import dump
+
+from ..extension.sklearn import TfidfVectorizer, Text2Doc
+from .util import load_model
+
+from ..dataset.customer_review import load_train, \
+    load_test, load_test_label, CORPUS_SIZE
+
+console = Console()
+
+def empty_model():
+    return Pipeline(
+        [('text2doc', Text2Doc('icu')),
+         ('tfidf', TfidfVectorizer(tf_method='log_norm', idf_method='probabilistic', drop_punct=True,
+                                   lowercase=True, show_progress=True)),
+         ('logreg', LogisticRegression(penalty='l2', C=0.02987625339246147, fit_intercept=False, solver='liblinear'))
+         ]
+    )
+
+def build(max_rows=-1, save=True):
+    try:
+        import pandas as pd
+    except ImportError:
+        console.log(("pandas package is not a general sadedegel dependency."
+                     " But we do have a dependency on building our prebuilt models"))
+
+    raw = load_train()
+    df = pd.DataFrame.from_records(raw)
+
+    df = shuffle(df, random_state=42)
+
+    if max_rows > 0:
+        df = df.sample(max_rows, random_state=42)
+
+    console.log(f"Corpus Size: {CORPUS_SIZE}")
+
+    pipeline = empty_model()
+
+    pipeline.fit(df.text, df.review_class)
+
+    if save:
+        model_dir = Path(dirname(__file__)) / 'model'
+
+        model_dir.mkdir(parents=True, exist_ok=True)
+
+        pipeline.steps[0][1].Doc = None
+
+        dump(pipeline, (model_dir / 'customer_review_classification.joblib').absolute(), compress=('gzip', 9))
+
+    evaluate(pipeline, max_rows=int(round(max_rows*0.20)))
+
+    console.log('Model build [green]DONE[/green]')
+
+def load(model_name='customer_review_classification'):
+    return load_model(model_name)
+
+def evaluate(model=None, scoring='f1', max_rows=-1):
+    try:
+        import pandas as pd
+    except ImportError:
+        console.log(("pandas package is not a general sadedegel dependency."
+                     " But we do have a dependency on building our prebuilt models"))
+    model = load()
+
+    raw_test = load_test()
+    test = pd.DataFrame.from_records(raw_test)
+    true_labels = pd.DataFrame.from_records(load_test_label())
+
+    if max_rows > 0:
+        test = test[:max_rows]
+        true_labels = true_labels[:max_rows]
+
+    if scoring=='f1':
+        y_pred = model.predict(test.text)
+        console.log(f"Model test accuracy (f1-macro): {f1_score(true_labels.review_class, y_pred, average='macro')}")
+    if scoring=='auc':
+        y_pred = model.predict_proba(test.text)
+        console.log(f"Model test roc_auc score: {roc_auc_score(true_labels.review_class, y_pred, multi_class='ovr')}")
+
+
+if __name__ == "__main__":
+    build(save=True)
\ No newline at end of file
diff --git a/sadedegel/prebuilt/model/customer_review_classification.joblib b/sadedegel/prebuilt/model/customer_review_classification.joblib
new file mode 100644
index 00000000..6c9d0314
--- /dev/null
+++ b/sadedegel/prebuilt/model/customer_review_classification.joblib
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eaa49b33373fa41e2512892747a3a778bca561d8f75afcb003a498b29876a36
+size 12102106


Test patch:
diff --git a/tests/prebuilt/context.py b/tests/prebuilt/context.py
index dcc7739d..916cfe0e 100644
--- a/tests/prebuilt/context.py
+++ b/tests/prebuilt/context.py
@@ -7,7 +7,8 @@
 from sadedegel.prebuilt import news_classification, tweet_profanity, telco_sentiment  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.tscorpus import CATEGORIES # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.profanity import CLASS_VALUES # noqa # pylint: disable=unused-import, wrong-import-position
-from sadedegel.prebuilt import tweet_sentiment , movie_reviews # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.prebuilt import tweet_sentiment , movie_reviews, customer_reviews_classification # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.tweet_sentiment import CLASS_VALUES as SENTIMENT_VALUES  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.movie_sentiment import CLASS_VALUES as SENTIMENT_VALUES_M  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.telco_sentiment import CLASS_VALUES as SENTIMENT_VALUES_T  # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.dataset.customer_review import CLASS_VALUES as CLASS_VALUES_CUST # noqa # pylint: disable=unused-import, wrong-import-position
diff --git a/tests/prebuilt/test_customer_reviews.py b/tests/prebuilt/test_customer_reviews.py
new file mode 100644
index 00000000..a649ce76
--- /dev/null
+++ b/tests/prebuilt/test_customer_reviews.py
@@ -0,0 +1,14 @@
+from .context import customer_reviews_classification, CLASS_VALUES_CUST
+
+def test_model_load():
+    model = customer_reviews_classification.load()
+    pred_hotel = model.predict(['odalar çok kirliydi'])
+    pred_pc = model.predict(['ram çok düşük'])
+
+    assert CLASS_VALUES_CUST[pred_hotel[0]] == 'Turizm'
+    assert CLASS_VALUES_CUST[pred_pc[0]] == 'Bilgisayar'
+
+    prob_hotel = model.predict_proba(['odalar çok kirliydi'])
+
+    assert prob_hotel[0][30] == prob_hotel[0].max()
+    assert prob_hotel.shape == (1, 32)
\ No newline at end of file