Repository: GlobalMaksimum/sadedegel (language: python)

Issue:
Implement Hotel Review Sentiment Pre-Built Model
 I implemented the Hotel Review dataset in #254 now I will work on prebuilt model. The dataset is split into 2 in paper, thus, there is a small train size(5800). I require permission to work on small data and compare performance with paper.


Discussion / hints:

Fix patch:
diff --git a/sadedegel/prebuilt/README.md b/sadedegel/prebuilt/README.md
index 5640b1a4..67f02b08 100644
--- a/sadedegel/prebuilt/README.md
+++ b/sadedegel/prebuilt/README.md
@@ -168,3 +168,36 @@ Current prebuilt customer review classification model has a macro-F1 score of `0
 
 If you want to compare benchmark results:
 > The model on [Kaggle](https://www.kaggle.com/savasy/multiclass-classification-data-for-turkish-tc32) where we got dataset from has F1 score of `0.84`.
+
+###  Hotel Sentiment Classification
+
+Using sadedegel built-in pipeline, the prebuilt model for hotel review sentiment classification assigns text into two
+classes, `0` and `1`, for `NEGATIVE` and `POSITIVE` sentiments consecutively.
+
+#### Loading and Predicting with the Model:
+
+```python
+from sadedegel.prebuilt import hotel_sentiment
+
+# We load our prebuilt model:
+model = hotel_sentiment.load()
+
+# Here we feed our text to get predictions:
+y_pred = model.predict(['otel çok temiz, çalışanlar güler yüzlü, yemekler fena değil'])
+
+# You can also get probabilities with:
+y_probs = model.predict_proba(['otel çok temiz, çalışanlar güler yüzlü, yemekler fena değil'])
+
+# You can check original test results on holdout set:
+hotel_sentiment.evaluate()
+
+# You convert class id's back to labels by importing:
+from sadedegel.dataset.hotel_sentiment import CLASS_VALUES
+
+# And simply map them to get the converted string list:
+y_pred_label = [CLASS_VALUES[idx] for idx in y_pred]
+```
+
+#### Accuracy
+Current prebuilt hotel sentiment classification model has an accuracy of `0.863` on holdout test set. Comparable
+benchmark results can be found [here](https://ieeexplore.ieee.org/document/7907484) where they obtained an Accuracy score of 0.797 with SVM.
diff --git a/sadedegel/prebuilt/hotel_sentiment.py b/sadedegel/prebuilt/hotel_sentiment.py
new file mode 100644
index 00000000..3dc7e777
--- /dev/null
+++ b/sadedegel/prebuilt/hotel_sentiment.py
@@ -0,0 +1,85 @@
+from os.path import dirname
+from pathlib import Path
+
+from joblib import dump
+from rich.console import Console
+from sklearn.metrics import accuracy_score
+from sklearn.pipeline import Pipeline
+from sklearn.linear_model import SGDClassifier
+from sklearn.utils import shuffle
+
+from .util import load_model
+from ..dataset.hotel_sentiment import load_hotel_sentiment_train, load_hotel_sentiment_test, \
+    load_hotel_sentiment_test_label, CORPUS_SIZE
+from ..extension.sklearn import HashVectorizer, Text2Doc
+
+console = Console()
+
+
+def empty_model():
+    return Pipeline(
+        [('text2doc', Text2Doc("icu")),
+         ('hash', HashVectorizer(n_features=44995, alternate_sign=True)),
+         ('sgd', SGDClassifier(alpha=0.009427412171474367, penalty='l2', loss='modified_huber', random_state=42))]
+    )
+
+
+def build(save=True):
+    try:
+        import pandas as pd
+    except ImportError:
+        console.log(("pandas package is not a general sadedegel dependency."
+                     " But we do have a dependency on building our prebuilt models"))
+
+    raw = load_hotel_sentiment_train()
+    df = pd.DataFrame.from_records(raw)
+    df = shuffle(df)
+
+    console.log(f"Corpus Size: {CORPUS_SIZE}")
+
+    pipeline = empty_model()
+
+    pipeline.fit(df.text, df.sentiment_class)
+
+    evaluate(pipeline)
+
+    console.log("Model build [green]DONE[/green]")
+
+    if save:
+        model_dir = Path(dirname(__file__)) / 'model'
+
+        model_dir.mkdir(parents=True, exist_ok=True)
+
+        pipeline.steps[0][1].Doc = None
+
+        dump(pipeline, (model_dir / 'hotel_sentiment_classification.joblib').absolute(), compress=('gzip', 9))
+
+        console.log("Model save [green]DONE[/green]")
+
+
+def load(model_name="hotel_sentiment_classification"):
+    return load_model(model_name)
+
+
+def evaluate(model=None):
+    try:
+        import pandas as pd
+    except ImportError:
+        console.log(("pandas package is not a general sadedegel dependency."
+                     " But we do have a dependency on building our prebuilt models"))
+
+    if model is None:
+        model = load()
+
+    test = pd.DataFrame.from_records(load_hotel_sentiment_test())
+    testLabel = pd.DataFrame.from_records(load_hotel_sentiment_test_label())
+
+    test = test.merge(testLabel, on='id')
+
+    y_pred = model.predict(test.text)
+
+    console.log(f"Model test accuracy (accuracy): {accuracy_score(test.sentiment_class, y_pred)}")
+
+
+if __name__ == "__main__":
+    build(save=True)
diff --git a/sadedegel/prebuilt/model/hotel_sentiment_classification.joblib b/sadedegel/prebuilt/model/hotel_sentiment_classification.joblib
new file mode 100644
index 00000000..1de92061
--- /dev/null
+++ b/sadedegel/prebuilt/model/hotel_sentiment_classification.joblib
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d8d67ad170a0081c7684798b8c74d3840c3a4222db1ee9b07317405c33410e8
+size 51013


Test patch:
diff --git a/tests/datasets/test_hotel_sentiment.py b/tests/datasets/test_hotel_sentiment.py
index e8b81a09..595efa70 100644
--- a/tests/datasets/test_hotel_sentiment.py
+++ b/tests/datasets/test_hotel_sentiment.py
@@ -6,7 +6,7 @@
 from .context import hotel_sentiment
 
 
-@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data")).exists()')
+@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data/hotel_sentiment")).exists()')
 def test_data_load():
     data = hotel_sentiment.load_hotel_sentiment_train()
     for i, row in enumerate(data):
@@ -16,7 +16,7 @@ def test_data_load():
         assert hotel_sentiment.CLASS_VALUES[row['sentiment_class']] in ['POSITIVE', 'NEGATIVE']
     assert i + 1 == 5800
 
-@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data")).exists()')
+@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data/hotel_sentiment")).exists()')
 def test_data_load_test():
     data = hotel_sentiment.load_hotel_sentiment_test()
     for i, row in enumerate(data):
@@ -25,7 +25,7 @@ def test_data_load_test():
         assert isinstance(row['text'], str)
     assert i +1 == 5800
 
-@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data")).exists()')
+@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data/hotel_sentiment")).exists()')
 def test_data_load_label():
     data = hotel_sentiment.load_hotel_sentiment_test_label()
     for i, row in enumerate(data):
diff --git a/tests/prebuilt/context.py b/tests/prebuilt/context.py
index 916cfe0e..008fcc14 100644
--- a/tests/prebuilt/context.py
+++ b/tests/prebuilt/context.py
@@ -7,8 +7,9 @@
 from sadedegel.prebuilt import news_classification, tweet_profanity, telco_sentiment  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.tscorpus import CATEGORIES # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.profanity import CLASS_VALUES # noqa # pylint: disable=unused-import, wrong-import-position
-from sadedegel.prebuilt import tweet_sentiment , movie_reviews, customer_reviews_classification # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.prebuilt import tweet_sentiment , movie_reviews, customer_reviews_classification, hotel_sentiment # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.tweet_sentiment import CLASS_VALUES as SENTIMENT_VALUES  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.movie_sentiment import CLASS_VALUES as SENTIMENT_VALUES_M  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.telco_sentiment import CLASS_VALUES as SENTIMENT_VALUES_T  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.customer_review import CLASS_VALUES as CLASS_VALUES_CUST # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.dataset.hotel_sentiment import CLASS_VALUES as CLASS_VALUES_HOTEL # noqa # pylint: disable=unused-import, wrong-import-position
diff --git a/tests/prebuilt/test_hotel_sentiment.py b/tests/prebuilt/test_hotel_sentiment.py
new file mode 100644
index 00000000..c6fcc7e1
--- /dev/null
+++ b/tests/prebuilt/test_hotel_sentiment.py
@@ -0,0 +1,22 @@
+import pytest
+from .context import hotel_sentiment, CLASS_VALUES_HOTEL
+
+
+def test_model_load():
+    _ = hotel_sentiment.load()
+
+
+def test_inference():
+    model = hotel_sentiment.load()
+
+    pred = model.predict(['asla gidilmeyecek bir otel hasta olduk otel tam anlamıyla bir fiyasko satın alırken',
+                          'Çok güzeldi 4 cü gidişimiz tesise cok memnun kaldık.'])
+
+    assert CLASS_VALUES_HOTEL[pred[0]] == 'NEGATIVE'
+    assert CLASS_VALUES_HOTEL[pred[1]] == 'POSITIVE'
+
+    probability = model.predict_proba(
+        ['asla gidilmeyecek bir otel hasta olduk otel tam anlamıyla bir fiyasko satın alırken',
+         'Çok güzeldi 4 cü gidişimiz tesise cok memnun kaldık.'])
+
+    assert probability.shape == (2, 2)