Repository: GlobalMaksimum/sadedegel (language: python)

Issue:
Food Delivery Reviews Classification Prebuilt Model
For the purpose of adding different use cases of our prebuilt function we could implement a food delivery review classification model. There are decent datasets available on kaggle [here](https://www.kaggle.com/berkaycokuner/yemek-sepeti-comments) and [here](https://www.kaggle.com/dgknrsln/yorumsepeti) (we're going to merge both).

- In total both datasets contians 550K+ instances after preprocessing and removing duplicates.
- Data is collected from Turkish food delivery web sites.
- The data contains reviews about the food delivery and scoring for several aspects of the service.


For this, these steps going be taken:

- Prepare the dataset for sentiment classification,
- Build/optimize a model and test model accuracy,
- Publish the public prebuilt model


Discussion / hints:

Fix patch:
diff --git a/sadedegel/dataset/food_review/__init__.py b/sadedegel/dataset/food_review/__init__.py
new file mode 100644
index 00000000..084a94df
--- /dev/null
+++ b/sadedegel/dataset/food_review/__init__.py
@@ -0,0 +1,1 @@
+from ._core import load_food_review_train, load_food_review_test, CLASS_VALUES
diff --git a/sadedegel/dataset/food_review/__main__.py b/sadedegel/dataset/food_review/__main__.py
new file mode 100644
index 00000000..9d7f6146
--- /dev/null
+++ b/sadedegel/dataset/food_review/__main__.py
@@ -0,0 +1,85 @@
+import os.path
+from pathlib import Path
+import sys
+from itertools import tee
+from smart_open import open
+import click
+import boto3
+from loguru import logger
+from ._core import load_food_review_train, load_food_review_test, CLASS_VALUES, TRAIN_SIZE, TEST_SIZE
+from zipfile import ZipFile
+from rich.console import Console
+
+console = Console()
+
+logger.disable("sadedegel")
+
+
+@click.group(help="Food Review Dataset Commandline")
+def cli():
+    pass
+
+
+@cli.command()
+@click.option("--access-key", help="Access Key ID to download dataset.", prompt=True,
+              default=lambda: os.environ.get('sadedegel_access_key', ''))
+@click.option("--secret-key", help="Secret Key ID to download dataset.", prompt=True,
+              default=lambda: os.environ.get('sadedegel_secret_key', ''))
+@click.option("--data_home", '-d', help="Data home directory", default="~/.sadedegel_data")
+def download(access_key, secret_key, data_home):
+    """Download tokenization corpus from cloud with your key."""
+
+    data_home = Path(os.path.expanduser(data_home))
+    data_home.mkdir(parents=True, exist_ok=True)
+    console.print(f"Data directory for  data {data_home}")
+
+    transport_params = {
+        'session': boto3.Session(aws_access_key_id=access_key,
+                                 aws_secret_access_key=secret_key),
+        'resource_kwargs': {
+            'endpoint_url': 'https://storage.googleapis.com',
+        }
+    }
+
+    url = f"s3://sadedegel/dataset/food_review.zip"
+
+    with open(url, 'rb', transport_params=transport_params) as fp:
+        with ZipFile(fp) as zp:
+            zp.extractall(data_home)
+
+
+@cli.command()
+def validate():
+    """Sanity check on corpus
+    """
+    with console.status("[bold yellow]Validating train"):
+        train = load_food_review_train()
+
+        train_clone, train = tee(train, 2)
+
+        n_train = sum(1 for _ in train_clone)
+
+        if n_train == TRAIN_SIZE:
+            console.log("Cardinality check [yellow]DONE[/yellow]")
+        else:
+            console.log("Cardinality check [red]FAILED[/red]")
+            console.log(f"| (train)| : {n_train}")
+            sys.exit(1)
+
+    with console.status("[bold yellow]Validate test"):
+        test = set((d['id'] for d in load_food_review_test()))
+
+        test_clone, test = tee(test, 2)
+
+        n_test = sum(1 for _ in test_clone)
+
+        if n_test == TEST_SIZE:
+            console.log("Cardinality check [yellow]DONE[/yellow]")
+        else:
+            console.log("Cardinality check [red]FAILED[/red]")
+            console.log(f"| (train)| : {n_test}")
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    cli()
diff --git a/sadedegel/dataset/food_review/_core.py b/sadedegel/dataset/food_review/_core.py
new file mode 100644
index 00000000..011844eb
--- /dev/null
+++ b/sadedegel/dataset/food_review/_core.py
@@ -0,0 +1,76 @@
+from rich.console import Console
+from pathlib import Path
+import gzip
+import csv
+
+
+TRAIN_SIZE, TEST_SIZE = 470396, 117599
+CLASS_VALUES = ["NEGATIVE", "POSITIVE"]
+
+console = Console()
+
+__general_download_message__ = """Ensure that you have properly downloaded food review corpus using
+
+            python -m sadedegel.dataset.food_review download --access-key xxx --secret-key xxxx
+
+        Unfortunately due to data licensing issues we could not share data publicly. 
+        Get in touch with sadedegel team to obtain a download key.
+        """
+
+
+def check_directory_structure(path: str) -> bool:
+    base_dir = Path(path).expanduser()
+
+    dataset_dir = base_dir / 'food_review'
+
+    if not base_dir.exists():
+        console.log(f"Dataset base directory ([bold red]{base_dir}[/bold red]) does not exist")
+
+    elif not dataset_dir.exists():
+        console.log(
+            f" directory ([bold red]{dataset_dir}[/bold red]) does not exist")
+
+    else:
+        return True
+
+    console.log(__general_download_message__)
+
+    return False
+
+
+def load_food_review_train(data_home="~/.sadedegel_data"):
+    if not check_directory_structure(data_home):
+        raise Exception(" validation error")
+
+    train_csv = Path(data_home).expanduser() / "food_review"
+    train_csv = train_csv / "food_reviews_train.csv.gz"
+
+    with gzip.open(train_csv, "rt") as csvfile:
+        rd = csv.DictReader(csvfile)
+
+        for rec in rd:
+            yield dict(id=rec['text_uuid'],
+                       text=rec['review'],
+                       sentiment_class=int(rec['sentiment_class']),
+                       speed=int(rec['speed']),
+                       service=int(rec['service']),
+                       flavour=int(rec['flavour']))
+
+
+def load_food_review_test(data_home="~/.sadedegel_data"):
+    if not check_directory_structure(data_home):
+        raise Exception(" validation error")
+
+    train_csv = Path(data_home).expanduser() / "food_review"
+    train_csv = train_csv / "food_reviews_test.csv.gz"
+
+    with gzip.open(train_csv, "rt") as csvfile:
+        rd = csv.DictReader(csvfile)
+
+        for rec in rd:
+            yield dict(id=rec['text_uuid'],
+                       text=rec['review'],
+                       sentiment_class=int(rec['sentiment_class']),
+                       speed=int(rec['speed']),
+                       service=int(rec['service']),
+                       flavour=int(rec['flavour']))
diff --git a/sadedegel/prebuilt/README.md b/sadedegel/prebuilt/README.md
index a4f65405..bbaea45f 100644
--- a/sadedegel/prebuilt/README.md
+++ b/sadedegel/prebuilt/README.md
@@ -165,3 +165,31 @@ Current prebuilt customer review classification model has a macro-F1 score of `0
 
 If you want to compare benchmark results:
 > The model on [Kaggle](https://www.kaggle.com/savasy/multiclass-classification-data-for-turkish-tc32) where we got dataset from has F1 score of `0.84`.
+
+### Online Food Delivery Review Sentiment Classification
+Classifier assigns each Turkish review texts into two classes ('NEGATIVE', 'POSITIVE') by using sadedegel buil-in pipeline.
+
+#### Loading and Predicting with the Model:
+```python
+from sadedegel.prebuilt import food_reviews
+# We load our prebuilt model:
+model = food_reviews.load()
+
+# Here we enter our text to get sentiment predictions.
+y_pred = model.predict(['sıradan bir yemek yorumu'])
+
+# You can also get probabilities with:
+y_probs = model.predict_proba(['sıradan bir yemek yorumu'])
+
+# You can check original test results on holdout set:
+food_reviews.evaluate()
+
+# You can conver class id's back to labels by importing:
+from sadedegel.dataset.food_review import CLASS_VALUES
+
+# And simply map them to get converted string list:
+y_pred_label = [CLASS_VALUES[idx] for idx in y_pred]
+```
+
+#### Accuracy
+Current prebuilt food review classification model has a macro-F1 score of `0.896` on holdout test set model never seen before.
diff --git a/sadedegel/prebuilt/food_reviews.py b/sadedegel/prebuilt/food_reviews.py
new file mode 100644
index 00000000..30f9bda6
--- /dev/null
+++ b/sadedegel/prebuilt/food_reviews.py
@@ -0,0 +1,88 @@
+from os.path import dirname
+from pathlib import Path
+
+from joblib import dump
+from rich.console import Console
+from sklearn.linear_model import SGDClassifier
+from sklearn.metrics import f1_score, roc_auc_score
+from sklearn.utils import shuffle
+from sklearn.pipeline import Pipeline
+
+from ..extension.sklearn import Text2Doc, HashVectorizer
+from .util import load_model
+
+from ..dataset.food_review import load_food_review_train, load_food_review_test
+
+console = Console()
+
+
+def empty_model():
+    return Pipeline(
+        [('text2doc', Text2Doc('icu')),
+         ('hash', HashVectorizer(n_features=1050000)),
+         ('sgd', SGDClassifier(loss='log', alpha= 0.00018808839341359152, penalty='elasticnet',
+                               learning_rate='optimal'))
+         ]
+    )
+
+
+def build(max_rows=-1, save=True):
+    try:
+        import pandas as pd
+    except ImportError:
+        console.log(("pandas package is not a general sadedegel dependency."
+                     " But we do have a dependency on building our prebuilt models"))
+
+    raw = load_food_review_train()
+    df = pd.DataFrame.from_records(raw)
+    df = shuffle(df, random_state=42)
+
+    if max_rows > 0:
+        df = df.sample(max_rows, random_state=42)
+
+    console.log(f"Corpus Size: {len(df)}")
+
+    pipeline = empty_model()
+
+    pipeline.fit(df.text, df.sentiment_class)
+
+    console.log("Model build [green]DONE[/green]")
+
+    if save:
+        model_dir = Path(dirname(__file__)) / 'model'
+
+        model_dir.mkdir(parents=True, exist_ok=True)
+
+        pipeline.steps[0][1].Doc = None
+
+        dump(pipeline, (model_dir / 'food_sentiment.joblib').absolute(), compress=('gzip', 9))
+
+def load(model_name='food_sentiment'):
+    return load_model(model_name)
+
+def evaluate(scoring='f1', max_rows=-1):
+    try:
+        import pandas as pd
+    except ImportError:
+        console.log(("pandas package is not a general sadedegel dependency."
+                     " But we do have a dependency on building our prebuilt models"))
+    model = load()
+
+    raw_test = load_food_review_test()
+    test = pd.DataFrame.from_records(raw_test)
+    true_labels = test.sentiment_class
+
+    if max_rows > 0:
+        test = test[:max_rows]
+        true_labels = true_labels[:max_rows]
+
+    if scoring=='f1':
+        y_pred = model.predict(test.text)
+        console.log(f"Model test accuracy (f1-macro): {f1_score(true_labels, y_pred, average='macro')}")
+    if scoring=='auc':
+        y_pred = model.predict_proba(test.text)
+        console.log(f"Model test roc_auc score: {roc_auc_score(true_labels, y_pred[:,1])}")
+
+
+if __name__ == "__main__":
+    build(save=True)
\ No newline at end of file
diff --git a/sadedegel/prebuilt/model/food_sentiment.joblib b/sadedegel/prebuilt/model/food_sentiment.joblib
new file mode 100644
index 00000000..697091e2
--- /dev/null
+++ b/sadedegel/prebuilt/model/food_sentiment.joblib
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0a768a9301de2d3daa5c922e52bc1e563625409d572fabf0358cdf3ccf1b86e
+size 36707


Test patch:
diff --git a/tests/datasets/context.py b/tests/datasets/context.py
index c1f7ee28..71ad1074 100644
--- a/tests/datasets/context.py
+++ b/tests/datasets/context.py
@@ -17,3 +17,5 @@
 from sadedegel.bblock.cli.__main__ import tok_eval # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset import util # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset import file_paths, CorpusTypeEnum # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.dataset.food_review import load_food_review_train, load_food_review_test # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.dataset.food_review import CLASS_VALUES as FOOD_CLASS_VALUES # noqa # pylint: disable=unused-import, wrong-import-position
diff --git a/tests/datasets/test_food_review.py b/tests/datasets/test_food_review.py
new file mode 100644
index 00000000..e1f86163
--- /dev/null
+++ b/tests/datasets/test_food_review.py
@@ -0,0 +1,44 @@
+from pathlib import Path  # pylint: disable=unused-import
+from os.path import expanduser  # pylint: disable=unused-import
+
+import pandas as pd
+import pytest
+from .context import load_food_review_train, load_food_review_test, FOOD_CLASS_VALUES
+
+
+@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data/food_review")).exists()')
+def test_dataset_import():
+    raw_train = load_food_review_train()
+    raw_test = load_food_review_test()
+
+    for i, rev in enumerate(raw_train):
+        assert isinstance(rev, dict)
+        assert rev.get("id") is not None
+        assert rev.get("text") is not None
+        assert rev.get("sentiment_class") is not None
+    assert i + 1 == 470396
+
+    for i, rev in enumerate(raw_test):
+        assert isinstance(rev, dict)
+        assert rev.get("id") is not None
+        assert rev.get("text") is not None
+        assert rev.get("sentiment_class") is not None
+    assert i + 1 == 117599
+
+
+@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data/food_review")).exists()')
+def test_sentiment_class():
+    assert FOOD_CLASS_VALUES[0] == "NEGATIVE"
+    assert FOOD_CLASS_VALUES[1] == "POSITIVE"
+
+
+@pytest.mark.skipif('not Path(expanduser("~/.sadedegel_data/food_review")).exists()')
+def test_from_records():
+    raw_train = load_food_review_train()
+    raw_test = load_food_review_test()
+
+    train_df = pd.DataFrame().from_records(raw_train)
+    test_df = pd.DataFrame().from_records(raw_test)
+
+    assert set(train_df.columns.tolist()) == {"id", "text", "sentiment_class", "speed", "service", "flavour"}
+    assert set(test_df.columns.tolist()) == {"id", "text", "sentiment_class", "speed", "service", "flavour"}
diff --git a/tests/prebuilt/context.py b/tests/prebuilt/context.py
index 916cfe0e..90a3ed03 100644
--- a/tests/prebuilt/context.py
+++ b/tests/prebuilt/context.py
@@ -7,8 +7,9 @@
 from sadedegel.prebuilt import news_classification, tweet_profanity, telco_sentiment  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.tscorpus import CATEGORIES # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.profanity import CLASS_VALUES # noqa # pylint: disable=unused-import, wrong-import-position
-from sadedegel.prebuilt import tweet_sentiment , movie_reviews, customer_reviews_classification # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.prebuilt import tweet_sentiment , movie_reviews, customer_reviews_classification, food_reviews # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.tweet_sentiment import CLASS_VALUES as SENTIMENT_VALUES  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.movie_sentiment import CLASS_VALUES as SENTIMENT_VALUES_M  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.telco_sentiment import CLASS_VALUES as SENTIMENT_VALUES_T  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.customer_review import CLASS_VALUES as CLASS_VALUES_CUST # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.dataset.food_review import CLASS_VALUES as CLASS_VALUES_FOOD # noqa # pylint: disable=unused-import, wrong-import-position
diff --git a/tests/prebuilt/test_food_reviews.py b/tests/prebuilt/test_food_reviews.py
new file mode 100644
index 00000000..2cdf37a6
--- /dev/null
+++ b/tests/prebuilt/test_food_reviews.py
@@ -0,0 +1,12 @@
+from .context import food_reviews, CLASS_VALUES_FOOD
+
+def test_model_load():
+    model = food_reviews.load()
+    pred_neg = model.predict(['iki saatte geldi'])
+    pred_pos = model.predict(['müdavimi olduk'])
+
+    assert pred_neg == 0
+    assert pred_pos == 1
+
+    assert CLASS_VALUES_FOOD[pred_pos[0]] == 'POSITIVE'
+    assert CLASS_VALUES_FOOD[pred_neg[0]] == 'NEGATIVE'