Repository: GlobalMaksimum/sadedegel (language: python)

Issue:
Implement Dataset Download for Tweet Sentiment Prebuilt Model
- Dataset is deleted from feature branch by @ertugrul-dmr .
- `build()` needs dataset imported as a `sadedegel.dataset`.
- Implement dataset import similar to `sadedegel.dataset.profanity` 


Discussion / hints:

Fix patch:
diff --git a/sadedegel/dataset/tweet_sentiment/__init__.py b/sadedegel/dataset/tweet_sentiment/__init__.py
new file mode 100644
index 00000000..ee9381f5
--- /dev/null
+++ b/sadedegel/dataset/tweet_sentiment/__init__.py
@@ -0,0 +1,2 @@
+from ._core import load_tweet_sentiment_train
+from ._core import CLASS_VALUES, CORPUS_SIZE
\ No newline at end of file
diff --git a/sadedegel/dataset/tweet_sentiment/__main__.py b/sadedegel/dataset/tweet_sentiment/__main__.py
new file mode 100644
index 00000000..e13169c4
--- /dev/null
+++ b/sadedegel/dataset/tweet_sentiment/__main__.py
@@ -0,0 +1,86 @@
+import os.path
+from pathlib import Path
+import sys
+from itertools import tee
+
+from smart_open import open
+
+import click
+
+import boto3
+
+from loguru import logger
+
+from ._core import load_tweet_sentiment_train, CORPUS_SIZE, CLASS_VALUES
+
+from zipfile import ZipFile
+
+from rich.console import Console
+
+console = Console()
+
+logger.disable("sadedegel")
+
+
+@click.group(help="Twitter Sentiment Dataset Commandline")
+def cli():
+    pass
+
+
+@cli.command()
+@click.option("--access-key", help="Access Key ID to download dataset.", prompt=True,
+              default=lambda: os.environ.get('sadedegel_access_key', ''))
+@click.option("--secret-key", help="Secret Key ID to download dataset.", prompt=True,
+              default=lambda: os.environ.get('sadedegel_secret_key', ''))
+@click.option("--data_home", '-d', help="Data home directory", default="~/.sadedegel_data")
+def download(access_key, secret_key, data_home):
+    """Download twitter sentiment corpus from cloud with your key."""
+
+    data_home = Path(os.path.expanduser(data_home)) / 'tweet_sentiment'
+    data_home.mkdir(parents=True, exist_ok=True)
+    console.print(f"Data directory for twitter sentiment data {data_home}")
+
+    transport_params = {
+        'session': boto3.Session(aws_access_key_id=access_key,
+                                 aws_secret_access_key=secret_key),
+        'resource_kwargs': {
+            'endpoint_url': 'https://storage.googleapis.com',
+        }
+    }
+
+    url = f"s3://sadedegel/dataset/tweet_sentiment_v1.zip"
+
+    with open(url, 'rb', transport_params=transport_params) as fp:
+        with ZipFile(fp) as zp:
+            zp.extractall(data_home)
+
+
+@cli.command()
+def validate():
+    """Sanity check on corpus
+    """
+    with console.status("[bold yellow]Validating train"):
+        train = load_tweet_sentiment_train()
+
+        train_clone, train = tee(train, 2)
+
+        n_train = sum(1 for _ in train_clone)
+        categories = set([row['sentiment'] for row in train])
+
+        if n_train == CORPUS_SIZE:
+            console.log("Cardinality check [yellow]DONE[/yellow]")
+        else:
+            console.log("Cardinality check [red]FAILED[/red]")
+            console.log(f"|Tweet sentiment (train)| : {n_train}")
+            sys.exit(1)
+
+        if categories == set(['POSITIVE', 'NEGATIVE']):
+            console.log("Label check [yellow]DONE[/yellow]")
+        else:
+            console.log("Cardinality check [red]FAILED[/red]")
+            console.log(f"|Tweet sentiment labels (train)| : {categories}")
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    cli()
diff --git a/sadedegel/dataset/tweet_sentiment/_core.py b/sadedegel/dataset/tweet_sentiment/_core.py
new file mode 100644
index 00000000..e51317bb
--- /dev/null
+++ b/sadedegel/dataset/tweet_sentiment/_core.py
@@ -0,0 +1,51 @@
+import csv
+import gzip
+from pathlib import Path
+from rich.console import Console
+
+CLASS_VALUES = ['POSITIVE', 'NEGATIVE']
+CORPUS_SIZE = 11117
+
+console = Console()
+
+__general_download_message__ = """Ensure that you have properly downloaded tweet sentiment corpus using
+
+            python -m sadedegel.dataset.tweet_sentiment download --access-key xxx --secret-key xxxx
+
+        Unfortunately due to data licensing issues we could not share data publicly. 
+        Get in touch with sadedegel team to obtain a download key.
+        """
+
+
+def check_directory_structure(path: str) -> bool:
+    base_dir = Path(path).expanduser()
+
+    tweet_sentiment_dir = base_dir / 'tweet_sentiment'
+
+    if not base_dir.exists():
+        console.log(f"Dataset base directory ([bold red]{base_dir}[/bold red]) does not exist")
+
+    elif not tweet_sentiment_dir.exists():
+        console.log(
+            f"Tweet sentiment directory ([bold red]{tweet_sentiment_dir}[/bold red]) does not exist")
+
+    else:
+        return True
+
+    console.log(__general_download_message__)
+
+    return False
+
+
+def load_tweet_sentiment_train(data_home="~/.sadedegel_data"):
+    if not check_directory_structure(data_home):
+        raise Exception("Tweet Sentiment Corpus validation error")
+
+    train_csv = Path(data_home).expanduser() / "tweet_sentiment" / "tweet_sentiment_v1"
+    train_csv = train_csv / "tweet_sentiment_train.csv.gz"
+
+    with gzip.open(train_csv, "rt") as csvfile:
+        rd = csv.DictReader(csvfile)
+
+        for rec in rd:
+            yield dict(text_uuid=rec['text_uuid'], text=rec['text'], sentiment=CLASS_VALUES.index(rec['sentiment']))


Test patch:
diff --git a/tests/datasets/context.py b/tests/datasets/context.py
index b6eca895..8db9cfcc 100644
--- a/tests/datasets/context.py
+++ b/tests/datasets/context.py
@@ -6,6 +6,7 @@
 from sadedegel.dataset import load_raw_corpus, load_sentence_corpus,load_annotated_corpus # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.extended import load_extended_metadata, load_extended_sents_corpus, load_extended_raw_corpus  # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset.tscorpus import load_tokenization_raw,load_tokenization_tokenized, check_and_display, CORPUS_SIZE # noqa # pylint: disable=unused-import, wrong-import-position
+from sadedegel.dataset.tweet_sentiment import load_tweet_sentiment_train, CLASS_VALUES # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.bblock.cli.__main__ import tok_eval # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset import util # noqa # pylint: disable=unused-import, wrong-import-position
 from sadedegel.dataset import file_paths, CorpusTypeEnum # noqa # pylint: disable=unused-import, wrong-import-position
diff --git a/tests/datasets/test_tweet_sentiment.py b/tests/datasets/test_tweet_sentiment.py
new file mode 100644
index 00000000..a0c583c4
--- /dev/null
+++ b/tests/datasets/test_tweet_sentiment.py
@@ -0,0 +1,11 @@
+from .context import load_tweet_sentiment_train, CLASS_VALUES
+
+
+def test_data_load():
+    data = load_tweet_sentiment_train()
+    for i, row in enumerate(data):
+        assert any(key in row.keys() for key in ['text_uuid', 'text', 'sentiment'])
+        assert isinstance(row['text_uuid'], str)
+        assert isinstance(row['text'], str)
+        assert CLASS_VALUES[row['sentiment']] in ['POSITIVE', 'NEGATIVE']
+    assert i == 11116