Repository: flairNLP/flair

Issue:
Document how to run tests locally (py.test)
The local setup (virtualenv / pipenv) and how to run tests (py.test) should be clearly documented. 

Additionally I would propose to change the default config for tests to exclude the external files (downloads from S3). Currently the env var `TRAVIS=true` needs to be set in order to exclude external S3 files.


Discussion:

Resolving patch:
diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py
--- a/flair/data_fetcher.py
+++ b/flair/data_fetcher.py
@@ -3,6 +3,7 @@
 import os
 import logging
 from enum import Enum
+from pathlib import Path
 
 from flair.data import Sentence, TaggedCorpus, Token
 
@@ -35,17 +36,22 @@ class NLPTask(Enum):
 class NLPTaskDataFetcher:
 
     @staticmethod
-    def fetch_data(task: NLPTask) -> TaggedCorpus:
+    def fetch_data(task: NLPTask, base_path: Path = None) -> TaggedCorpus:
         """
         Helper function to fetch a TaggedCorpus for a specific NLPTask. For this to work you need to first download
         and put into the appropriate folder structure the corresponsing NLP task data. The tutorials on
         https://github.com/zalandoresearch/flair give more info on how to do this. Alternatively, you can use this
         code to create your own data fetchers.
         :param task: specification of the NLPTask you wish to get
+        :param base_path: path to data folder containing tasks sub folders
         :return: a TaggedCorpus consisting of train, dev and test data
         """
 
-        data_folder = os.path.join('resources', 'tasks', str(task.value).lower())
+        if not base_path:
+            base_path = Path('resources') / 'tasks'
+
+        data_folder = str(base_path / str(task.value).lower())
+
         log.info("Reading data from {}".format(data_folder))
 
         # the CoNLL 2000 task on chunking has three columns: text, pos and np (chunk)
@@ -177,7 +183,7 @@ def fetch_data(task: NLPTask) -> TaggedCorpus:
 
     @staticmethod
     def fetch_column_corpus(
-            data_folder: str,
+            data_folder: Path,
             column_format: Dict[int, str],
             train_file: str,
             test_file: str,
@@ -195,6 +201,9 @@ def fetch_column_corpus(
         :return: a TaggedCorpus with annotated train, dev and test data
         """
 
+        # TODO: move all paths to use pathlib.Path, for now convert to str for compatibility
+        data_folder = str(data_folder)
+
         # get train and test data
         sentences_train: List[Sentence] = NLPTaskDataFetcher.read_column_data(
             os.path.join(data_folder, train_file), column_format)


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,10 @@
+import pytest
+from pathlib import Path
+
+@pytest.fixture(scope="module")
+def resources_path():
+    return Path(__file__).parent / 'resources'
+
+@pytest.fixture(scope="module")
+def tasks_base_path(resources_path):
+    return resources_path / 'tasks'
diff --git a/tests/test_data.py b/tests/test_data.py
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -47,11 +47,11 @@ def test_sentence_to_plain_string():
     assert ('I love Berlin .' == sentence.to_tokenized_string())
 
 
-def test_sentence_to_real_string():
+def test_sentence_to_real_string(tasks_base_path):
     sentence: Sentence = Sentence('I love Berlin.', use_tokenizer=True)
     assert ('I love Berlin.' == sentence.to_plain_string())
 
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.GERMEVAL)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.GERMEVAL, tasks_base_path)
 
     sentence = corpus.train[0]
     assert (
@@ -501,4 +501,4 @@ def test_sentence_to_dict():
     assert ('Facebook, Inc. is a company, and Google is one as well.' == dict['text'])
     assert ('Facebook, Inc.' == dict['entities'][0]['text'])
     assert ('Google' == dict['entities'][1]['text'])
-    assert (0 == len(dict['labels']))
\ No newline at end of file
+    assert (0 == len(dict['labels']))
diff --git a/tests/test_data_fetchers.py b/tests/test_data_fetchers.py
--- a/tests/test_data_fetchers.py
+++ b/tests/test_data_fetchers.py
@@ -1,46 +1,46 @@
 from flair.data_fetcher import NLPTask, NLPTaskDataFetcher
 
 
-def test_load_imdb_data():
+def test_load_imdb_data(tasks_base_path):
     # get training, test and dev data
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, tasks_base_path)
 
     assert len(corpus.train) == 5
     assert len(corpus.dev) == 5
     assert len(corpus.test) == 5
 
 
-def test_load_ag_news_data():
+def test_load_ag_news_data(tasks_base_path):
     # get training, test and dev data
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.AG_NEWS)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.AG_NEWS, tasks_base_path)
 
     assert len(corpus.train) == 10
     assert len(corpus.dev) == 10
     assert len(corpus.test) == 10
 
 
-def test_load_sequence_labeling_data():
+def test_load_sequence_labeling_data(tasks_base_path):
     # get training, test and dev data
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION, tasks_base_path)
 
     assert len(corpus.train) == 6
     assert len(corpus.dev) == 1
     assert len(corpus.test) == 1
 
 
-def test_load_germeval_data():
+def test_load_germeval_data(tasks_base_path):
     # get training, test and dev data
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.GERMEVAL)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.GERMEVAL, tasks_base_path)
 
     assert len(corpus.train) == 2
     assert len(corpus.dev) == 1
     assert len(corpus.test) == 1
 
 
-def test_load_ud_english_data():
+def test_load_ud_english_data(tasks_base_path):
     # get training, test and dev data
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.UD_ENGLISH)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.UD_ENGLISH, tasks_base_path)
 
     assert len(corpus.train) == 6
     assert len(corpus.test) == 4
-    assert len(corpus.dev) == 2
\ No newline at end of file
+    assert len(corpus.dev) == 2
diff --git a/tests/test_language_model_trainer.py b/tests/test_language_model_trainer.py
--- a/tests/test_language_model_trainer.py
+++ b/tests/test_language_model_trainer.py
@@ -1,4 +1,5 @@
 import shutil
+from pathlib import Path
 
 from flair.data import Dictionary, Sentence
 from flair.embeddings import CharLMEmbeddings
@@ -14,7 +15,7 @@ def test_training():
     language_model: LanguageModel = LanguageModel(dictionary, is_forward_lm=True, hidden_size=128, nlayers=1)
 
     # get the example corpus and process at character level in forward direction
-    corpus: TextCorpus = TextCorpus('resources/corpora/lorem_ipsum',
+    corpus: TextCorpus = TextCorpus(str(Path(__file__).parent / 'resources/corpora/lorem_ipsum'),
                                     dictionary,
                                     language_model.is_forward_lm,
                                     character_level=True)
diff --git a/tests/test_sequence_labeler_trainer.py b/tests/test_sequence_labeler_trainer.py
--- a/tests/test_sequence_labeler_trainer.py
+++ b/tests/test_sequence_labeler_trainer.py
@@ -6,9 +6,9 @@
 from flair.trainers import SequenceTaggerTrainer
 
 
-def test_training():
+def test_training(tasks_base_path):
 
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION, tasks_base_path)
     tag_dictionary = corpus.make_tag_dictionary('ner')
 
     embeddings = WordEmbeddings('glove')
diff --git a/tests/test_text_classifier.py b/tests/test_text_classifier.py
--- a/tests/test_text_classifier.py
+++ b/tests/test_text_classifier.py
@@ -8,8 +8,8 @@
 
 
 @pytest.fixture
-def init() -> Tuple[TaggedCorpus, Dictionary, TextClassifier]:
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.AG_NEWS)
+def init(tasks_base_path) -> Tuple[TaggedCorpus, Dictionary, TextClassifier]:
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.AG_NEWS, tasks_base_path)
     label_dict = corpus.make_label_dictionary()
 
     glove_embedding: WordEmbeddings = WordEmbeddings('en-glove')
@@ -20,8 +20,8 @@ def init() -> Tuple[TaggedCorpus, Dictionary, TextClassifier]:
     return corpus, label_dict, model
 
 
-def test_labels_to_indices():
-    corpus, label_dict, model = init()
+def test_labels_to_indices(tasks_base_path):
+    corpus, label_dict, model = init(tasks_base_path)
 
     result = model._labels_to_indices(corpus.train)
 
@@ -32,8 +32,8 @@ def test_labels_to_indices():
         assert(expected == actual)
 
 
-def test_labels_to_one_hot():
-    corpus, label_dict, model = init()
+def test_labels_to_one_hot(tasks_base_path):
+    corpus, label_dict, model = init(tasks_base_path)
 
     result = model._labels_to_one_hot(corpus.train)
 
@@ -45,4 +45,4 @@ def test_labels_to_one_hot():
             if idx == expected:
                 assert(actual[idx] == 1)
             else:
-                assert(actual[idx] == 0)
\ No newline at end of file
+                assert(actual[idx] == 0)
diff --git a/tests/test_text_classifier_trainer.py b/tests/test_text_classifier_trainer.py
--- a/tests/test_text_classifier_trainer.py
+++ b/tests/test_text_classifier_trainer.py
@@ -8,8 +8,8 @@
 from flair.trainers.text_classification_trainer import TextClassifierTrainer
 
 
-def test_text_classifier_single_label():
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB)
+def test_text_classifier_single_label(tasks_base_path):
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, tasks_base_path)
     label_dict = corpus.make_label_dictionary()
 
     glove_embedding: WordEmbeddings = WordEmbeddings('en-glove')
@@ -32,8 +32,8 @@ def test_text_classifier_single_label():
     shutil.rmtree('./results')
 
 
-def test_text_classifier_mulit_label():
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB)
+def test_text_classifier_mulit_label(tasks_base_path):
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, tasks_base_path)
     label_dict = corpus.make_label_dictionary()
 
     glove_embedding: WordEmbeddings = WordEmbeddings('en-glove')
@@ -53,4 +53,4 @@ def test_text_classifier_mulit_label():
             assert(type(l.score) is float)
 
     # clean up results directory
-    shutil.rmtree('./results')
\ No newline at end of file
+    shutil.rmtree('./results')
diff --git a/tests/test_visual.py b/tests/test_visual.py
--- a/tests/test_visual.py
+++ b/tests/test_visual.py
@@ -13,7 +13,7 @@
 
 
 @pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
-def test_visualize_word_emeddings():
+def test_visualize_word_emeddings(resources_path):
 
     with open('./resources/visual/snippet.txt') as f:
         sentences = [x for x in f.read().split('\n') if x]
@@ -80,25 +80,25 @@ def test_visualize():
     os.remove('./resources/visual/char_embeddings.html')
 
 
-def test_highlighter():
-    with open('./resources/visual/snippet.txt') as f:
+def test_highlighter(resources_path):
+    with (resources_path / 'visual/snippet.txt').open() as f:
         sentences = [x for x in f.read().split('\n') if x]
 
     embeddings = CharLMEmbeddings('news-forward')
 
     features = embeddings.lm.get_representation(sentences[0]).squeeze()
 
-    Highlighter().highlight_selection(features, sentences[0], n=1000, file_='./resources/visual/highligh.html')
+    Highlighter().highlight_selection(features, sentences[0], n=1000, file_=str(resources_path / 'visual/highligh.html'))
 
     # clean up directory
-    os.remove('./resources/visual/highligh.html')
+    (resources_path / 'visual/highligh.html').unlink()
 
 
-def test_plotting_training_curves_and_weights():
+def test_plotting_training_curves_and_weights(resources_path):
     plotter = Plotter()
-    plotter.plot_training_curves('./resources/visual/loss.tsv')
-    plotter.plot_weights('./resources/visual/weights.txt')
+    plotter.plot_training_curves(resources_path / 'visual/loss.tsv')
+    plotter.plot_weights(resources_path / 'visual/weights.txt')
 
     # clean up directory
-    os.remove('./resources/visual/weights.png')
-    os.remove('./resources/visual/training.png')
\ No newline at end of file
+    (resources_path / 'visual/weights.png').unlink()
+    (resources_path / 'visual/training.png').unlink()