Repository: flairNLP/flair

Issue:
Get rid of `TRAVIS` environment variable
As proposed in https://github.com/zalandoresearch/flair/pull/161, we should get rid of the `TRAVIS` env var to skip tests and instead have some flag to run all tests ("slow tests" or similar). The default unit tests should be self-contained and fast.


Discussion:

Resolving patch:
diff --git a/flair/embeddings.py b/flair/embeddings.py
--- a/flair/embeddings.py
+++ b/flair/embeddings.py
@@ -207,10 +207,10 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
                     word_embedding = self.precomputed_word_embeddings[token.text]
                 elif token.text.lower() in self.precomputed_word_embeddings:
                     word_embedding = self.precomputed_word_embeddings[token.text.lower()]
-                elif re.sub('\d', '#', token.text.lower()) in self.precomputed_word_embeddings:
-                    word_embedding = self.precomputed_word_embeddings[re.sub('\d', '#', token.text.lower())]
-                elif re.sub('\d', '0', token.text.lower()) in self.precomputed_word_embeddings:
-                    word_embedding = self.precomputed_word_embeddings[re.sub('\d', '0', token.text.lower())]
+                elif re.sub(r'\d', '#', token.text.lower()) in self.precomputed_word_embeddings:
+                    word_embedding = self.precomputed_word_embeddings[re.sub(r'\d', '#', token.text.lower())]
+                elif re.sub(r'\d', '0', token.text.lower()) in self.precomputed_word_embeddings:
+                    word_embedding = self.precomputed_word_embeddings[re.sub(r'\d', '0', token.text.lower())]
                 else:
                     word_embedding = np.zeros(self.embedding_length, dtype='float')
 


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,10 +1,43 @@
 import pytest
 from pathlib import Path
 
+
 @pytest.fixture(scope="module")
 def resources_path():
     return Path(__file__).parent / 'resources'
 
+
 @pytest.fixture(scope="module")
 def tasks_base_path(resources_path):
     return resources_path / 'tasks'
+
+
+@pytest.fixture(scope="module")
+def results_base_path(resources_path):
+    return resources_path / 'results'
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--runslow", action="store_true", default=False, help="run slow tests"
+    )
+    parser.addoption(
+        "--runintegration", action="store_true", default=False, help="run integration tests"
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow") and config.getoption("--runintegration"):
+        return
+
+    if not config.getoption("--runslow"):
+        skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+        for item in items:
+            if "slow" in item.keywords:
+                item.add_marker(skip_slow)
+
+    if not config.getoption("--runintegration"):
+        skip_integration = pytest.mark.skip(reason="need --runintegration option to run")
+        for item in items:
+            if "integration" in item.keywords:
+                item.add_marker(skip_integration)
diff --git a/tests/test_data.py b/tests/test_data.py
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -237,9 +237,6 @@ def test_label_set_confidence():
 
     assert (0.2 == label.score)
 
-    # with pytest.raises(ValueError):
-    #     label.name = ''
-
 
 def test_tagged_corpus_make_label_dictionary():
     sentence_1 = Sentence('sentence 1', labels=[Label('class_1')])
diff --git a/tests/test_embeddings.py b/tests/test_embeddings.py
--- a/tests/test_embeddings.py
+++ b/tests/test_embeddings.py
@@ -1,4 +1,3 @@
-import os
 import pytest
 
 from flair.embeddings import WordEmbeddings, TokenEmbeddings, CharLMEmbeddings, StackedEmbeddings, \
@@ -7,77 +6,77 @@
 from flair.data import Sentence
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_en_glove():
     load_and_apply_word_embeddings('en-glove')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_en_numberbatch():
     load_and_apply_word_embeddings('en-numberbatch')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_en_extvec():
     load_and_apply_word_embeddings('en-extvec')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_en_crawl():
     load_and_apply_word_embeddings('en-crawl')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_en_news():
     load_and_apply_word_embeddings('en-news')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_de_fasttext():
     load_and_apply_word_embeddings('de-fasttext')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_de_numberbatch():
     load_and_apply_word_embeddings('de-numberbatch')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_sv_fasttext():
     load_and_apply_word_embeddings('sv-fasttext')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_news_forward():
     load_and_apply_char_lm_embeddings('news-forward')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_news_backward():
     load_and_apply_char_lm_embeddings('news-backward')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_mix_forward():
     load_and_apply_char_lm_embeddings('mix-forward')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_mix_backward():
     load_and_apply_char_lm_embeddings('mix-backward')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_german_forward():
     load_and_apply_char_lm_embeddings('german-forward')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_german_backward():
     load_and_apply_char_lm_embeddings('german-backward')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_stacked_embeddings():
     text = 'I love Berlin.'
     sentence: Sentence = Sentence(text)
@@ -98,23 +97,12 @@ def test_stacked_embeddings():
         assert(len(token.get_embedding()) == 0)
 
 
-@pytest.fixture
-def init_document_embeddings():
-    text = 'I love Berlin. Berlin is a great place to live.'
-    sentence: Sentence = Sentence(text)
-
-    glove: TokenEmbeddings = WordEmbeddings('en-glove')
-    charlm: TokenEmbeddings = CharLMEmbeddings('mix-backward')
-
-    return sentence, glove, charlm
-
-
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_document_lstm_embeddings():
     sentence, glove, charlm = init_document_embeddings()
 
     embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings([glove, charlm], hidden_states=128,
-                                                                bidirectional=False, use_first_representation=False)
+                                                                bidirectional=False)
 
     embeddings.embed(sentence)
 
@@ -126,12 +114,12 @@ def test_document_lstm_embeddings():
     assert (len(sentence.get_embedding()) == 0)
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_document_bidirectional_lstm_embeddings():
     sentence, glove, charlm = init_document_embeddings()
 
     embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings([glove, charlm], hidden_states=128,
-                                                                bidirectional=True, use_first_representation=False)
+                                                                bidirectional=True)
 
     embeddings.embed(sentence)
 
@@ -143,12 +131,12 @@ def test_document_bidirectional_lstm_embeddings():
     assert (len(sentence.get_embedding()) == 0)
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_document_bidirectional_lstm_embeddings_using_first_representation():
     sentence, glove, charlm = init_document_embeddings()
 
     embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings([glove, charlm], hidden_states=128,
-                                                                bidirectional=True, use_first_representation=True)
+                                                                bidirectional=True)
 
     embeddings.embed(sentence)
 
@@ -160,12 +148,12 @@ def test_document_bidirectional_lstm_embeddings_using_first_representation():
     assert (len(sentence.get_embedding()) == 0)
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_document_lstm_embeddings_using_first_representation():
     sentence, glove, charlm = init_document_embeddings()
 
     embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings([glove, charlm], hidden_states=128,
-                                                                bidirectional=False, use_first_representation=True)
+                                                                bidirectional=False)
 
     embeddings.embed(sentence)
 
@@ -177,7 +165,7 @@ def test_document_lstm_embeddings_using_first_representation():
     assert (len(sentence.get_embedding()) == 0)
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_document_mean_embeddings():
     text = 'I love Berlin. Berlin is a great place to live.'
     sentence: Sentence = Sentence(text)
@@ -196,6 +184,16 @@ def test_document_mean_embeddings():
     assert (len(sentence.get_embedding()) == 0)
 
 
+def init_document_embeddings():
+    text = 'I love Berlin. Berlin is a great place to live.'
+    sentence: Sentence = Sentence(text)
+
+    glove: TokenEmbeddings = WordEmbeddings('en-glove')
+    charlm: TokenEmbeddings = CharLMEmbeddings('mix-backward')
+
+    return sentence, glove, charlm
+
+
 def load_and_apply_word_embeddings(emb_type: str):
     text = 'I love Berlin.'
     sentence: Sentence = Sentence(text)
@@ -221,4 +219,4 @@ def load_and_apply_char_lm_embeddings(emb_type: str):
 
         token.clear_embeddings()
 
-        assert(len(token.get_embedding()) == 0)
\ No newline at end of file
+        assert(len(token.get_embedding()) == 0)
diff --git a/tests/test_language_model_trainer.py b/tests/test_language_model_trainer.py
deleted file mode 100644
--- a/tests/test_language_model_trainer.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import shutil
-from pathlib import Path
-
-from flair.data import Dictionary, Sentence
-from flair.embeddings import CharLMEmbeddings
-from flair.models import LanguageModel
-from flair.trainers.language_model_trainer import LanguageModelTrainer, TextCorpus
-
-
-def test_training():
-    # get default dictionary
-    dictionary: Dictionary = Dictionary.load('chars')
-
-    # init forward LM with 128 hidden states and 1 layer
-    language_model: LanguageModel = LanguageModel(dictionary, is_forward_lm=True, hidden_size=128, nlayers=1)
-
-    # get the example corpus and process at character level in forward direction
-    corpus: TextCorpus = TextCorpus(str(Path(__file__).parent / 'resources/corpora/lorem_ipsum'),
-                                    dictionary,
-                                    language_model.is_forward_lm,
-                                    character_level=True)
-
-    # train the language model
-    trainer: LanguageModelTrainer = LanguageModelTrainer(language_model, corpus)
-    trainer.train('./results', sequence_length=10, mini_batch_size=10, max_epochs=5)
-
-    # use the character LM as embeddings to embed the example sentence 'I love Berlin'
-    char_lm_embeddings = CharLMEmbeddings('./results/best-lm.pt')
-    sentence = Sentence('I love Berlin')
-    char_lm_embeddings.embed(sentence)
-    print(sentence[1].embedding.size())
-
-    # clean up results directory
-    shutil.rmtree('./results', ignore_errors=True)
-
-
diff --git a/tests/test_model_integration.py b/tests/test_model_integration.py
--- a/tests/test_model_integration.py
+++ b/tests/test_model_integration.py
@@ -1,16 +1,19 @@
 import os
 import shutil
+import pytest
 
-from flair.data import Sentence
+from flair.data import Dictionary, Sentence
 from flair.data_fetcher import NLPTaskDataFetcher, NLPTask
 from flair.embeddings import WordEmbeddings, CharLMEmbeddings, DocumentLSTMEmbeddings, TokenEmbeddings
-from flair.models import SequenceTagger, TextClassifier
+from flair.models import SequenceTagger, TextClassifier, LanguageModel
 from flair.trainers import SequenceTaggerTrainer, TextClassifierTrainer
+from flair.trainers.language_model_trainer import LanguageModelTrainer, TextCorpus
 
 
-def test_train_load_use_tagger():
+@pytest.mark.integration
+def test_train_load_use_tagger(results_base_path, tasks_base_path):
 
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION, base_path=tasks_base_path)
     tag_dictionary = corpus.make_tag_dictionary('ner')
 
     embeddings = WordEmbeddings('glove')
@@ -24,9 +27,9 @@ def test_train_load_use_tagger():
     # initialize trainer
     trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=True)
 
-    trainer.train('./results', learning_rate=0.1, mini_batch_size=2, max_epochs=3)
+    trainer.train(str(results_base_path), learning_rate=0.1, mini_batch_size=2, max_epochs=2)
 
-    loaded_model: SequenceTagger = SequenceTagger.load_from_file('./results/final-model.pt')
+    loaded_model: SequenceTagger = SequenceTagger.load_from_file(results_base_path / 'final-model.pt')
 
     sentence = Sentence('I love Berlin')
     sentence_empty = Sentence('       ')
@@ -36,12 +39,13 @@ def test_train_load_use_tagger():
     loaded_model.predict([sentence_empty])
 
     # clean up results directory
-    shutil.rmtree('./results')
+    shutil.rmtree(results_base_path)
 
 
-def test_train_charlm_load_use_tagger():
+@pytest.mark.integration
+def test_train_charlm_load_use_tagger(results_base_path, tasks_base_path):
 
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION, base_path=tasks_base_path)
     tag_dictionary = corpus.make_tag_dictionary('ner')
 
     embeddings = CharLMEmbeddings('news-forward-fast')
@@ -55,9 +59,9 @@ def test_train_charlm_load_use_tagger():
     # initialize trainer
     trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=True)
 
-    trainer.train('./results', learning_rate=0.1, mini_batch_size=2, max_epochs=3)
+    trainer.train(str(results_base_path), learning_rate=0.1, mini_batch_size=2, max_epochs=2)
 
-    loaded_model: SequenceTagger = SequenceTagger.load_from_file('./results/final-model.pt')
+    loaded_model: SequenceTagger = SequenceTagger.load_from_file(results_base_path / 'final-model.pt')
 
     sentence = Sentence('I love Berlin')
     sentence_empty = Sentence('       ')
@@ -67,17 +71,19 @@ def test_train_charlm_load_use_tagger():
     loaded_model.predict([sentence_empty])
 
     # clean up results directory
-    shutil.rmtree('./results')
+    shutil.rmtree(results_base_path)
 
 
-def test_train_charlm_changed_chache_load_use_tagger():
+@pytest.mark.integration
+def test_train_charlm_changed_chache_load_use_tagger(results_base_path, tasks_base_path):
 
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION, base_path=tasks_base_path)
     tag_dictionary = corpus.make_tag_dictionary('ner')
 
     # make a temporary cache directory that we remove afterwards
-    os.makedirs('./results/cache/', exist_ok=True)
-    embeddings = CharLMEmbeddings('news-forward-fast', cache_directory='./results/cache/')
+    cache_dir = results_base_path / 'cache'
+    os.makedirs(cache_dir, exist_ok=True)
+    embeddings = CharLMEmbeddings('news-forward-fast', cache_directory=cache_dir)
 
     tagger: SequenceTagger = SequenceTagger(hidden_size=256,
                                             embeddings=embeddings,
@@ -88,12 +94,12 @@ def test_train_charlm_changed_chache_load_use_tagger():
     # initialize trainer
     trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=True)
 
-    trainer.train('./results', learning_rate=0.1, mini_batch_size=2, max_epochs=3)
+    trainer.train(str(results_base_path), learning_rate=0.1, mini_batch_size=2, max_epochs=2)
 
     # remove the cache directory
-    shutil.rmtree('./results/cache')
+    shutil.rmtree(cache_dir)
 
-    loaded_model: SequenceTagger = SequenceTagger.load_from_file('./results/final-model.pt')
+    loaded_model: SequenceTagger = SequenceTagger.load_from_file(results_base_path / 'final-model.pt')
 
     sentence = Sentence('I love Berlin')
     sentence_empty = Sentence('       ')
@@ -103,12 +109,13 @@ def test_train_charlm_changed_chache_load_use_tagger():
     loaded_model.predict([sentence_empty])
 
     # clean up results directory
-    shutil.rmtree('./results')
+    shutil.rmtree(results_base_path)
 
 
-def test_train_charlm_nochache_load_use_tagger():
+@pytest.mark.integration
+def test_train_charlm_nochache_load_use_tagger(results_base_path, tasks_base_path):
 
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION, base_path=tasks_base_path)
     tag_dictionary = corpus.make_tag_dictionary('ner')
 
     embeddings = CharLMEmbeddings('news-forward-fast', use_cache=False)
@@ -122,9 +129,9 @@ def test_train_charlm_nochache_load_use_tagger():
     # initialize trainer
     trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=True)
 
-    trainer.train('./results', learning_rate=0.1, mini_batch_size=2, max_epochs=3)
+    trainer.train(str(results_base_path), learning_rate=0.1, mini_batch_size=2, max_epochs=2)
 
-    loaded_model: SequenceTagger = SequenceTagger.load_from_file('./results/final-model.pt')
+    loaded_model: SequenceTagger = SequenceTagger.load_from_file(results_base_path / 'final-model.pt')
 
     sentence = Sentence('I love Berlin')
     sentence_empty = Sentence('       ')
@@ -134,9 +141,10 @@ def test_train_charlm_nochache_load_use_tagger():
     loaded_model.predict([sentence_empty])
 
     # clean up results directory
-    shutil.rmtree('./results')
+    shutil.rmtree(results_base_path)
 
 
+@pytest.mark.integration
 def test_load_use_serialized_tagger():
 
     loaded_model: SequenceTagger = SequenceTagger.load('ner')
@@ -148,9 +156,20 @@ def test_load_use_serialized_tagger():
     loaded_model.predict([sentence, sentence_empty])
     loaded_model.predict([sentence_empty])
 
+    sentence.clear_embeddings()
+    sentence_empty.clear_embeddings()
 
-def test_train_load_use_classifier():
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB)
+    loaded_model: SequenceTagger = SequenceTagger.load('pos')
+
+    loaded_model.predict(sentence)
+    loaded_model.predict([sentence, sentence_empty])
+    loaded_model.predict([sentence_empty])
+
+
+@pytest.mark.integration
+def test_train_load_use_classifier(results_base_path, tasks_base_path):
+
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, base_path=tasks_base_path)
     label_dict = corpus.make_label_dictionary()
 
     glove_embedding: WordEmbeddings = WordEmbeddings('en-glove')
@@ -159,8 +178,8 @@ def test_train_load_use_classifier():
 
     model = TextClassifier(document_embeddings, label_dict, False)
 
-    trainer = TextClassifierTrainer(model, corpus, label_dict, False)
-    trainer.train('./results', max_epochs=2)
+    trainer = TextClassifierTrainer(model, corpus, label_dict, test_mode=True)
+    trainer.train(str(results_base_path), max_epochs=2)
 
     sentence = Sentence("Berlin is a really nice city.")
 
@@ -170,7 +189,7 @@ def test_train_load_use_classifier():
             assert (0.0 <= l.score <= 1.0)
             assert (type(l.score) is float)
 
-        loaded_model = TextClassifier.load_from_file('./results/final-model.pt')
+    loaded_model = TextClassifier.load_from_file(results_base_path / 'final-model.pt')
 
     sentence = Sentence('I love Berlin')
     sentence_empty = Sentence('       ')
@@ -180,11 +199,13 @@ def test_train_load_use_classifier():
     loaded_model.predict([sentence_empty])
 
     # clean up results directory
-    shutil.rmtree('./results')
+    shutil.rmtree(results_base_path)
 
 
-def test_train_charlm_load_use_classifier():
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB)
+@pytest.mark.integration
+def test_train_charlm_load_use_classifier(results_base_path, tasks_base_path):
+
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, base_path=tasks_base_path)
     label_dict = corpus.make_label_dictionary()
 
     glove_embedding: TokenEmbeddings = CharLMEmbeddings('news-forward-fast')
@@ -193,8 +214,8 @@ def test_train_charlm_load_use_classifier():
 
     model = TextClassifier(document_embeddings, label_dict, False)
 
-    trainer = TextClassifierTrainer(model, corpus, label_dict, False)
-    trainer.train('./results', max_epochs=2)
+    trainer = TextClassifierTrainer(model, corpus, label_dict, test_mode=True)
+    trainer.train(str(results_base_path), max_epochs=2)
 
     sentence = Sentence("Berlin is a really nice city.")
 
@@ -204,7 +225,7 @@ def test_train_charlm_load_use_classifier():
             assert (0.0 <= l.score <= 1.0)
             assert (type(l.score) is float)
 
-        loaded_model = TextClassifier.load_from_file('./results/final-model.pt')
+    loaded_model = TextClassifier.load_from_file(results_base_path / 'final-model.pt')
 
     sentence = Sentence('I love Berlin')
     sentence_empty = Sentence('       ')
@@ -214,11 +235,13 @@ def test_train_charlm_load_use_classifier():
     loaded_model.predict([sentence_empty])
 
     # clean up results directory
-    shutil.rmtree('./results')
+    shutil.rmtree(results_base_path)
+
 
+@pytest.mark.integration
+def test_train_charlm_nocache_load_use_classifier(results_base_path, tasks_base_path):
 
-def test_train_charlm__nocache_load_use_classifier():
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB)
+    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, base_path=tasks_base_path)
     label_dict = corpus.make_label_dictionary()
 
     glove_embedding: TokenEmbeddings = CharLMEmbeddings('news-forward-fast', use_cache=False)
@@ -228,8 +251,8 @@ def test_train_charlm__nocache_load_use_classifier():
 
     model = TextClassifier(document_embeddings, label_dict, False)
 
-    trainer = TextClassifierTrainer(model, corpus, label_dict, False)
-    trainer.train('./results', max_epochs=2)
+    trainer = TextClassifierTrainer(model, corpus, label_dict, test_mode=True)
+    trainer.train(str(results_base_path), max_epochs=2)
 
     sentence = Sentence("Berlin is a really nice city.")
 
@@ -239,7 +262,7 @@ def test_train_charlm__nocache_load_use_classifier():
             assert (0.0 <= l.score <= 1.0)
             assert (type(l.score) is float)
 
-        loaded_model = TextClassifier.load_from_file('./results/final-model.pt')
+    loaded_model = TextClassifier.load_from_file(results_base_path / 'final-model.pt')
 
     sentence = Sentence('I love Berlin')
     sentence_empty = Sentence('       ')
@@ -249,4 +272,31 @@ def test_train_charlm__nocache_load_use_classifier():
     loaded_model.predict([sentence_empty])
 
     # clean up results directory
-    shutil.rmtree('./results')
\ No newline at end of file
+    shutil.rmtree(results_base_path)
+
+
+@pytest.mark.integration
+def test_train_language_model(results_base_path, resources_path):
+    # get default dictionary
+    dictionary: Dictionary = Dictionary.load('chars')
+
+    # init forward LM with 128 hidden states and 1 layer
+    language_model: LanguageModel = LanguageModel(dictionary, is_forward_lm=True, hidden_size=128, nlayers=1)
+
+    # get the example corpus and process at character level in forward direction
+    corpus: TextCorpus = TextCorpus(str(resources_path / 'corpora/lorem_ipsum'),
+                                    dictionary,
+                                    language_model.is_forward_lm,
+                                    character_level=True)
+
+    # train the language model
+    trainer: LanguageModelTrainer = LanguageModelTrainer(language_model, corpus, test_mode=True)
+    trainer.train(str(results_base_path), sequence_length=10, mini_batch_size=10, max_epochs=2)
+
+    # use the character LM as embeddings to embed the example sentence 'I love Berlin'
+    char_lm_embeddings = CharLMEmbeddings(str(results_base_path / 'best-lm.pt'))
+    sentence = Sentence('I love Berlin')
+    char_lm_embeddings.embed(sentence)
+
+    # clean up results directory
+    shutil.rmtree(results_base_path, ignore_errors=True)
diff --git a/tests/test_sequence_labeler_trainer.py b/tests/test_sequence_labeler_trainer.py
deleted file mode 100644
--- a/tests/test_sequence_labeler_trainer.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import shutil
-
-from flair.data_fetcher import NLPTaskDataFetcher, NLPTask
-from flair.embeddings import WordEmbeddings
-from flair.models import SequenceTagger
-from flair.trainers import SequenceTaggerTrainer
-
-
-def test_training(tasks_base_path):
-
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.FASHION, tasks_base_path)
-    tag_dictionary = corpus.make_tag_dictionary('ner')
-
-    embeddings = WordEmbeddings('glove')
-
-    tagger: SequenceTagger = SequenceTagger(hidden_size=256,
-                                            embeddings=embeddings,
-                                            tag_dictionary=tag_dictionary,
-                                            tag_type='ner',
-                                            use_crf=False)
-
-    # initialize trainer
-    trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=True)
-
-    trainer.train('./results', learning_rate=0.1, mini_batch_size=2, max_epochs=10)
-
-    # clean up results directory
-    shutil.rmtree('./results')
diff --git a/tests/test_sequence_tagger.py b/tests/test_sequence_tagger.py
deleted file mode 100644
--- a/tests/test_sequence_tagger.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import os
-
-import pytest
-
-from flair.data import Sentence
-from flair.models import SequenceTagger
-
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
-def test_tag_sentence():
-
-    # test tagging
-    sentence = Sentence('I love Berlin')
-
-    tagger = SequenceTagger.load('ner')
-
-    tagger.predict(sentence)
-
-    # test re-tagging
-    tagger = SequenceTagger.load('pos')
-
-    tagger.predict(sentence)
diff --git a/tests/test_text_classifier.py b/tests/test_text_classifier.py
--- a/tests/test_text_classifier.py
+++ b/tests/test_text_classifier.py
@@ -7,7 +7,6 @@
 from flair.models.text_classification_model import TextClassifier
 
 
-@pytest.fixture
 def init(tasks_base_path) -> Tuple[TaggedCorpus, Dictionary, TextClassifier]:
     corpus = NLPTaskDataFetcher.fetch_data(NLPTask.AG_NEWS, tasks_base_path)
     label_dict = corpus.make_label_dictionary()
diff --git a/tests/test_text_classifier_trainer.py b/tests/test_text_classifier_trainer.py
deleted file mode 100644
--- a/tests/test_text_classifier_trainer.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import shutil
-
-from flair.data import Sentence
-
-from flair.data_fetcher import NLPTaskDataFetcher, NLPTask
-from flair.embeddings import WordEmbeddings, DocumentMeanEmbeddings, DocumentLSTMEmbeddings
-from flair.models.text_classification_model import TextClassifier
-from flair.trainers.text_classification_trainer import TextClassifierTrainer
-
-
-def test_text_classifier_single_label(tasks_base_path):
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, tasks_base_path)
-    label_dict = corpus.make_label_dictionary()
-
-    glove_embedding: WordEmbeddings = WordEmbeddings('en-glove')
-    document_embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings([glove_embedding], 128, 1, False, 64, False, False)
-
-    model = TextClassifier(document_embeddings, label_dict, False)
-
-    trainer = TextClassifierTrainer(model, corpus, label_dict, False)
-    trainer.train('./results', max_epochs=2)
-
-    sentence = Sentence("Berlin is a really nice city.")
-
-    for s in model.predict(sentence):
-        for l in s.labels:
-            assert(l.value is not None)
-            assert(0.0 <= l.score <= 1.0)
-            assert(type(l.score) is float)
-
-    # clean up results directory
-    shutil.rmtree('./results')
-
-
-def test_text_classifier_mulit_label(tasks_base_path):
-    corpus = NLPTaskDataFetcher.fetch_data(NLPTask.IMDB, tasks_base_path)
-    label_dict = corpus.make_label_dictionary()
-
-    glove_embedding: WordEmbeddings = WordEmbeddings('en-glove')
-    document_embeddings: DocumentMeanEmbeddings = DocumentMeanEmbeddings([glove_embedding])
-
-    model = TextClassifier(document_embeddings, label_dict, True)
-
-    trainer = TextClassifierTrainer(model, corpus, label_dict, False)
-    trainer.train('./results', max_epochs=2)
-
-    sentence = Sentence("Berlin is a really nice city.")
-
-    for s in model.predict(sentence):
-        for l in s.labels:
-            assert(l.value is not None)
-            assert(0.0 <= l.score <= 1.0)
-            assert(type(l.score) is float)
-
-    # clean up results directory
-    shutil.rmtree('./results')
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -4,7 +4,6 @@
 from flair.training_utils import calculate_micro_avg_metric, calculate_class_metrics, convert_labels_to_one_hot
 
 
-@pytest.fixture
 def init():
     y_true = [[0, 1, 1], [0, 0, 1], [1, 1, 0]]
     y_pred = [[0, 1, 1], [0, 0, 0], [1, 0, 0]]
diff --git a/tests/test_visual.py b/tests/test_visual.py
--- a/tests/test_visual.py
+++ b/tests/test_visual.py
@@ -1,5 +1,4 @@
 import os
-import shutil
 
 import pytest
 
@@ -12,10 +11,10 @@
 from flair.visual.training_curves import Plotter
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
+@pytest.mark.slow
 def test_visualize_word_emeddings(resources_path):
 
-    with open('./resources/visual/snippet.txt') as f:
+    with open(resources_path / 'visual/snippet.txt') as f:
         sentences = [x for x in f.read().split('\n') if x]
 
     sentences = [Sentence(x) for x in sentences]
@@ -26,16 +25,16 @@ def test_visualize_word_emeddings(resources_path):
     embeddings = StackedEmbeddings([charlm_embedding_backward, charlm_embedding_forward])
 
     visualizer = Visualizer()
-    visualizer.visualize_word_emeddings(embeddings, sentences, './resources/visual/sentence_embeddings.html')
+    visualizer.visualize_word_emeddings(embeddings, sentences, str(resources_path / 'visual/sentence_embeddings.html'))
 
     # clean up directory
-    os.remove('./resources/visual/sentence_embeddings.html')
+    os.remove(resources_path / 'visual/sentence_embeddings.html')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
-def test_visualize_word_emeddings():
+@pytest.mark.slow
+def test_visualize_word_emeddings(resources_path):
 
-    with open('./resources/visual/snippet.txt') as f:
+    with open(resources_path / 'visual/snippet.txt') as f:
         sentences = [x for x in f.read().split('\n') if x]
 
     sentences = [Sentence(x) for x in sentences]
@@ -43,16 +42,16 @@ def test_visualize_word_emeddings():
     charlm_embedding_forward = CharLMEmbeddings('news-forward')
 
     visualizer = Visualizer()
-    visualizer.visualize_char_emeddings(charlm_embedding_forward, sentences, './resources/visual/sentence_embeddings.html')
+    visualizer.visualize_char_emeddings(charlm_embedding_forward, sentences, str(resources_path / 'visual/sentence_embeddings.html'))
 
     # clean up directory
-    os.remove('./resources/visual/sentence_embeddings.html')
+    os.remove(resources_path / 'visual/sentence_embeddings.html')
 
 
-@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.")
-def test_visualize():
+@pytest.mark.slow
+def test_visualize(resources_path):
 
-    with open('./resources/visual/snippet.txt') as f:
+    with open(resources_path / 'visual/snippet.txt') as f:
         sentences = [x for x in f.read().split('\n') if x]
 
     sentences = [Sentence(x) for x in sentences]
@@ -74,10 +73,10 @@ def test_visualize():
     trans_ = tSNE()
     reduced = trans_.fit(X)
 
-    visualizer.visualize(reduced, contexts, './resources/visual/char_embeddings.html')
+    visualizer.visualize(reduced, contexts, str(resources_path / 'visual/char_embeddings.html'))
 
     # clean up directory
-    os.remove('./resources/visual/char_embeddings.html')
+    os.remove(resources_path / 'visual/char_embeddings.html')
 
 
 def test_highlighter(resources_path):