Repository: flairNLP/flair

Issue:
Multi-Dataset Training
Some tasks have multiple datasets available over which we might want to train at the same time. 

For instance, there are currently [6 English treebanks](http://universaldependencies.org/treebanks/en-comparison.html) listed on the UD web page over which we might want to train a PoS tagger for English. Or we might want to train a multilingual PoS tagger for several languages, in which case we would want to simultaneously train over UD corpora for English, German etc.  

**Task**: Add data and training methods to allow for training of models over multiple datasets. 


Discussion:

Resolving patch:
diff --git a/flair/data.py b/flair/data.py
--- a/flair/data.py
+++ b/flair/data.py
@@ -1,3 +1,4 @@
+from abc import abstractmethod
 from typing import List, Dict, Union
 
 import torch
@@ -10,7 +11,6 @@
 from segtok.tokenizer import split_contractions
 from segtok.tokenizer import word_tokenizer
 
-
 log = logging.getLogger(__name__)
 
 
@@ -324,17 +324,21 @@ def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[
             # otherwise assumes whitespace tokenized text
             else:
                 # add each word in tokenized string as Token object to Sentence
-                offset = 0
-                for word in text.split(' '):
-                    if word:
-                        try:
-                            word_offset = text.index(word, offset)
-                        except:
-                            word_offset = offset
-
-                        token = Token(word, start_position=word_offset)
-                        self.add_token(token)
-                        offset += len(word) + 1
+                word = ''
+                for index, char in enumerate(text):
+                    if char == ' ':
+                        if len(word) > 0:
+                            token = Token(word, start_position=index-len(word))
+                            self.add_token(token)
+
+                        word = ''
+                    else:
+                        word += char
+                # increment for last token in sentence if not followed by whtespace
+                index += 1
+                if len(word) > 0:
+                    token = Token(word, start_position=index-len(word))
+                    self.add_token(token)
 
     def get_token(self, token_id: int) -> Token:
         for token in self.tokens:
@@ -589,26 +593,75 @@ def __len__(self) -> int:
         return len(self.tokens)
 
 
-class TaggedCorpus:
-    def __init__(self, train: List[Sentence], dev: List[Sentence], test: List[Sentence]):
-        self.train: List[Sentence] = train
-        self.dev: List[Sentence] = dev
-        self.test: List[Sentence] = test
+class Corpus:
+
+    @property
+    @abstractmethod
+    def train(self) -> List[Sentence]:
+        pass
+
+    @property
+    @abstractmethod
+    def dev(self) -> List[Sentence]:
+        pass
+
+    @property
+    @abstractmethod
+    def test(self) -> List[Sentence]:
+        pass
+
+    @abstractmethod
+    def downsample(self, percentage: float = 0.1, only_downsample_train=False):
+        """Downsamples this corpus to a percentage of the sentences."""
+        pass
+
+    @abstractmethod
+    def get_all_sentences(self) -> List[Sentence]:
+        """Gets all sentences in the corpus (train, dev and test splits together)."""
+        pass
+
+    @abstractmethod
+    def make_tag_dictionary(self, tag_type: str) -> Dictionary:
+        """Produces a dictionary of token tags of tag_type."""
+        pass
+
+    @abstractmethod
+    def make_label_dictionary(self) -> Dictionary:
+        """
+        Creates a dictionary of all labels assigned to the sentences in the corpus.
+        :return: dictionary of labels
+        """
+        pass
+
+
+class TaggedCorpus(Corpus):
+    def __init__(self, train: List[Sentence], dev: List[Sentence], test: List[Sentence], name: str = 'corpus'):
+        self._train: List[Sentence] = train
+        self._dev: List[Sentence] = dev
+        self._test: List[Sentence] = test
+        self.name: str = name
+
+    @property
+    def train(self) -> List[Sentence]:
+        return self._train
+
+    @property
+    def dev(self) -> List[Sentence]:
+        return self._dev
+
+    @property
+    def test(self) -> List[Sentence]:
+        return self._test
 
     def downsample(self, percentage: float = 0.1, only_downsample_train=False):
 
-        self.train = self._downsample_to_proportion(self.train, percentage)
+        self._train = self._downsample_to_proportion(self.train, percentage)
         if not only_downsample_train:
-            self.dev = self._downsample_to_proportion(self.dev, percentage)
-            self.test = self._downsample_to_proportion(self.test, percentage)
+            self._dev = self._downsample_to_proportion(self.dev, percentage)
+            self._test = self._downsample_to_proportion(self.test, percentage)
 
         return self
 
-    def clear_embeddings(self):
-        for sentence in self.get_all_sentences():
-            for token in sentence.tokens:
-                token.clear_embeddings()
-
     def get_all_sentences(self) -> List[Sentence]:
         all_sentences: List[Sentence] = []
         all_sentences.extend(self.train)
@@ -751,7 +804,6 @@ def iob2(tags):
     Tags in IOB1 format are converted to IOB2.
     """
     for i, tag in enumerate(tags):
-        # print(tag)
         if tag.value == 'O':
             continue
         split = tag.value.split('-')
@@ -791,3 +843,71 @@ def iob_iobes(tags):
         else:
             raise Exception('Invalid IOB format!')
     return new_tags
+
+
+class MultiCorpus(Corpus):
+
+    def __init__(self, corpora: List[TaggedCorpus]):
+        self.corpora: List[TaggedCorpus] = corpora
+
+    @property
+    def train(self) -> List[Sentence]:
+        train: List[Sentence] = []
+        for corpus in self.corpora:
+            train.extend(corpus.train)
+        return train
+
+    @property
+    def dev(self) -> List[Sentence]:
+        dev: List[Sentence] = []
+        for corpus in self.corpora:
+            dev.extend(corpus.dev)
+        return dev
+
+    @property
+    def test(self) -> List[Sentence]:
+        test: List[Sentence] = []
+        for corpus in self.corpora:
+            test.extend(corpus.test)
+        return test
+
+    def __str__(self):
+        return '\n'.join([str(corpus) for corpus in self.corpora])
+
+    def get_all_sentences(self) -> List[Sentence]:
+        sentences = []
+        for corpus in self.corpora:
+            sentences.extend(corpus.get_all_sentences())
+        return sentences
+
+    def downsample(self, percentage: float = 0.1, only_downsample_train=False):
+
+        for corpus in self.corpora:
+            corpus.downsample(percentage, only_downsample_train)
+
+        return self
+
+    def make_tag_dictionary(self, tag_type: str) -> Dictionary:
+
+        # Make the tag dictionary
+        tag_dictionary: Dictionary = Dictionary()
+        tag_dictionary.add_item('O')
+        for corpus in self.corpora:
+            for sentence in corpus.get_all_sentences():
+                for token in sentence.tokens:
+                    token: Token = token
+                    tag_dictionary.add_item(token.get_tag(tag_type).value)
+        tag_dictionary.add_item('<START>')
+        tag_dictionary.add_item('<STOP>')
+        return tag_dictionary
+
+    def make_label_dictionary(self) -> Dictionary:
+
+        label_dictionary: Dictionary = Dictionary(add_unk=False)
+        for corpus in self.corpora:
+            labels = set(corpus._get_all_label_names())
+
+            for label in labels:
+                label_dictionary.add_item(label)
+
+        return label_dictionary
diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py
--- a/flair/data_fetcher.py
+++ b/flair/data_fetcher.py
@@ -1,29 +1,35 @@
-from typing import List, Dict
+from typing import List, Dict, Union
 import re
 import os
 import logging
 from enum import Enum
 from pathlib import Path
 
-from flair.data import Sentence, TaggedCorpus, Token
+from flair.data import Sentence, TaggedCorpus, Token, MultiCorpus
 
 log = logging.getLogger(__name__)
 
 
 class NLPTask(Enum):
-    # conll column format
-    CONLL_03 = 'conll_03'
+    # conll 2000 column format
     CONLL_2000 = 'conll_2000'
-    CONLL_03_GERMAN = 'conll_03-ger'
-    ONTONER = 'onto-ner'
-    FASHION = 'fashion'
-    GERMEVAL = 'germeval'
-    SRL = 'srl'
-    WSD = 'wsd'
+
+    # conll 03 NER column format
+    CONLL_03 = 'conll_03'
+    CONLL_03_GERMAN = 'conll_03_german'
+    CONLL_03_DUTCH = 'conll_03_dutch'
+    CONLL_03_SPANISH = 'conll_03_spanish'
 
     # conll-u format
     UD_ENGLISH = 'ud_english'
     UD_GERMAN = 'ud_german'
+
+    # other datasets
+    ONTONER = 'ontoner'
+    FASHION = 'fashion'
+    GERMEVAL = 'germeval'
+    SRL = 'srl'
+    WSD = 'wsd'
     CONLL_12 = 'conll_12'
     PENN = 'penn'
     ONTONOTES = 'ontonotes'
@@ -36,7 +42,11 @@ class NLPTask(Enum):
 class NLPTaskDataFetcher:
 
     @staticmethod
-    def fetch_data(task: NLPTask, base_path: Path = None) -> TaggedCorpus:
+    def fetch_corpora(tasks: List[Union[NLPTask, str]], base_path: Path = None) -> MultiCorpus:
+        return MultiCorpus([NLPTaskDataFetcher.fetch_data(task, base_path) for task in tasks])
+
+    @staticmethod
+    def fetch_data(task: Union[NLPTask, str], base_path: Path = None) -> TaggedCorpus:
         """
         Helper function to fetch a TaggedCorpus for a specific NLPTask. For this to work you need to first download
         and put into the appropriate folder structure the corresponsing NLP task data. The tutorials on
@@ -50,143 +60,82 @@ def fetch_data(task: NLPTask, base_path: Path = None) -> TaggedCorpus:
         if not base_path:
             base_path = Path('resources') / 'tasks'
 
-        data_folder = str(base_path / str(task.value).lower())
+        # get string value if enum is passed
+        task = task.value if type(task) is NLPTask else task
 
-        log.info("Reading data from {}".format(data_folder))
+        data_folder = str(base_path / task.lower())
 
         # the CoNLL 2000 task on chunking has three columns: text, pos and np (chunk)
-        if task == NLPTask.CONLL_2000:
+        if task == NLPTask.CONLL_2000.value:
             columns = {0: 'text', 1: 'pos', 2: 'np'}
 
             return NLPTaskDataFetcher.fetch_column_corpus(data_folder,
                                                           columns,
-                                                          train_file='train.txt',
-                                                          test_file='test.txt',
                                                           tag_to_biloes='np')
 
         # many NER tasks follow the CoNLL 03 format with four colulms: text, pos, np and ner tag
-        if task == NLPTask.CONLL_03 or task == NLPTask.ONTONER or task == NLPTask.FASHION:
+        if task == NLPTask.CONLL_03.value or task == NLPTask.ONTONER.value or task == NLPTask.FASHION.value:
             columns = {0: 'text', 1: 'pos', 2: 'np', 3: 'ner'}
 
             return NLPTaskDataFetcher.fetch_column_corpus(data_folder,
                                                           columns,
-                                                          train_file='eng.train',
-                                                          test_file='eng.testb',
-                                                          dev_file='eng.testa',
                                                           tag_to_biloes='ner',
                                                           )
 
         # the CoNLL 03 task for German has an additional lemma column
-        if task == NLPTask.CONLL_03_GERMAN:
+        if task == NLPTask.CONLL_03_GERMAN.value:
             columns = {0: 'text', 1: 'lemma', 2: 'pos', 3: 'np', 4: 'ner'}
 
             return NLPTaskDataFetcher.fetch_column_corpus(data_folder,
                                                           columns,
-                                                          train_file='deu.train',
-                                                          test_file='deu.testb',
-                                                          dev_file='deu.testa',
+                                                          tag_to_biloes='ner')
+
+        # the CoNLL 03 task for Dutch has no NP column
+        if task == NLPTask.CONLL_03_DUTCH.value:
+            columns = {0: 'text', 1: 'pos', 2: 'ner'}
+
+            return NLPTaskDataFetcher.fetch_column_corpus(data_folder,
+                                                          columns,
+                                                          tag_to_biloes='ner')
+
+        # the CoNLL 03 task for Spanish only has two columns
+        if task == NLPTask.CONLL_03_SPANISH.value:
+            columns = {0: 'text', 1: 'ner'}
+
+            return NLPTaskDataFetcher.fetch_column_corpus(data_folder,
+                                                          columns,
                                                           tag_to_biloes='ner')
 
         # the GERMEVAL task only has two columns: text and ner
-        if task == NLPTask.GERMEVAL:
+        if task == NLPTask.GERMEVAL.value:
             columns = {1: 'text', 2: 'ner'}
 
             return NLPTaskDataFetcher.fetch_column_corpus(data_folder,
                                                           columns,
-                                                          train_file='NER-de-train.tsv',
-                                                          test_file='NER-de-test.tsv',
-                                                          dev_file='NER-de-dev.tsv',
                                                           tag_to_biloes='ner')
 
         # WSD tasks may be put into this column format
-        if task == NLPTask.WSD:
+        if task == NLPTask.WSD.value:
             columns = {0: 'text', 1: 'lemma', 2: 'pos', 3: 'sense'}
-
             return NLPTaskDataFetcher.fetch_column_corpus(data_folder,
                                                           columns,
                                                           train_file='semcor.tsv',
                                                           test_file='semeval2015.tsv')
 
         # the UD corpora follow the CoNLL-U format, for which we have a special reader
-        if task == NLPTask.UD_ENGLISH:
-            # get train, test and dev data
-            sentences_train: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'en_ewt-ud-train.conllu'))
-            sentences_test: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'en_ewt-ud-test.conllu'))
-            sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'en_ewt-ud-dev.conllu'))
-
-            return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
-
-        if task == NLPTask.UD_GERMAN:
-            # get train, test and dev data
-            sentences_train: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'de_gsd-ud-train.conllu'))
-            sentences_test: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'de_gsd-ud-test.conllu'))
-            sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'de_gsd-ud-dev.conllu'))
-
-            return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
-
-        if task == NLPTask.ONTONOTES:
-            # get train, test and dev data
-            sentences_train: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'train.conllu'))
-            sentences_test: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'test.conllu'))
-            sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'dev.conllu'))
-
-            return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
-
-        if task == NLPTask.CONLL_12:
-            # get train, test and dev data
-            sentences_train: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'train.propbank.conllu'))
-            sentences_test: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'test.propbank.conllu'))
-            sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'dev.propbank.conllu'))
-            return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
-
-        if task == NLPTask.PENN:
-            sentences_train: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'train.conll'))
-            sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'valid.conll'))
-            sentences_test: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
-                os.path.join(data_folder, 'test.conll'))
-
-            return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
-
-        # for text classifiers, we use our own special format
-        if task == NLPTask.IMDB:
-            sentences_train: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
-                os.path.join(data_folder, 'train.txt'))
-            sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
-                os.path.join(data_folder, 'dev.txt'))
-            sentences_test: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
-                os.path.join(data_folder, 'test.txt'))
-            return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
+        if task.startswith('ud_') or task in [NLPTask.ONTONOTES.value, NLPTask.CONLL_12.value, NLPTask.PENN.value]:
+            return NLPTaskDataFetcher.fetch_ud_corpus(data_folder)
 
         # for text classifiers, we use our own special format
-        if task == NLPTask.AG_NEWS:
-            sentences_train: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
-                os.path.join(data_folder, 'train.txt'))
-            sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
-                os.path.join(data_folder, 'dev.txt'))
-            sentences_test: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
-                os.path.join(data_folder, 'test.txt'))
-            return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
+        if task == NLPTask.IMDB.value or task == NLPTask.AG_NEWS.value:
+            return NLPTaskDataFetcher.fetch_classification_corpus(data_folder)
 
     @staticmethod
     def fetch_column_corpus(
             data_folder: Path,
             column_format: Dict[int, str],
-            train_file: str,
-            test_file: str,
+            train_file=None,
+            test_file=None,
             dev_file=None,
             tag_to_biloes=None) -> TaggedCorpus:
         """
@@ -204,6 +153,25 @@ def fetch_column_corpus(
         # TODO: move all paths to use pathlib.Path, for now convert to str for compatibility
         data_folder = str(data_folder)
 
+        # automatically identify train / test / dev files
+        if train_file is None:
+            for file in os.listdir(data_folder):
+                if 'train' in file and not '54019' in file:
+                    train_file = file
+                if 'test' in file:
+                    test_file = file
+                if 'dev' in file:
+                    dev_file = file
+                if 'testa' in file:
+                    dev_file = file
+                if 'testb' in file:
+                    test_file = file
+
+        log.info("Reading data from {}".format(data_folder))
+        log.info("Train: {}".format(train_file))
+        log.info("Dev: {}".format(dev_file))
+        log.info("Test: {}".format(test_file))
+
         # get train and test data
         sentences_train: List[Sentence] = NLPTaskDataFetcher.read_column_data(
             os.path.join(data_folder, train_file), column_format)
@@ -214,8 +182,8 @@ def fetch_column_corpus(
             sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_column_data(
                 os.path.join(data_folder, dev_file), column_format)
         else:
-            # sample dev data from train
-            sentences_dev: List[Sentence] = [sentences_train[i] for i in NLPTaskDataFetcher.__sample()]
+            # sample 10% of training sentences as dev data
+            sentences_dev: List[Sentence] = [sentences_train[i] for i in NLPTaskDataFetcher.__sample(len(sentences_train), 0.1)]
             sentences_train = [x for x in sentences_train if x not in sentences_dev]
 
         if tag_to_biloes is not None:
@@ -224,8 +192,143 @@ def fetch_column_corpus(
                 sentence: Sentence = sentence
                 sentence.convert_tag_scheme(tag_type=tag_to_biloes, target_scheme='iobes')
 
+        return TaggedCorpus(sentences_train, sentences_dev, sentences_test, name=data_folder)
+
+    @staticmethod
+    def fetch_ud_corpus(
+            data_folder: Path,
+            train_file=None,
+            test_file=None,
+            dev_file=None) -> TaggedCorpus:
+        """
+        Helper function to get a TaggedCorpus from CoNLL-U column-formatted task data such as the UD corpora
+
+        :param data_folder: base folder with the task data
+        :param train_file: the name of the train file
+        :param test_file: the name of the test file
+        :param dev_file: the name of the dev file, if None, dev data is sampled from train
+        :return: a TaggedCorpus with annotated train, dev and test data
+        """
+
+        # TODO: move all paths to use pathlib.Path, for now convert to str for compatibility
+        data_folder = str(data_folder)
+
+        # automatically identify train / test / dev files
+        if train_file is None:
+            for file in os.listdir(data_folder):
+                if 'train' in file:
+                    train_file = file
+                if 'test' in file:
+                    test_file = file
+                if 'dev' in file:
+                    dev_file = file
+                if 'testa' in file:
+                    dev_file = file
+                if 'testb' in file:
+                    test_file = file
+
+        log.info("Reading data from {}".format(data_folder))
+        log.info("Train: {}".format(train_file))
+        log.info("Dev: {}".format(dev_file))
+        log.info("Test: {}".format(test_file))
+
+        sentences_train: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
+            os.path.join(data_folder, train_file))
+        sentences_test: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
+            os.path.join(data_folder, test_file))
+        sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_conll_ud(
+            os.path.join(data_folder, dev_file))
+
+        return TaggedCorpus(sentences_train, sentences_dev, sentences_test, name=data_folder)
+
+    @staticmethod
+    def fetch_classification_corpus(
+            data_folder: Path,
+            train_file=None,
+            test_file=None,
+            dev_file=None) -> TaggedCorpus:
+        """
+        Helper function to get a TaggedCorpus from text classification-formatted task data
+
+        :param data_folder: base folder with the task data
+        :param train_file: the name of the train file
+        :param test_file: the name of the test file
+        :param dev_file: the name of the dev file, if None, dev data is sampled from train
+        :return: a TaggedCorpus with annotated train, dev and test data
+        """
+
+        # TODO: move all paths to use pathlib.Path, for now convert to str for compatibility
+        data_folder = str(data_folder)
+
+        # automatically identify train / test / dev files
+        if train_file is None:
+            for file in os.listdir(data_folder):
+                if 'train' in file:
+                    train_file = file
+                if 'test' in file:
+                    test_file = file
+                if 'dev' in file:
+                    dev_file = file
+                if 'testa' in file:
+                    dev_file = file
+                if 'testb' in file:
+                    test_file = file
+
+        log.info("Reading data from {}".format(data_folder))
+        log.info("Train: {}".format(train_file))
+        log.info("Dev: {}".format(dev_file))
+        log.info("Test: {}".format(test_file))
+
+        sentences_train: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
+            os.path.join(data_folder, train_file))
+        sentences_test: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
+            os.path.join(data_folder, test_file))
+        sentences_dev: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(
+            os.path.join(data_folder, dev_file))
+
         return TaggedCorpus(sentences_train, sentences_dev, sentences_test)
 
+    @staticmethod
+    def read_text_classification_file(path_to_file, max_tokens_per_doc=-1):
+        """
+        Reads a data file for text classification. The file should contain one document/text per line.
+        The line should have the following format:
+        __label__<class_name> <text>
+        If you have a multi class task, you can have as many labels as you want at the beginning of the line, e.g.,
+        __label__<class_name_1> __label__<class_name_2> <text>
+        :param path_to_file: the path to the data file
+        :param max_tokens_per_doc: Take only documents that contain number of tokens less or equal to this value. If
+        set to -1 all documents are taken.
+        :return: list of sentences
+        """
+        label_prefix = '__label__'
+        sentences = []
+
+        with open(path_to_file) as f:
+            for line in f:
+                words = line.split()
+
+                labels = []
+                l_len = 0
+
+                for i in range(len(words)):
+                    if words[i].startswith(label_prefix):
+                        l_len += len(words[i]) + 1
+                        label = words[i].replace(label_prefix, "")
+                        labels.append(label)
+                    else:
+                        break
+
+                text = line[l_len:].strip()
+
+                if text and labels:
+                    sentence = Sentence(text, labels=labels, use_tokenizer=True)
+                    if (len(sentence) > max_tokens_per_doc):
+                        sentence.tokens = sentence.tokens[:max_tokens_per_doc]
+                    sentences.append(sentence)
+
+        return sentences
+
     @staticmethod
     def read_column_data(path_to_column_file: str,
                          column_name_map: Dict[int, str],
@@ -242,7 +345,11 @@ def read_column_data(path_to_column_file: str,
         """
         sentences: List[Sentence] = []
 
-        lines: List[str] = open(path_to_column_file).read().strip().split('\n')
+        try:
+            lines: List[str] = open(path_to_column_file, encoding='utf-8').read().strip().split('\n')
+        except:
+            log.info('UTF-8 can\'t read: {} ... using "latin-1" instead.'.format(path_to_column_file))
+            lines: List[str] = open(path_to_column_file, encoding='latin1').read().strip().split('\n')
 
         # most data sets have the token text in the first column, if not, pass 'text' as column
         text_column: int = 0
@@ -292,7 +399,7 @@ def read_conll_ud(path_to_conll_file: str) -> List[Sentence]:
         sentence: Sentence = Sentence()
         for line in lines:
 
-            fields: List[str] = re.split("\s+", line)
+            fields: List[str] = re.split("\t+", line)
             if line == '':
                 if len(sentence) > 0:
                     sentences.append(sentence)
@@ -359,117 +466,15 @@ def read_text_classification_file(path_to_file, max_tokens_per_doc=-1):
 
                 if text and labels:
                     sentence = Sentence(text, labels=labels, use_tokenizer=True)
-                    if(len(sentence) > max_tokens_per_doc):
+                    if (len(sentence) > max_tokens_per_doc):
                         sentence.tokens = sentence.tokens[:max_tokens_per_doc]
                     sentences.append(sentence)
 
         return sentences
 
     @staticmethod
-    def __sample():
-        sample = [7199, 2012, 7426, 1374, 2590, 4401, 7659, 2441, 4209, 6997, 6907, 4789, 3292, 4874, 7836, 2065, 1804,
-                  2409,
-                  6353, 86, 1412, 5431, 3275, 7696, 3325, 7678, 6888, 5326, 5782, 3739, 4972, 6350, 7167, 6558, 918,
-                  6444,
-                  5368, 731, 244, 2029, 6200, 5088, 4688, 2580, 2153, 5477, 714, 1570, 6651, 5724, 4090, 167, 1689,
-                  6166,
-                  7304, 3705, 256, 5689, 6282, 707, 5390, 1367, 4167, 16, 6554, 5093, 3944, 5008, 3510, 1741, 1, 4464,
-                  173,
-                  5362, 6827, 35, 1662, 3136, 1516, 3826, 1575, 6771, 5965, 1449, 7806, 632, 5870, 3566, 1434, 2361,
-                  6348,
-                  5140, 7765, 4800, 6541, 7910, 2021, 1041, 3171, 2137, 495, 2249, 7334, 4806, 844, 3848, 7396, 3861,
-                  1337,
-                  430, 1325, 36, 2289, 720, 4182, 3955, 3451, 192, 3715, 3144, 1140, 2874, 6728, 4877, 1876, 2551, 2910,
-                  260,
-                  7767, 7206, 5577, 6707, 3392, 1830, 842, 5264, 4042, 3572, 331, 6995, 2307, 5664, 2878, 1115, 1880,
-                  1548,
-                  3740, 860, 1799, 2099, 7359, 4648, 2264, 1018, 5417, 3052, 2480, 2256, 6672, 6647, 1272, 1986, 7063,
-                  4071,
-                  3199, 3652, 1797, 1693, 2008, 4138, 7428, 3083, 1494, 4911, 728, 1556, 7651, 2535, 2160, 4014, 1438,
-                  6148,
-                  551, 476, 4198, 3835, 1489, 6404, 7346, 1178, 607, 7693, 4146, 6655, 4355, 1571, 522, 5835, 622, 1267,
-                  6778, 5236, 5211, 5039, 3836, 1751, 1019, 6952, 7610, 7677, 4224, 1485, 4101, 5793, 6708, 5741, 4630,
-                  5857,
-                  6959, 847, 4375, 3458, 4936, 6887, 5, 3150, 5551, 4840, 2618, 7456, 7600, 5995, 5270, 5496, 4316,
-                  1479,
-                  517, 2940, 2337, 7461, 3296, 4133, 491, 6408, 7609, 4290, 5028, 7471, 6337, 488, 5033, 5967, 1209,
-                  5511,
-                  5449, 3837, 4760, 4490, 6550, 2676, 371, 3962, 4507, 5268, 4285, 5257, 859, 14, 4487, 5669, 6594,
-                  6544,
-                  7427, 5624, 4882, 7425, 2378, 1498, 931, 7253, 2638, 2897, 5670, 6463, 5300, 6802, 4229, 7076, 6848,
-                  6414,
-                  1465, 7243, 989, 7204, 1926, 1255, 1794, 2115, 3975, 6987, 3166, 105, 3856, 3272, 3977, 4097, 2612,
-                  2869,
-                  6022, 153, 3357, 2439, 6491, 766, 3840, 2683, 5074, 159, 5407, 3029, 4815, 1782, 4970, 6250, 5377,
-                  6473,
-                  5151, 4687, 798, 5214, 3364, 6412, 7125, 3495, 2385, 4476, 863, 5493, 5830, 938, 2979, 7808, 4830,
-                  4180,
-                  1565, 4818, 702, 1442, 4673, 6920, 2089, 1930, 2036, 1436, 6632, 1006, 5256, 5666, 6401, 3415, 4693,
-                  5890,
-                  7124, 3853, 884, 4650, 4550, 7406, 3394, 6715, 6754, 3932, 599, 1816, 3273, 5016, 2918, 526, 6883,
-                  3089,
-                  64, 1305, 7442, 6837, 783, 4536, 100, 4951, 2933, 3750, 3232, 7150, 1934, 3576, 2900, 7883, 964, 4025,
-                  28,
-                  1732, 382, 166, 6053, 6320, 2058, 652, 3182, 6836, 4547, 419, 1600, 6891, 6235, 7208, 7190, 7144,
-                  3133,
-                  4775, 4892, 895, 4428, 7929, 7297, 7773, 5325, 2799, 5645, 1192, 1672, 2540, 6812, 5441, 2681, 342,
-                  333,
-                  2161, 593, 5463, 1568, 5252, 4194, 2280, 2423, 2118, 7455, 4553, 5960, 3163, 7147, 4305, 5599, 2775,
-                  5334,
-                  4727, 6926, 2189, 7778, 7245, 2066, 1259, 2074, 7866, 7403, 4642, 5490, 3563, 6923, 3934, 5728, 5425,
-                  2369,
-                  375, 3578, 2732, 2675, 6167, 6726, 4211, 2241, 4585, 4272, 882, 1821, 3904, 6864, 5723, 4708, 3226,
-                  7151,
-                  3911, 4274, 4945, 3719, 7467, 7712, 5068, 7181, 745, 2846, 2695, 3707, 1076, 1077, 2698, 5699, 1040,
-                  6338,
-                  631, 1609, 896, 3607, 6801, 3593, 1698, 91, 639, 2826, 2937, 493, 4218, 5958, 2765, 4926, 4546, 7400,
-                  1909,
-                  5693, 1871, 1687, 6589, 4334, 2748, 7129, 3332, 42, 345, 709, 4685, 6624, 377, 3204, 2603, 7183, 6123,
-                  4249, 1531, 7, 703, 6978, 2856, 7871, 7290, 369, 582, 4704, 4979, 66, 1139, 87, 5166, 967, 2727, 5920,
-                  6806, 5997, 1301, 5826, 1805, 4347, 4870, 4213, 4254, 504, 3865, 189, 6393, 7281, 2907, 656, 6617,
-                  1807,
-                  6258, 3605, 1009, 3694, 3004, 2870, 7710, 2608, 400, 7635, 4392, 3055, 942, 2952, 3441, 902, 5892,
-                  574,
-                  5418, 6212, 1602, 5619, 7094, 1168, 3877, 3888, 1618, 6564, 455, 4581, 3258, 2606, 4643, 2454, 2763,
-                  5332,
-                  6158, 940, 2343, 7902, 3438, 6117, 2198, 3842, 4773, 1492, 2424, 7662, 6559, 1196, 3203, 5286, 6764,
-                  3829,
-                  4746, 1117, 2120, 1378, 5614, 4871, 4024, 5489, 3312, 1094, 1838, 3964, 3151, 4545, 5795, 1739, 4920,
-                  5690,
-                  2570, 3530, 2751, 1426, 2631, 88, 7728, 3741, 5654, 3157, 5557, 6668, 7309, 7313, 807, 4376, 4512,
-                  6786,
-                  7898, 2429, 3890, 2418, 2243, 2330, 4561, 6119, 2864, 5570, 2485, 5499, 4983, 6257, 3692, 1563, 1939,
-                  126,
-                  3299, 2811, 7933, 465, 5976, 3712, 4478, 7671, 3143, 1947, 6133, 1928, 5725, 5747, 1107, 163, 3610,
-                  3723,
-                  1496, 7477, 53, 6548, 5548, 4357, 4963, 5896, 5361, 7295, 7632, 3559, 6740, 6312, 6890, 3303, 625,
-                  7681,
-                  7174, 6928, 1088, 2133, 4276, 5299, 4488, 5354, 3044, 3321, 409, 6218, 2255, 829, 2129, 673, 1588,
-                  6824,
-                  1297, 6996, 4324, 7423, 5209, 7617, 3041, 78, 5518, 5392, 4967, 3704, 497, 858, 1833, 5108, 6095,
-                  6039,
-                  6705, 5561, 5888, 3883, 1048, 1119, 1292, 5639, 4358, 2487, 1235, 125, 4453, 3035, 3304, 6938, 2670,
-                  4322,
-                  648, 1785, 6114, 6056, 1515, 4628, 5036, 37, 1226, 6081, 4473, 953, 5009, 217, 5952, 755, 2604, 3060,
-                  3322,
-                  6087, 604, 2260, 7897, 3129, 616, 1593, 69, 230, 1526, 6349, 6452, 4235, 1752, 4288, 6377, 1229, 395,
-                  4326,
-                  5845, 5314, 1542, 6483, 2844, 7088, 4702, 3300, 97, 7817, 6804, 471, 3624, 3773, 7057, 2391, 22, 3293,
-                  6619, 1933, 6871, 164, 7796, 6744, 1589, 1802, 2880, 7093, 906, 389, 7892, 976, 848, 4076, 7818, 5556,
-                  3507, 4740, 4359, 7105, 2938, 683, 4292, 1849, 3121, 5618, 4407, 2883, 7502, 5922, 6130, 301, 4370,
-                  7019,
-                  3009, 425, 2601, 3592, 790, 2656, 5455, 257, 1500, 3544, 818, 2221, 3313, 3426, 5915, 7155, 3110,
-                  4425,
-                  5255, 2140, 5632, 614, 1663, 1787, 4023, 1734, 4528, 3318, 4099, 5383, 3999, 722, 3866, 1401, 1299,
-                  2926,
-                  1360, 1916, 3259, 2420, 1409, 2817, 5961, 782, 1636, 4168, 1344, 4327, 7780, 7335, 3017, 6582, 4623,
-                  7198,
-                  2499, 2139, 3821, 4822, 2552, 4904, 4328, 6666, 4389, 3687, 1014, 7829, 4802, 5149, 4199, 1866, 1992,
-                  2893,
-                  6957, 3099, 1212, 672, 4616, 758, 6421, 2281, 6528, 3148, 4197, 1317, 4258, 1407, 6618, 2562, 4448,
-                  6137,
-                  6151, 1817, 3278, 3982, 5144, 3311, 3453, 1722, 4912, 3641, 5560, 2234, 6645, 3084, 4890, 557, 1455,
-                  4152,
-                  5784, 7221, 3078, 6961, 23, 4281, 6012, 156, 5109, 6984, 6140, 6730, 4965, 7123, 85, 2912, 5192, 1425,
-                  1993, 4056, 598]
+    def __sample(total_number_of_sentences: int, percentage: float = 0.1) -> List[int]:
+        import random
+        sample_size: int = round(total_number_of_sentences * percentage)
+        sample = random.sample(range(1, total_number_of_sentences), sample_size)
         return sample
diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py
--- a/flair/trainers/trainer.py
+++ b/flair/trainers/trainer.py
@@ -9,7 +9,7 @@
 
 import flair
 import flair.nn
-from flair.data import Sentence, Token, TaggedCorpus
+from flair.data import Sentence, Token, Label, MultiCorpus, Corpus
 from flair.models import TextClassifier, SequenceTagger
 from flair.training_utils import Metric, init_output_file, WeightExtractor, clear_embeddings, EvaluationMetric
 
@@ -18,9 +18,9 @@
 
 class ModelTrainer:
 
-    def __init__(self, model: flair.nn.Model, corpus: TaggedCorpus) -> None:
+    def __init__(self, model: flair.nn.Model, corpus: Corpus) -> None:
         self.model: flair.nn.Model = model
-        self.corpus: TaggedCorpus = corpus
+        self.corpus: Corpus = corpus
 
     def train(self,
               base_path: str,
@@ -144,7 +144,8 @@ def train(self,
                         'DEV', self.corpus.dev, evaluation_metric, embeddings_in_memory, mini_batch_size)
 
                 test_metric, test_loss = self._calculate_evaluation_results_for(
-                    'TEST', self.corpus.test, evaluation_metric, embeddings_in_memory, mini_batch_size, base_path + '/test.tsv')
+                    'TEST', self.corpus.test, evaluation_metric, embeddings_in_memory, mini_batch_size,
+                    base_path + '/test.tsv')
 
                 with open(loss_txt, 'a') as f:
                     train_metric_str = train_metric.to_tsv() if train_metric is not None else Metric.to_empty_tsv()
@@ -208,9 +209,27 @@ def train(self,
                      f'{test_metric.f_score(class_name):.4f}')
         self._log_line()
 
-        return test_metric.micro_avg_f_score()
+        # if we are training over multiple datasets, do evaluation for each
+        if type(self.corpus) is MultiCorpus:
+            for subcorpus in self.corpus.corpora:
+                self._log_line()
+                self._calculate_evaluation_results_for(subcorpus.name, subcorpus.test, evaluation_metric,
+                                                       embeddings_in_memory, mini_batch_size, base_path + '/test.tsv')
+
+        # get and return the final test score of best model
+        if evaluation_metric == EvaluationMetric.MACRO_ACCURACY:
+            final_score = test_metric.macro_avg_accuracy()
+        elif evaluation_metric == EvaluationMetric.MICRO_ACCURACY:
+            final_score = test_metric.micro_avg_accuracy()
+        elif evaluation_metric == EvaluationMetric.MACRO_F1_SCORE:
+            final_score = test_metric.macro_avg_f_score()
+        else:
+            final_score = test_metric.micro_avg_f_score()
+
+        return final_score
 
-    def _calculate_evaluation_results_for(self, dataset_name, dataset, evaluation_metric, embeddings_in_memory, mini_batch_size,
+    def _calculate_evaluation_results_for(self, dataset_name, dataset, evaluation_metric, embeddings_in_memory,
+                                          mini_batch_size,
                                           out_path=None):
 
         metric, loss = ModelTrainer.evaluate(self.model, dataset, mini_batch_size=mini_batch_size,


Test patch:
diff --git a/tests/resources/tasks/fashion_nodev/test.tsv b/tests/resources/tasks/fashion_nodev/test.tsv
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/fashion_nodev/test.tsv
@@ -0,0 +1,21 @@
+Most _ _ O
+wedding _ _ B-Occasion
+dresses _ _ B-NominalProduct
+, _ _ O
+for _ _ O
+example _ _ O
+, _ _ O
+are _ _ O
+simply _ _ O
+too _ _ O
+enormous _ _ O
+and _ _ O
+terrifyingly _ _ O
+loaded _ _ O
+with _ _ O
+sentimental _ _ O
+value _ _ O
+for _ _ O
+DIY _ _ B-ProductDesign
+dyeing _ _ I-ProductDesign
+. _ _ O
\ No newline at end of file
diff --git a/tests/resources/tasks/fashion_nodev/train.tsv b/tests/resources/tasks/fashion_nodev/train.tsv
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/fashion_nodev/train.tsv
@@ -0,0 +1,122 @@
+From _ _ O
+the _ _ O
+charming _ _ O
+Arlésienne _ _ B-NamedPerson
+to _ _ O
+the _ _ O
+shepherdess _ _ O
+in _ _ O
+a _ _ O
+fairy _ _ O
+tale _ _ O
+, _ _ O
+with _ _ O
+faille _ _ B-ProductPart
+, _ _ O
+piqué _ _ B-ProductPart
+, _ _ O
+taffeta _ _ B-ProductPart
+, _ _ O
+tulle _ _ B-ProductPart
+, _ _ O
+embroidery _ _ B-ProductPart
+, _ _ O
+lace _ _ B-ProductPart
+, _ _ O
+the _ _ O
+repertoire _ _ O
+is _ _ O
+inexhaustible _ _ O
+. _ _ O
+
+
+
+
+Subscribe _ _ O
+to _ _ O
+Highsnobiety _ _ B-NamedOrganizationPublisher
+on _ _ O
+YouTube _ _ B-NamedOrganizationOther
+Eric _ _ B-NamedPerson
+Schoenborn _ _ I-NamedPerson
+and _ _ O
+Ed _ _ B-NamedPerson
+Selego _ _ I-NamedPerson
+have _ _ O
+joined _ _ O
+forces _ _ O
+with _ _ O
+Nocturnal _ _ B-NamedOrganizationBrand
+skate _ _ B-Activity
+shop _ _ O
+to _ _ O
+turn _ _ O
+Drexel _ _ B-NamedLocation
+University _ _ I-NamedLocation
+’ _ _ O
+s _ _ O
+Leonard _ _ B-NamedLocation
+Pearlstein _ _ I-NamedLocation
+Gallery _ _ I-NamedLocation
+into _ _ O
+an _ _ O
+interactive _ _ O
+skate _ _ B-Activity
+pop _ _ O
+- _ _ O
+up _ _ O
+park _ _ O
+. _ _ O
+
+Philly _ _ B-NamedPerson
+Radness _ _ I-NamedPerson
+accounts _ _ O
+for _ _ O
+the _ _ O
+second _ _ O
+installment _ _ O
+in _ _ O
+the _ _ O
+Phenomenal _ _ O
+Radness _ _ O
+project _ _ O
+, _ _ O
+after _ _ O
+its _ _ O
+debut _ _ O
+in _ _ O
+Miami _ _ B-NamedLocation
+a _ _ O
+few _ _ O
+years _ _ O
+ago _ _ O
+. _ _ O
+
+Milan _ _ B-NamedLocation
+was _ _ O
+all _ _ O
+the _ _ O
+really _ _ O
+big _ _ O
+girls _ _ O
+. _ _ O
+
+It _ _ O
+was _ _ O
+the _ _ O
+best _ _ O
+! _ _ O
+
+We _ _ O
+go _ _ O
+to _ _ O
+flea _ _ O
+markets _ _ O
+together _ _ O
+when _ _ O
+we _ _ O
+' _ _ O
+re _ _ O
+in _ _ O
+LA _ _ B-NamedLocation
+. _ _ O
\ No newline at end of file
diff --git a/tests/test_data.py b/tests/test_data.py
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -30,6 +30,28 @@ def test_create_sentence_without_tokenizer():
     assert ('love' == sentence.tokens[1].text)
     assert ('Berlin.' == sentence.tokens[2].text)
 
+def test_token_indices():
+
+    text = ':    nation on'
+    sentence = Sentence(text)
+    assert (text == sentence.to_original_text())
+
+    text = ':    nation on'
+    sentence = Sentence(text, use_tokenizer=True)
+    assert (text == sentence.to_original_text())
+
+    text = 'I love Berlin.'
+    sentence = Sentence(text)
+    assert (text == sentence.to_original_text())
+
+    text = 'Schartau sagte dem " Tagesspiegel " vom Freitag , Fischer sei " in einer Weise aufgetreten , die alles andere als überzeugend war " .'
+    sentence = Sentence(text)
+    assert (text == sentence.to_original_text())
+
+    text = 'Schartau sagte dem " Tagesspiegel " vom Freitag , Fischer sei " in einer Weise aufgetreten , die alles andere als überzeugend war " .'
+    sentence = Sentence(text, use_tokenizer=True)
+    assert (text == sentence.to_original_text())
+
 
 def test_create_sentence_with_tokenizer():
     sentence: Sentence = Sentence('I love Berlin.', use_tokenizer=True)
diff --git a/tests/test_data_fetchers.py b/tests/test_data_fetchers.py
--- a/tests/test_data_fetchers.py
+++ b/tests/test_data_fetchers.py
@@ -44,3 +44,21 @@ def test_load_ud_english_data(tasks_base_path):
     assert len(corpus.train) == 6
     assert len(corpus.test) == 4
     assert len(corpus.dev) == 2
+
+
+def test_load_no_dev_data(tasks_base_path):
+    # get training, test and dev data
+    corpus = NLPTaskDataFetcher.fetch_column_corpus(tasks_base_path / 'fashion_nodev', {0: 'text', 2: 'ner'})
+
+    assert len(corpus.train) == 5
+    assert len(corpus.dev) == 1
+    assert len(corpus.test) == 1
+
+
+def test_multi_corpus(tasks_base_path):
+    # get two corpora as one
+    corpus = NLPTaskDataFetcher.fetch_corpora([NLPTask.FASHION, NLPTask.GERMEVAL])
+
+    assert len(corpus.train) == 8
+    assert len(corpus.dev) == 2
+    assert len(corpus.test) == 2
\ No newline at end of file
diff --git a/tests/test_model_integration.py b/tests/test_model_integration.py
--- a/tests/test_model_integration.py
+++ b/tests/test_model_integration.py
@@ -312,3 +312,35 @@ def test_train_language_model(results_base_path, resources_path):
 
     # clean up results directory
     shutil.rmtree(results_base_path, ignore_errors=True)
+
+
+@pytest.mark.integration
+def test_train_load_use_tagger_multicorpus(results_base_path, tasks_base_path):
+
+    corpus = NLPTaskDataFetcher.fetch_corpora([NLPTask.FASHION, NLPTask.GERMEVAL], base_path=tasks_base_path)
+    tag_dictionary = corpus.make_tag_dictionary('ner')
+
+    embeddings = WordEmbeddings('glove')
+
+    tagger: SequenceTagger = SequenceTagger(hidden_size=256,
+                                            embeddings=embeddings,
+                                            tag_dictionary=tag_dictionary,
+                                            tag_type='ner',
+                                            use_crf=False)
+
+    # initialize trainer
+    trainer: ModelTrainer = ModelTrainer(tagger, corpus)
+
+    trainer.train(str(results_base_path), learning_rate=0.1, mini_batch_size=2, max_epochs=2, test_mode=True)
+
+    loaded_model: SequenceTagger = SequenceTagger.load_from_file(results_base_path / 'final-model.pt')
+
+    sentence = Sentence('I love Berlin')
+    sentence_empty = Sentence('       ')
+
+    loaded_model.predict(sentence)
+    loaded_model.predict([sentence, sentence_empty])
+    loaded_model.predict([sentence_empty])
+
+    # clean up results directory
+    shutil.rmtree(results_base_path)