Repository: flairNLP/flair

Issue:
Truncating sentences with max_tokens_per_doc when in_memory=False
It appears that argument `max_tokens_per_doc` has no effect when creating a `ClassificationCorpus` if `in_memory=False` (which is the default for `ClassificationCorpus`).

Looking at `ClassificationDataset` in `datasets.py`, I see that `max_tokens_per_doc` is not used when `self.in_memory` is False.

I also get an error due to long lines with the following code:

```python
from pathlib import Path 
from flair.datasets import ClassificationCorpus
from flair.embeddings import DocumentRNNEmbeddings, BertEmbeddings
from flair.models import TextClassifier
from flair.trainers import ModelTrainer

data_folder = Path('data_fasttext_format') # some of the sentences have long lines
corpus: ClassificationCorpus = ClassificationCorpus(data_folder,train_file='train.txt',dev_file='val.txt',test_file='val.txt', max_tokens_per_doc=20)
label_dict = corpus.make_label_dictionary()                                     
embs = [BertEmbeddings()]
document_embeddings: DocumentRNNEmbeddings = DocumentRNNEmbeddings(embs, rnn_type='LSTM')
classifier = TextClassifier(document_embeddings, label_dictionary=label_dict, multi_label=False)
trainer = ModelTrainer(classifier, corpus) 
trainer.train('data_fasttext_format')
```

This results in error ``RuntimeError: index out of range at /pytorch/aten/src/TH/generic/THTensorEvenMoreMath.cpp:191``. However, when I use `in_memory=True` when creating the `ClassificationDataset`, the sentences are truncated as expected and there are no errors.

Enviroment:
- Linux
- Python 3.6.2
- flair-0.4.2



Discussion:
Thanks for reporting this - looks like an error. I'll put in a PR to fix this!

Resolving patch:
diff --git a/flair/datasets.py b/flair/datasets.py
--- a/flair/datasets.py
+++ b/flair/datasets.py
@@ -810,6 +810,7 @@ def __init__(
 
         self.total_sentence_count: int = 0
         self.max_chars_per_doc = max_chars_per_doc
+        self.max_tokens_per_doc = max_tokens_per_doc
 
         self.path_to_file = path_to_file
 
@@ -826,12 +827,6 @@ def __init__(
                     sentence = self._parse_line_to_sentence(
                         line, self.label_prefix, use_tokenizer
                     )
-                    if (
-                        sentence is not None
-                        and len(sentence) > max_tokens_per_doc
-                        and max_tokens_per_doc > 0
-                    ):
-                        sentence.tokens = sentence.tokens[:max_tokens_per_doc]
                     if sentence is not None and len(sentence.tokens) > 0:
                         self.sentences.append(sentence)
                         self.total_sentence_count += 1
@@ -865,6 +860,14 @@ def _parse_line_to_sentence(
 
         if text and labels:
             sentence = Sentence(text, labels=labels, use_tokenizer=use_tokenizer)
+
+            if (
+                sentence is not None
+                and len(sentence) > self.max_tokens_per_doc
+                and self.max_tokens_per_doc > 0
+            ):
+                sentence.tokens = sentence.tokens[: self.max_tokens_per_doc]
+
             return sentence
         return None
 


Test patch:
diff --git a/tests/test_data_fetchers.py b/tests/test_data_fetchers.py
--- a/tests/test_data_fetchers.py
+++ b/tests/test_data_fetchers.py
@@ -28,6 +28,28 @@ def test_load_imdb_data_streaming(tasks_base_path):
     assert len(corpus.test) == 5
 
 
+def test_load_imdb_data_max_tokens(tasks_base_path):
+    # get training, test and dev data
+    corpus = flair.datasets.ClassificationCorpus(
+        tasks_base_path / "imdb", in_memory=True, max_tokens_per_doc=3
+    )
+
+    assert len(corpus.train[0]) <= 3
+    assert len(corpus.dev[0]) <= 3
+    assert len(corpus.test[0]) <= 3
+
+
+def test_load_imdb_data_streaming_max_tokens(tasks_base_path):
+    # get training, test and dev data
+    corpus = flair.datasets.ClassificationCorpus(
+        tasks_base_path / "imdb", in_memory=False, max_tokens_per_doc=3
+    )
+
+    assert len(corpus.train[0]) <= 3
+    assert len(corpus.dev[0]) <= 3
+    assert len(corpus.test[0]) <= 3
+
+
 def test_load_ag_news_data(tasks_base_path):
     # get training, test and dev data
     corpus = flair.datasets.ClassificationCorpus(tasks_base_path / "ag_news")