Repository: flairNLP/flair

Issue:
Text classification error when using character embeddings whilst employing an LSTM
I am trying to do a simple binary text classification using `DocumentRNNEmbeddings` and `TextClassifier` where the embedding used is the `CharacterEmbeddings()`. 

The model looks like so:

```
def model(config, language='english'):
    """

    assert torch.cuda.is_available(), "Do not run this model with GPU support."
    # Specfiy the directory where all our results and data will be stored.
    lang_dir = Path('char_' + str(language) + "/")

    if language == 'english':
        # Note the name of the path (test,dev and train get identified automatically)
        corpus = ClassificationCorpus(DATA_ROOT_FASTTEXT / lang_dir)
        # Set word embeddings here
        word_embeddings = [CharacterEmbeddings()]
        # document_embeddings = [CharacterEmbeddings()]
    else:
        raise ValueError("Language {} is not supported.".format(language))

    # Combine embeddings to make a "document"
    document_embeddings = DocumentRNNEmbeddings(word_embeddings,
                                                hidden_size=config.hidden_size,
                                                reproject_words=config.reproject_words,
                                                bidirectional=config.bidirectional,
                                                rnn_type=config.rnn_type,
                                                rnn_layers=config.rnn_layers,
                                                reproject_words_dimension=config.reproject_words_dimension)

    # Classify said document using a TextClassifer
    classifier = TextClassifier(document_embeddings,
                                label_dictionary=corpus.make_label_dictionary(),
                                multi_label=False)

    # Specify a training instance
    trainer = ModelTrainer(classifier, corpus)

    # Train model
    trainer.train(DATA_ROOT_FASTTEXT / lang_dir,
                  learning_rate=config.learning_rate,
                  mini_batch_size=config.mini_batch_size,
                  anneal_factor=config.anneal_factor,
                  patience=config.patience,
                  max_epochs=config.max_epochs)
```

where `config` is simply a class which stores all the model parameters.

```
# Config parameters for the model training
config = Config(
    
    # RNN model specification
    hidden_size=8,
    reproject_words=True,
    bidirectional=True,
    rnn_type="LSTM", # LSTM does not work for some reason
    rnn_layers=1,
    reproject_words_dimension=None,
    dropout=0.5,
    
    # Training
    evaluation_metric=EvaluationMetric.MICRO_F1_SCORE,
    learning_rate=0.001,
    anneal_factor=0.5,
    mini_batch_size=64,
    patience=3,
    max_epochs=10
)
```


**Expected behavior**

The expected behaviour when I set `rnn_type="LSTM"` is for `flair` to recognise `LSTM`.

What I get is this error:

```
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-19-e27894ad7da5> in <module>
      1 assert torch.cuda.is_available(), "Do not run this model with GPU support."
      2 # Run model
----> 3 model(config)

~/cloud/model.py in model(config, language, optimise)
     77                   anneal_factor=config.anneal_factor,
     78                   patience=config.patience,
---> 79                   max_epochs=config.max_epochs)
     80 
     81 

~/anaconda3/envs/py36/lib/python3.6/site-packages/flair/trainers/trainer.py in train(self, base_path, evaluation_metric, learning_rate, mini_batch_size, eval_mini_batch_size, max_epochs, anneal_factor, patience, train_with_dev, monitor_train, embeddings_in_memory, checkpoint, save_final_model, anneal_with_restarts, shuffle, param_selection_mode, num_workers, **kwargs)
     91                 [Sentence("d", labels=["0.1"])],
     92                 eval_mini_batch_size,
---> 93                 embeddings_in_memory,
     94             )
     95             if log_train:

~/anaconda3/envs/py36/lib/python3.6/site-packages/flair/models/text_classification_model.py in evaluate(self, sentences, eval_mini_batch_size, embeddings_in_memory, out_path)
    175                 batch_count += 1
    176 
--> 177                 labels, loss = self.forward_labels_and_loss(batch)
    178 
    179                 clear_embeddings(

~/anaconda3/envs/py36/lib/python3.6/site-packages/flair/models/text_classification_model.py in forward_labels_and_loss(self, sentences)
    106         self, sentences: Union[Sentence, List[Sentence]]
    107     ) -> (List[List[Label]], torch.tensor):
--> 108         scores = self.forward(sentences)
    109         labels = self._obtain_labels(scores)
    110         loss = self._calculate_loss(scores, sentences)

~/anaconda3/envs/py36/lib/python3.6/site-packages/flair/models/text_classification_model.py in forward(self, sentences)
     68 
     69     def forward(self, sentences) -> List[List[float]]:
---> 70         self.document_embeddings.embed(sentences)
     71 
     72         text_embedding_list = [

~/anaconda3/envs/py36/lib/python3.6/site-packages/flair/embeddings.py in embed(self, sentences)
   2079         self.rnn.flatten_parameters()
   2080 
-> 2081         rnn_out, hidden = self.rnn(packed)
   2082 
   2083         outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(rnn_out)

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    491             result = self._slow_forward(*input, **kwargs)
    492         else:
--> 493             result = self.forward(*input, **kwargs)
    494         for hook in self._forward_hooks.values():
    495             hook_result = hook(self, input, result)

~/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
    206 
    207         self.check_forward_args(input, hx, batch_sizes)
--> 208         _impl = _rnn_impls[self.mode]
    209         if batch_sizes is None:
    210             result = _impl(input, hx, self._get_flat_weights(), self.bias, self.num_layers,

KeyError: 'LSTM'
```
Which is a bit odd.

**EDIT**

I [just tried your example in tutorial 7](https://github.com/zalandoresearch/flair/blob/master/resources/docs/TUTORIAL_7_TRAINING_A_MODEL.md), and it does not work either when we select `rnn_type='LSTM'` for `DocumentRNNEmbeddings`.


Discussion:
Thanks for reporting this - I can reproduce the error... very odd. I'll take a closer look.

Resolving patch:
diff --git a/flair/embeddings.py b/flair/embeddings.py
--- a/flair/embeddings.py
+++ b/flair/embeddings.py
@@ -1958,7 +1958,7 @@ def __init__(
         :param dropout: the dropout value to be used
         :param word_dropout: the word dropout value to be used, if 0.0 word dropout is not used
         :param locked_dropout: the locked dropout value to be used, if 0.0 locked dropout is not used
-        :param rnn_type: 'GRU', 'LSTM',  'RNN_TANH' or 'RNN_RELU'
+        :param rnn_type: 'GRU' or 'LSTM'
         """
         super().__init__()
 
@@ -1981,17 +1981,25 @@ def __init__(
         if self.reproject_words and reproject_words_dimension is not None:
             self.embeddings_dimension = reproject_words_dimension
 
-        # bidirectional RNN on top of embedding layer
         self.word_reprojection_map = torch.nn.Linear(
             self.length_of_all_token_embeddings, self.embeddings_dimension
         )
-        self.rnn = torch.nn.RNNBase(
-            rnn_type,
-            self.embeddings_dimension,
-            hidden_size,
-            num_layers=rnn_layers,
-            bidirectional=self.bidirectional,
-        )
+
+        # bidirectional RNN on top of embedding layer
+        if rnn_type == "LSTM":
+            self.rnn = torch.nn.LSTM(
+                self.embeddings_dimension,
+                hidden_size,
+                num_layers=rnn_layers,
+                bidirectional=self.bidirectional,
+            )
+        else:
+            self.rnn = torch.nn.GRU(
+                self.embeddings_dimension,
+                hidden_size,
+                num_layers=rnn_layers,
+                bidirectional=self.bidirectional,
+            )
 
         self.name = "document_" + self.rnn._get_name()
 


Test patch: