# Buggy code
def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
    self.context_embeddings.embed(sentences)

    # if we keep a pooling, it needs to be updated continuously
    for sentence in sentences:
        for token in sentence.tokens:
            # update embedding
            local_embedding = token._embeddings[self.context_embeddings.name]

            # check token.text is empty or not
            if token.text:
                if token.text[0].isupper() or not self.only_capitalized:
                    if token.text not in self.word_embeddings:
                        self.word_embeddings[token.text] = local_embedding
                        self.word_count[token.text] = 1
                    else:
                        aggregated_embedding = self.aggregate_op(
                            self.word_embeddings[token.text], local_embedding
                        )
                        if self.pooling == "fade":
                            aggregated_embedding /= 2
                        self.word_embeddings[token.text] = aggregated_embedding
                        self.word_count[token.text] += 1

    # add embeddings after updating
    for sentence in sentences:
        for token in sentence.tokens:
            if token.text in self.word_embeddings:
                base = (
                    self.word_embeddings[token.text] / self.word_count[token.text]
                    if self.pooling == "mean"
                    else self.word_embeddings[token.text]
                )
            else:
                base = token._embeddings[self.context_embeddings.name]

            token.set_embedding(self.name, base)

    return sentences


# Traceback (RuntimeError)
2019-11-12 09:10:16,480 Reading data from /home/username/.flair/datasets/conll_03_dutch
2019-11-12 09:10:16,480 Train: /home/username/.flair/datasets/conll_03_dutch/ned.train
2019-11-12 09:10:16,480 Dev: /home/username/.flair/datasets/conll_03_dutch/ned.testa
2019-11-12 09:10:16,480 Test: /home/username/.flair/datasets/conll_03_dutch/ned.testb
2019-11-12 09:10:28,986 ----------------------------------------------------------------------------------------------------
2019-11-12 09:10:28,987 Model: "SequenceTagger(
(embeddings): StackedEmbeddings(
(list_embedding_0): WordEmbeddings('nl')
(list_embedding_1): PooledFlairEmbeddings(
(context_embeddings): FlairEmbeddings(
(lm): LanguageModel(
(drop): Dropout(p=0.1, inplace=False)
(encoder): Embedding(7632, 100)
(rnn): LSTM(100, 2048)
(decoder): Linear(in_features=2048, out_features=7632, bias=True)
)
)
)
(list_embedding_2): PooledFlairEmbeddings(
(context_embeddings): FlairEmbeddings(
(lm): LanguageModel(
(drop): Dropout(p=0.1, inplace=False)
(encoder): Embedding(7632, 100)
(rnn): LSTM(100, 2048)
(decoder): Linear(in_features=2048, out_features=7632, bias=True)
)
)
)
)
(word_dropout): WordDropout(p=0.05)
(locked_dropout): LockedDropout(p=0.5)
(embedding2nn): Linear(in_features=8492, out_features=8492, bias=True)
(rnn): LSTM(8492, 256, batch_first=True, bidirectional=True)
(linear): Linear(in_features=512, out_features=20, bias=True)
)"
2019-11-12 09:10:28,987 ----------------------------------------------------------------------------------------------------
2019-11-12 09:10:28,987 Corpus: "Corpus: 15806 train + 2895 dev + 5195 test sentences"
2019-11-12 09:10:28,987 ----------------------------------------------------------------------------------------------------
2019-11-12 09:10:28,987 Parameters:
2019-11-12 09:10:28,987  - learning_rate: "0.1"
2019-11-12 09:10:28,987  - mini_batch_size: "32"
2019-11-12 09:10:28,987  - patience: "3"
2019-11-12 09:10:28,987  - anneal_factor: "0.5"
2019-11-12 09:10:28,987  - max_epochs: "150"
2019-11-12 09:10:28,987  - shuffle: "True"
2019-11-12 09:10:28,987  - train_with_dev: "True"
2019-11-12 09:10:28,987  - batch_growth_annealing: "False"
2019-11-12 09:10:28,987 ----------------------------------------------------------------------------------------------------
2019-11-12 09:10:28,987 Model training base path: "output/models/conll03-ner-test"
2019-11-12 09:10:28,987 ----------------------------------------------------------------------------------------------------
2019-11-12 09:10:28,987 Device: cuda:0
2019-11-12 09:10:28,987 ----------------------------------------------------------------------------------------------------
2019-11-12 09:10:28,987 Embeddings storage mode: cpu
2019-11-12 09:10:28,988 ----------------------------------------------------------------------------------------------------
train mode resetting embeddings
train mode resetting embeddings
2019-11-12 09:10:29,408 epoch 1 - iter 0/585 - loss 49.54404449 - samples/sec: 4426.54
Traceback (most recent call last):
File "embeddings/flair_evaluate_conll03-dutch.py", line 44, in <module>
max_epochs=150)
File "/home/username/.conda/envs/flair-test/lib/python3.7/site-packages/flair/trainers/trainer.py", line 325, in train
loss.backward()
File "/home/username/.conda/envs/flair-test/lib/python3.7/site-packages/torch/tensor.py", line 166, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "/home/username/.conda/envs/flair-test/lib/python3.7/site-packages/torch/autograd/__init__.py", line 99, in backward
allow_unreachable=True)  # allow_unreachable flag
RuntimeError: CUDA out of memory. Tried to allocate 276.00 MiB (GPU 0; 10.73 GiB total capacity; 8.33 GiB already allocated; 121.56 MiB free; 1.43 GiB cached)

# Fixed code
def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]:
    self.context_embeddings.embed(sentences)

    # if we keep a pooling, it needs to be updated continuously
    for sentence in sentences:
        for token in sentence.tokens:
            # update embedding
            local_embedding = token._embeddings[self.context_embeddings.name].cpu()

            # check token.text is empty or not
            if token.text:
                if token.text[0].isupper() or not self.only_capitalized:
                    if token.text not in self.word_embeddings:
                        self.word_embeddings[token.text] = local_embedding
                        self.word_count[token.text] = 1
                    else:
                        aggregated_embedding = self.aggregate_op(
                            self.word_embeddings[token.text], local_embedding
                        )
                        if self.pooling == "fade":
                            aggregated_embedding /= 2
                        self.word_embeddings[token.text] = aggregated_embedding
                        self.word_count[token.text] += 1

    # add embeddings after updating
    for sentence in sentences:
        for token in sentence.tokens:
            if token.text in self.word_embeddings:
                base = (
                    self.word_embeddings[token.text] / self.word_count[token.text]
                    if self.pooling == "mean"
                    else self.word_embeddings[token.text]
                )
            else:
                base = token._embeddings[self.context_embeddings.name]

            token.set_embedding(self.name, base)

    return sentences