# Buggy code
def reconstruct_tokens_from_subtokens(self, sentence, subtokens):
    word_iterator = iter(sentence)
    token = next(word_iterator)
    token_text = self._get_processed_token_text(token)
    token_subtoken_lengths = []
    reconstructed_token = ""
    subtoken_count = 0
    # iterate over subtokens and reconstruct tokens
    for subtoken_id, subtoken in enumerate(subtokens):
        # remove special markup
        subtoken = self._remove_special_markup(subtoken)

        # TODO check if this is necessary is this method is called before prepare_for_model
        # check if reconstructed token is special begin token ([CLS] or similar)
        if subtoken in self.special_tokens and subtoken_id == 0:
            continue

        # some BERT tokenizers somehow omit words - in such cases skip to next token
        if subtoken_count == 0 and not token_text.startswith(subtoken.lower()):
            token_subtoken_lengths.append(0)
            token = next(word_iterator)
            token_text = self._get_processed_token_text(token)

        subtoken_count += 1

        # append subtoken to reconstruct token
        reconstructed_token = reconstructed_token + subtoken

        # check if reconstructed token is the same as current token
        if reconstructed_token.lower() == token_text:
            # if so, add subtoken count
            token_subtoken_lengths.append(subtoken_count)

            # reset subtoken count and reconstructed token
            reconstructed_token = ""
            subtoken_count = 0

            # break from loop if all tokens are accounted for
            if len(token_subtoken_lengths) < len(sentence):
                token = next(word_iterator)
                token_text = self._get_processed_token_text(token)
            else:
                break
    # check if all tokens were matched to subtokens
    if token != sentence[-1]:
        log.error(
            f"Tokenization MISMATCH in sentence '{sentence.to_tokenized_string()}'"
        )
        log.error(f"Last matched: '{token}'")
        log.error(f"Last sentence: '{sentence[-1]}'")
        log.error(f"subtokenized: '{subtokens}'")
    return token_subtoken_lengths


# Traceback (ValueError)
Traceback (most recent call last):
File "train_flair.py", line 187, in <module>
sys.exit(main(sys.argv[1:]))
File "train_flair.py", line 182, in main
trainer.train(args.outfolder, checkpoint=True, **hp)
File "/opt/conda/lib/python3.6/site-packages/flair/trainers/trainer.py", line 364, in train
loss = self.model.forward_loss(batch_step)
File "/opt/conda/lib/python3.6/site-packages/flair/models/text_classification_model.py", line 146, in forward_loss
scores = self.forward(data_points)
File "/opt/conda/lib/python3.6/site-packages/flair/models/text_classification_model.py", line 100, in forward
self.document_embeddings.embed(sentences)
File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/base.py", line 61, in embed
self._add_embeddings_internal(sentences)
File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/document.py", line 369, in _add_embeddings_internal
self.embeddings.embed(sentences)
File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/token.py", line 71, in embed
embedding.embed(sentences)
File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/base.py", line 61, in embed
self._add_embeddings_internal(sentences)
File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/token.py", line 892, in _add_embeddings_internal
self._add_embeddings_to_sentences(batch)
File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/token.py", line 965, in _add_embeddings_to_sentences
longest_sequence_in_batch: int = len(max(subtokenized_sentences, key=len))
ValueError: max() arg is an empty sequence

# Fixed code
def reconstruct_tokens_from_subtokens(self, sentence, subtokens):
    word_iterator = iter(sentence)
    token = next(word_iterator)
    token_text = self._get_processed_token_text(token)
    token_subtoken_lengths = []
    reconstructed_token = ""
    subtoken_count = 0
    # iterate over subtokens and reconstruct tokens
    for subtoken_id, subtoken in enumerate(subtokens):
        # remove special markup
        subtoken = self._remove_special_markup(subtoken)

        # TODO check if this is necessary is this method is called before prepare_for_model
        # check if reconstructed token is special begin token ([CLS] or similar)
        if subtoken in self.special_tokens and subtoken_id == 0:
            continue

        # some BERT tokenizers somehow omit words - in such cases skip to next token
        if subtoken_count == 0 and not token_text.startswith(subtoken.lower()):
            while True:
                token_subtoken_lengths.append(0)
                token = next(word_iterator)
                token_text = self._get_processed_token_text(token)
                if token_text.startswith(subtoken.lower()):
                    break

        subtoken_count += 1

        # append subtoken to reconstruct token
        reconstructed_token = reconstructed_token + subtoken

        # check if reconstructed token is the same as current token
        if reconstructed_token.lower() == token_text:
            # if so, add subtoken count
            token_subtoken_lengths.append(subtoken_count)

            # reset subtoken count and reconstructed token
            reconstructed_token = ""
            subtoken_count = 0

            # break from loop if all tokens are accounted for
            if len(token_subtoken_lengths) < len(sentence):
                token = next(word_iterator)
                token_text = self._get_processed_token_text(token)
            else:
                break

    # if tokens are unaccounted for
    while len(token_subtoken_lengths) < len(sentence) and len(token.text) == 1:
        token_subtoken_lengths.append(0)
        if len(token_subtoken_lengths) == len(sentence):
            break
        token = next(word_iterator)

    # check if all tokens were matched to subtokens
    if token != sentence[-1]:
        log.error(
            f"Tokenization MISMATCH in sentence '{sentence.to_tokenized_string()}'"
        )
        log.error(f"Last matched: '{token}'")
        log.error(f"Last sentence: '{sentence[-1]}'")
        log.error(f"subtokenized: '{subtokens}'")
    return token_subtoken_lengths