# Buggy code
def __init__(
    self,
    model: str = "bert-base-uncased",
    layers: str = "-1,-2,-3,-4",
    pooling_operation: str = "first",
    batch_size: int = 1,
    use_scalar_mix: bool = False,
    fine_tune: bool = False,
    allow_long_sentences: bool = False,
    **kwargs,
):
    """
    Bidirectional transformer embeddings of words from various transformer architectures.
    :param model: name of transformer model (see https://huggingface.co/transformers/pretrained_models.html for
    options)
    :param layers: string indicating which layers to take for embedding (-1 is topmost layer)
    :param pooling_operation: how to get from token piece embeddings to token embedding. Either take the first
    subtoken ('first'), the last subtoken ('last'), both first and last ('first_last') or a mean over all ('mean')
    :param batch_size: How many sentence to push through transformer at once. Set to 1 by default since transformer
    models tend to be huge.
    :param use_scalar_mix: If True, uses a scalar mix of layers as embedding
    :param fine_tune: If True, allows transformers to be fine-tuned during training
    """
    super().__init__()

    # temporary fix to disable tokenizer parallelism warning
    # (see https://stackoverflow.com/questions/62691279/how-to-disable-tokenizers-parallelism-true-false-warning)
    import os

    os.environ["TOKENIZERS_PARALLELISM"] = "false"

    # load tokenizer and transformer model
    self.tokenizer = AutoTokenizer.from_pretrained(model, **kwargs)
    config = AutoConfig.from_pretrained(model, output_hidden_states=True, **kwargs)
    self.model = AutoModel.from_pretrained(model, config=config, **kwargs)

    self.allow_long_sentences = allow_long_sentences

    if allow_long_sentences:
        self.max_subtokens_sequence_length = self.tokenizer.model_max_length
        self.stride = self.tokenizer.model_max_length // 2
    else:
        self.max_subtokens_sequence_length = None
        self.stride = 0

    # model name
    self.name = "transformer-word-" + str(model)

    # when initializing, embeddings are in eval mode by default
    self.model.eval()
    self.model.to(flair.device)

    # embedding parameters
    if layers == "all":
        # send mini-token through to check how many layers the model has
        hidden_states = self.model(torch.tensor([1], device=flair.device).unsqueeze(0))[
            -1
        ]
        self.layer_indexes = [int(x) for x in range(len(hidden_states))]
    else:
        self.layer_indexes = [int(x) for x in layers.split(",")]
    # self.mix = ScalarMix(mixture_size=len(self.layer_indexes), trainable=False)
    self.pooling_operation = pooling_operation
    self.use_scalar_mix = use_scalar_mix
    self.fine_tune = fine_tune
    self.static_embeddings = not self.fine_tune
    self.batch_size = batch_size

    self.special_tokens = []
    # check if special tokens exist to circumvent error message
    if self.tokenizer._bos_token:
        self.special_tokens.append(self.tokenizer.bos_token)
    if self.tokenizer._cls_token:
        self.special_tokens.append(self.tokenizer.cls_token)

    # most models have an intial BOS token, except for XLNet, T5 and GPT2
    self.begin_offset = 1
    if type(self.tokenizer) == XLNetTokenizer:
        self.begin_offset = 0
    if type(self.tokenizer) == T5Tokenizer:
        self.begin_offset = 0
    if type(self.tokenizer) == GPT2Tokenizer:
        self.begin_offset = 0


# Traceback (TypeError)
----------------------------------------------------------------------------------------------------
2020-06-12 18:44:49,152 Model training base path: "dialogpt"
2020-06-12 18:44:49,152 ----------------------------------------------------------------------------------------------------

2020-06-12 18:44:49,153 Device: cuda:0
2020-06-12 18:44:49,154 ----------------------------------------------------------------------------------------------------
2020-06-12 18:44:49,155 Embeddings storage mode: cpu
2020-06-12 18:44:49,230 ----------------------------------------------------------------------------------------------------
2020-06-12 18:49:00,669 epoch 1 - iter 76/766 - loss 1.11887558 - samples/sec: 9.67

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-7-e12f92b71d0b> in <module>
7         mini_batch_chunk_size=2, # set this if you get OOM errors
8         max_epochs=1, # very few epochs of fine-tuning
----> 9         embeddings_storage_mode='cpu',
10         #checkpoint=True
11     )

~/venv36/lib64/python3.6/site-packages/flair/trainers/trainer.py in train(self, base_path, learning_rate, mini_batch_size, mini_batch_chunk_size, max_epochs, scheduler, anneal_factor, patience, initial_extra_patience, min_learning_rate, train_with_dev, monitor_train, monitor_test, embeddings_storage_mode, checkpoint, save_final_model, anneal_with_restarts, anneal_with_prestarts, batch_growth_annealing, shuffle, param_selection_mode, num_workers, sampler, use_amp, amp_opt_level, eval_on_train_fraction, eval_on_train_shuffle, **kwargs)
376                         if not param_selection_mode:
377                             weight_extractor.extract_weights(
--> 378                                 self.model.state_dict(), iteration
379                             )
380

~/venv36/lib64/python3.6/site-packages/flair/training_utils.py in extract_weights(self, state_dict, iteration)
262             vec = state_dict[key]
263             weights_to_watch = min(
--> 264                 self.number_of_weights, reduce(lambda x, y: x * y, list(vec.size()))
265             )
266

TypeError: reduce() of empty sequence with no initial value

# Fixed code
def __init__(
    self,
    model: str = "bert-base-uncased",
    layers: str = "-1,-2,-3,-4",
    pooling_operation: str = "first",
    batch_size: int = 1,
    use_scalar_mix: bool = False,
    fine_tune: bool = False,
    allow_long_sentences: bool = True,
    **kwargs,
):
    """
    Bidirectional transformer embeddings of words from various transformer architectures.
    :param model: name of transformer model (see https://huggingface.co/transformers/pretrained_models.html for
    options)
    :param layers: string indicating which layers to take for embedding (-1 is topmost layer)
    :param pooling_operation: how to get from token piece embeddings to token embedding. Either take the first
    subtoken ('first'), the last subtoken ('last'), both first and last ('first_last') or a mean over all ('mean')
    :param batch_size: How many sentence to push through transformer at once. Set to 1 by default since transformer
    models tend to be huge.
    :param use_scalar_mix: If True, uses a scalar mix of layers as embedding
    :param fine_tune: If True, allows transformers to be fine-tuned during training
    """
    super().__init__()

    # temporary fix to disable tokenizer parallelism warning
    # (see https://stackoverflow.com/questions/62691279/how-to-disable-tokenizers-parallelism-true-false-warning)
    import os

    os.environ["TOKENIZERS_PARALLELISM"] = "false"

    # load tokenizer and transformer model
    self.tokenizer = AutoTokenizer.from_pretrained(model, **kwargs)
    config = AutoConfig.from_pretrained(model, output_hidden_states=True, **kwargs)
    self.model = AutoModel.from_pretrained(model, config=config, **kwargs)

    self.allow_long_sentences = allow_long_sentences

    if allow_long_sentences:
        self.max_subtokens_sequence_length = self.tokenizer.model_max_length
        self.stride = self.tokenizer.model_max_length // 2
    else:
        self.max_subtokens_sequence_length = self.tokenizer.model_max_length
        self.stride = 0

    # model name
    self.name = "transformer-word-" + str(model)

    # when initializing, embeddings are in eval mode by default
    self.model.eval()
    self.model.to(flair.device)

    # embedding parameters
    if layers == "all":
        # send mini-token through to check how many layers the model has
        hidden_states = self.model(torch.tensor([1], device=flair.device).unsqueeze(0))[
            -1
        ]
        self.layer_indexes = [int(x) for x in range(len(hidden_states))]
    else:
        self.layer_indexes = [int(x) for x in layers.split(",")]
    # self.mix = ScalarMix(mixture_size=len(self.layer_indexes), trainable=False)
    self.pooling_operation = pooling_operation
    self.use_scalar_mix = use_scalar_mix
    self.fine_tune = fine_tune
    self.static_embeddings = not self.fine_tune
    self.batch_size = batch_size

    self.special_tokens = []
    # check if special tokens exist to circumvent error message
    if self.tokenizer._bos_token:
        self.special_tokens.append(self.tokenizer.bos_token)
    if self.tokenizer._cls_token:
        self.special_tokens.append(self.tokenizer.cls_token)

    # most models have an intial BOS token, except for XLNet, T5 and GPT2
    self.begin_offset = 1
    if type(self.tokenizer) == XLNetTokenizer:
        self.begin_offset = 0
    if type(self.tokenizer) == T5Tokenizer:
        self.begin_offset = 0
    if type(self.tokenizer) == GPT2Tokenizer:
        self.begin_offset = 0