# Buggy code
def __init__(
    self,
    tokenizer,
    max_seq_len,
    data_dir,
    train_filename="train.txt",
    dev_filename="dev.txt",
    test_filename="test.txt",
    dev_split=0.0,
    next_sent_pred=True,
    max_docs=None,
    proxies=None,
    **kwargs,
):
    """
    :param tokenizer: Used to split a sentence (str) into tokens.
    :param max_seq_len: Samples are truncated after this many tokens.
    :type max_seq_len: int
    :param data_dir: The directory in which the train and dev files can be found. Squad has a private test file
    :type data_dir: str
    :param label_list: list of labels to predict (strings). For most cases this should be: ["start_token", "end_token"]
    :type label_list: list
    :param metric: name of metric that shall be used for evaluation, e.g. "acc" or "f1_macro".
             Alternatively you can also supply a custom function, that takes preds and labels as args and returns a numerical value.
             For using multiple metrics supply them as a list, e.g ["acc", my_custom_metric_fn].
    :type metric: str, function, or list
    :param train_filename: The name of the file containing training data.
    :type train_filename: str
    :param dev_filename: The name of the file containing the dev data. If None and 0.0 < dev_split < 1.0 the dev set
                         will be a slice of the train set.
    :type dev_filename: str or None
    :param test_filename: None
    :type test_filename: str
    :param dev_split: The proportion of the train set that will sliced. Only works if dev_filename is set to None
    :type dev_split: float
    :param next_sent_pred: Whether to use next_sentence_prediction objective or not
    :type next_sent_pred: bool
    :param max_docs: maximum number of documents to include from input dataset
    :type max_docs: int
    :param proxies: proxy configuration to allow downloads of remote datasets.
                    Format as in  "requests" library: https://2.python-requests.org//en/latest/user/advanced/#proxies
    :type proxies: dict
    :param kwargs: placeholder for passing generic parameters
    :type kwargs: object
    """

    self.delimiter = ""
    self.max_docs = max_docs

    super(BertStyleLMProcessor, self).__init__(
        tokenizer=tokenizer,
        max_seq_len=max_seq_len,
        train_filename=train_filename,
        dev_filename=dev_filename,
        test_filename=test_filename,
        dev_split=dev_split,
        data_dir=data_dir,
        tasks={},
        proxies=proxies,
    )

    self.next_sent_pred = next_sent_pred

    self.add_task("lm", "acc", list(self.tokenizer.vocab))
    if self.next_sent_pred:
        self.add_task("nextsentence", "acc", ["False", "True"])


# Traceback (IndexError)
Train epoch 1/1 (Cur. train loss: 0.6664):  18%|█▊        | 30/170 [00:48<03:43,  1.60s/it]
Evaluating:   0%|          | 0/319 [00:00<?, ?it/s]
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-3-f9d4de447982> in <module>()
1 # 7. Let it grow! Watch the tracked metrics live on the public mlflow server: https://public-mlflow.deepset.ai
----> 2 model = trainer.train(model)

~/bertclassifier/FARM/farm/train.py in train(self, model)
224                     ):
225                         evalnr += 1
--> 226                         result = self.evaluator_dev.eval(model)
227                         self.evaluator_dev.log_results(result, "Dev", self.global_step)
228                         if self.early_stopping:

~/bertclassifier/FARM/farm/eval.py in eval(self, model, return_preds_and_labels)
71                 losses_per_head = model.logits_to_loss_per_head(logits=logits, **batch)
72                 preds = model.logits_to_preds(logits=logits, **batch)
---> 73                 labels = model.prepare_labels(**batch)
74
75             # stack results of all batches per prediction head

~/bertclassifier/FARM/farm/modeling/adaptive_model.py in prepare_labels(self, **kwargs)
170         #     all_labels.append(labels)
171         for head in self.prediction_heads:
--> 172             labels = head.prepare_labels(**kwargs)
173             all_labels.append(labels)
174         return all_labels

~/bertclassifier/FARM/farm/modeling/prediction_head.py in prepare_labels(self, **kwargs)
662         # we have a batch of sequences here. we need to convert for each token in each sequence.
663         for ids_for_sequence in label_ids:
--> 664             labels.append([self.label_list[int(x)] for x in ids_for_sequence if int(x) != -1])
665         return labels
666

~/bertclassifier/FARM/farm/modeling/prediction_head.py in <listcomp>(.0)
662         # we have a batch of sequences here. we need to convert for each token in each sequence.
663         for ids_for_sequence in label_ids:
--> 664             labels.append([self.label_list[int(x)] for x in ids_for_sequence if int(x) != -1])
665         return labels
666

IndexError: list index out of range

# Fixed code
def __init__(
    self,
    tokenizer,
    max_seq_len,
    data_dir,
    train_filename="train.txt",
    dev_filename="dev.txt",
    test_filename="test.txt",
    dev_split=0.0,
    next_sent_pred=True,
    max_docs=None,
    proxies=None,
    **kwargs,
):
    """
    :param tokenizer: Used to split a sentence (str) into tokens.
    :param max_seq_len: Samples are truncated after this many tokens.
    :type max_seq_len: int
    :param data_dir: The directory in which the train and dev files can be found. Squad has a private test file
    :type data_dir: str
    :param label_list: list of labels to predict (strings). For most cases this should be: ["start_token", "end_token"]
    :type label_list: list
    :param metric: name of metric that shall be used for evaluation, e.g. "acc" or "f1_macro".
             Alternatively you can also supply a custom function, that takes preds and labels as args and returns a numerical value.
             For using multiple metrics supply them as a list, e.g ["acc", my_custom_metric_fn].
    :type metric: str, function, or list
    :param train_filename: The name of the file containing training data.
    :type train_filename: str
    :param dev_filename: The name of the file containing the dev data. If None and 0.0 < dev_split < 1.0 the dev set
                         will be a slice of the train set.
    :type dev_filename: str or None
    :param test_filename: None
    :type test_filename: str
    :param dev_split: The proportion of the train set that will sliced. Only works if dev_filename is set to None
    :type dev_split: float
    :param next_sent_pred: Whether to use next_sentence_prediction objective or not
    :type next_sent_pred: bool
    :param max_docs: maximum number of documents to include from input dataset
    :type max_docs: int
    :param proxies: proxy configuration to allow downloads of remote datasets.
                    Format as in  "requests" library: https://2.python-requests.org//en/latest/user/advanced/#proxies
    :type proxies: dict
    :param kwargs: placeholder for passing generic parameters
    :type kwargs: object
    """

    self.delimiter = ""
    self.max_docs = max_docs

    super(BertStyleLMProcessor, self).__init__(
        tokenizer=tokenizer,
        max_seq_len=max_seq_len,
        train_filename=train_filename,
        dev_filename=dev_filename,
        test_filename=test_filename,
        dev_split=dev_split,
        data_dir=data_dir,
        tasks={},
        proxies=proxies,
    )

    self.next_sent_pred = next_sent_pred
    added_tokens = self.get_added_tokens()
    self.add_task("lm", "acc", list(self.tokenizer.vocab) + added_tokens)
    if self.next_sent_pred:
        self.add_task("nextsentence", "acc", ["False", "True"])