# Buggy code
def eval(self, model):
    """
    Performs evaluation on a given model.

    :param model: The model on which to perform evaluation
    :type model: AdaptiveModel
    :return all_results: A list of dictionaries, one for each prediction head. Each dictionary contains the metrics
                         and reports generated during evaluation.
    :rtype all_results: list of dicts
    """
    model.eval()

    # init empty lists per prediction head
    loss_all = [0 for _ in model.prediction_heads]
    preds_all = [[] for _ in model.prediction_heads]
    label_all = [[] for _ in model.prediction_heads]

    for step, batch in enumerate(
        tqdm(self.data_loader, desc="Evaluating", mininterval=10)
    ):
        batch = {key: batch[key].to(self.device) for key in batch}

        with torch.no_grad():
            logits = model.forward(**batch)
            # TODO logits_to_loss should be a single, overloaded function
            losses_per_head = model.logits_to_loss_per_head(logits=logits, **batch)
            preds = model.logits_to_preds(logits=logits, **batch)

            labels = model.prepare_labels(**batch)

        # stack results of all batches per prediction head
        for head_num, head in enumerate(model.prediction_heads):
            loss_all[head_num] += np.sum(to_numpy(losses_per_head[head_num]))
            if head.model_type == "span_classification":
                # TODO check why adaptive model doesnt pack preds into list of list of tuples
                preds_all[head_num] += preds
                label_all[head_num] += labels
            else:
                preds_all[head_num] += list(to_numpy(preds[head_num]))
                label_all[head_num] += list(to_numpy(labels[head_num]))

    # Evaluate per prediction head
    all_results = []
    for head_num, head in enumerate(model.prediction_heads):
        if head.model_type == "multilabel_text_classification":
            # converting from string preds back to multi-hot encoding
            from sklearn.preprocessing import MultiLabelBinarizer

            mlb = MultiLabelBinarizer(classes=head.label_list)
            # TODO check why .fit() should be called on predictions, rather than on labels
            preds_all[head_num] = mlb.fit_transform(preds_all[head_num])
            label_all[head_num] = mlb.transform(label_all[head_num])
        elif head.model_type == "span_classification":
            temp = head._aggregate_preds(preds_all[head_num])
            preds_all[head_num] = temp

        result = {
            "loss": loss_all[head_num] / len(self.data_loader.dataset),
            "task_name": head.task_name,
        }
        result.update(
            compute_metrics(
                metric=head.metric,
                preds=preds_all[head_num],
                labels=label_all[head_num],
            )
        )

        # Select type of report depending on prediction head output type
        if self.classification_report:
            if head.ph_output_type == "per_token":
                report_fn = token_classification_report
            elif head.ph_output_type == "per_sequence":
                report_fn = classification_report
            elif head.ph_output_type == "per_token_squad":
                report_fn = lambda *args, **kwargs: "not Implemented"
            elif head.ph_output_type == "per_sequence_continuous":
                report_fn = r2_score
            else:
                raise NotImplementedError

            # CHANGE PARAMETERS, not all report_fn accept digits
            if head.ph_output_type in ["per_sequence_continuous", "per_token"]:
                result["report"] = report_fn(label_all[head_num], preds_all[head_num])
            else:
                # supply labels as all possible combination because if ground truth labels do not cover
                # all values in label_list (maybe dev set is small), the report will break
                result["report"] = report_fn(
                    label_all[head_num],
                    preds_all[head_num],
                    digits=4,
                    labels=head.label_list,
                    target_names=head.label_list,
                )

        all_results.append(result)

    return all_results


# Traceback (TypeError)
${PYTHONENVHOME}/lib/python3.6/site-packages/numpy/lib/arraysetops.py:564: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
mask &amp;= (ar1 != a)
Traceback (most recent call last):
File "doc_classification_multilabel.py", line 97, in <module>
model = trainer.train(model)
File "${PYTHONENVHOME}/lib/python3.6/site-packages/farm-0.3.1-py3.6.egg/farm/train.py", line 163, in train
File "${PYTHONENVHOME}/lib/python3.6/site-packages/farm-0.3.1-py3.6.egg/farm/eval.py", line 138, in eval
File "${PYTHONENVHOME}/lib/python3.6/site-packages/sklearn/metrics/classification.py", line 1886, in classification_report
sample_weight=sample_weight)
File "${PYTHONENVHOME}/lib/python3.6/site-packages/sklearn/metrics/classification.py", line 1421, in precision_recall_fscore_support
labels=labels, samplewise=samplewise)
File "${PYTHONENVHOME}/lib/python3.6/site-packages/sklearn/metrics/classification.py", line 457, in multilabel_confusion_matrix
if np.max(labels) > np.max(present_labels):
File "<__array_function__ internals>", line 6, in amax
File "${PYTHONENVHOME}/lib/python3.6/site-packages/numpy/core/fromnumeric.py", line 2621, in amax
keepdims=keepdims, initial=initial, where=where)
File "${PYTHONENVHOME}/lib/python3.6/site-packages/numpy/core/fromnumeric.py", line 90, in _wrapreduction
return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
TypeError: cannot perform reduce with flexible type

# Fixed code
def eval(self, model):
    """
    Performs evaluation on a given model.

    :param model: The model on which to perform evaluation
    :type model: AdaptiveModel
    :return all_results: A list of dictionaries, one for each prediction head. Each dictionary contains the metrics
                         and reports generated during evaluation.
    :rtype all_results: list of dicts
    """
    model.eval()

    # init empty lists per prediction head
    loss_all = [0 for _ in model.prediction_heads]
    preds_all = [[] for _ in model.prediction_heads]
    label_all = [[] for _ in model.prediction_heads]

    for step, batch in enumerate(
        tqdm(self.data_loader, desc="Evaluating", mininterval=10)
    ):
        batch = {key: batch[key].to(self.device) for key in batch}

        with torch.no_grad():
            logits = model.forward(**batch)
            # TODO logits_to_loss should be a single, overloaded function
            losses_per_head = model.logits_to_loss_per_head(logits=logits, **batch)
            preds = model.logits_to_preds(logits=logits, **batch)

            labels = model.prepare_labels(**batch)

        # stack results of all batches per prediction head
        for head_num, head in enumerate(model.prediction_heads):
            loss_all[head_num] += np.sum(to_numpy(losses_per_head[head_num]))
            if head.model_type == "span_classification":
                # TODO check why adaptive model doesnt pack preds into list of list of tuples
                preds_all[head_num] += preds
                label_all[head_num] += labels
            else:
                preds_all[head_num] += list(to_numpy(preds[head_num]))
                label_all[head_num] += list(to_numpy(labels[head_num]))

    # Evaluate per prediction head
    all_results = []
    for head_num, head in enumerate(model.prediction_heads):
        if head.model_type == "multilabel_text_classification":
            # converting from string preds back to multi-hot encoding
            from sklearn.preprocessing import MultiLabelBinarizer

            mlb = MultiLabelBinarizer(classes=head.label_list)
            # TODO check why .fit() should be called on predictions, rather than on labels
            preds_all[head_num] = mlb.fit_transform(preds_all[head_num])
            label_all[head_num] = mlb.transform(label_all[head_num])
        elif head.model_type == "span_classification":
            temp = head._aggregate_preds(preds_all[head_num])
            preds_all[head_num] = temp

        result = {
            "loss": loss_all[head_num] / len(self.data_loader.dataset),
            "task_name": head.task_name,
        }
        result.update(
            compute_metrics(
                metric=head.metric,
                preds=preds_all[head_num],
                labels=label_all[head_num],
            )
        )

        # Select type of report depending on prediction head output type
        if self.classification_report:
            if head.ph_output_type == "per_token":
                report_fn = token_classification_report
            elif head.ph_output_type == "per_sequence":
                report_fn = classification_report
            elif head.ph_output_type == "per_token_squad":
                report_fn = lambda *args, **kwargs: "not Implemented"
            elif head.ph_output_type == "per_sequence_continuous":
                report_fn = r2_score
            else:
                raise NotImplementedError

            # CHANGE PARAMETERS, not all report_fn accept digits
            if head.ph_output_type in ["per_sequence_continuous", "per_token"]:
                result["report"] = report_fn(label_all[head_num], preds_all[head_num])
            else:
                # supply labels as all possible combination because if ground truth labels do not cover
                # all values in label_list (maybe dev set is small), the report will break
                if head.model_type == "multilabel_text_classification":
                    # For multilabel classification, we don't eval with string labels here, but with multihot vectors.
                    # Therefore we need to supply all possible label ids instead of label values.
                    all_possible_labels = list(range(len(head.label_list)))
                else:
                    all_possible_labels = head.label_list

                result["report"] = report_fn(
                    label_all[head_num],
                    preds_all[head_num],
                    digits=4,
                    labels=all_possible_labels,
                    target_names=head.label_list,
                )

        all_results.append(result)

    return all_results