audeep/backend/formatters.py
class ConfusionMatrixFormatter:
    """
    Formatter for printing a confusion matrix with labels to stdout.
    """

    def __init__(self,
                 decimals: int = None,
                 normalize: bool = False,
                 abbrev_labels: int = 3):
        """
        Create a new ConfusionMatrixFormatter.
        
        Number of decimals can only be specified if normalization is enabled, otherwise it is ignored.
        
        Parameters
        ----------
        decimals: int, optional
            Number of decimals to print for each confusion matrix entry
        normalize: bool, default False
            Normalize confusion matrix to represent probabilities instead of counts.
        abbrev_labels: int, default 3
            Abbreviate nominal labels to have the specified length
        """
        self._normalize = normalize
        self._abbrev_labels = abbrev_labels

        if normalize:
            if decimals is not None:
                self._decimals = decimals
            else:
                self._decimals = 2

    def format(self,
               confusion_matrix: np.ndarray,
               label_map: Mapping[str, int]) -> str:
        """
        Print the specified confusion matrix using the specified label map.
        
        The `confusion_matrix` parameter must be a square numpy matrix, and the `label_map` parameter must be a mapping
        of nominal to numeric labels, containing one entry for each confusion matrix row/column.
        
        Parameters
        ----------
        confusion_matrix: numpy.ndarray
            A confusion matrix
        label_map: map of str to int
            A label map

        Returns
        -------
        str
            A string representation of the confusion matrix
        """
        if len(confusion_matrix.shape) != 2:
            raise ValueError("confusion matrix must be 2D, is: {}".format(len(confusion_matrix.shape)))
        elif confusion_matrix.shape[0] != confusion_matrix.shape[1]:
            raise ValueError("confusion matrix must be square, is {}".format(confusion_matrix.shape))
        elif confusion_matrix.shape[0] != len(label_map):
            raise ValueError("number of labels must match confusion matrix size, expected: {}, got: {}"
                             .format(confusion_matrix.shape[0], len(label_map)))

        # order nominal labels by numeric values
        ordered_labels = sorted(label_map.items(), key=lambda t: t[1])
        ordered_labels = list(zip(*ordered_labels))[0]

        if self._abbrev_labels is not None:
            ordered_labels = [label[:self._abbrev_labels] if len(label) > self._abbrev_labels else label
                              for label in ordered_labels]

        max_label_len = max(map(len, ordered_labels))

        if self._normalize:
            confusion_matrix = confusion_matrix / np.sum(confusion_matrix, axis=1)[:, np.newaxis]

            column_width = max(2 + self._decimals, max_label_len)
            number_pattern = " %%%d.%df" % (column_width, self._decimals)
        else:
            max_digits = int(math.ceil(math.log10(np.max(confusion_matrix))))

            column_width = max(max_digits, max_label_len)
            number_pattern = " %%%dd" % column_width

        result = ""

        label_pattern = "%%%ds" % column_width

        # print header row
        for col in range(confusion_matrix.shape[1] + 1):
            if col == 0:
                result += " " * max_label_len
            else:
                result += " " + label_pattern % ordered_labels[col - 1]

        result += "\n"

        label_pattern = "%%%ds" % max_label_len

        for row in range(confusion_matrix.shape[0]):
            result += label_pattern % ordered_labels[row]

            for col in range(confusion_matrix.shape[1]):
                result += number_pattern % confusion_matrix[row, col]

            result += "\n"

        return result
audeep/backend/learners.py
class TensorflowMLPLearner(LearnerBase):
    """
    A learner using Tensorflow to build an MLP classifier.
    """

    def __init__(self,
                 checkpoint_dir: Path,
                 num_layers: int,
                 num_units: int,
                 learning_rate=0.001,
                 num_epochs=400,
                 keep_prob=0.6,
                 shuffle_training=False):
        """
        Create and initialize a new TensorflowMLPLearner with the specified parameters.
        
        Due to limitations of Tensorflow (or my inability to find a better solution), this learner has to serialize
        trained model to the hard disk between invocations of the `fit` and `predict` methods.
        
        Furthermore, the entire training data is copied to GPU memory to increase training performance. Thus, only 
        moderately-sized models can be trained using this learner.
        
        Parameters
        ----------
        checkpoint_dir: pathlib.Path
            A directory in which temporary models can be stored
        num_layers: int
            The number of layers in the MLP
        num_units: int
            The number of units in each layer
        learning_rate: float, default 0.001
            The learning rate for training
        num_epochs: int, default 400
            The number of training epochs
        keep_prob: float, default 0.6
            The probability to keep hidden activations
        shuffle_training: bool, default False
            Shuffle training data between epochs
        """
        self._checkpoint_dir = checkpoint_dir
        self._latest_checkpoint = None
        self._num_labels = None
        self._model = MLPModel(num_layers, num_units)
        self._learning_rate = learning_rate
        self._num_epochs = num_epochs
        self._keep_prob = keep_prob
        self._shuffle_training = shuffle_training

    def fit(self,
            data_set: DataSet):
        # generic parameter checks
        super().fit(data_set)

        self._num_labels = len(data_set.label_map)

        graph = tf.Graph()

        with graph.as_default():
            tf_inputs = tf.Variable(initial_value=data_set.features, trainable=False, dtype=tf.float32)
            tf_labels = tf.Variable(initial_value=data_set.labels_numeric, trainable=False, dtype=tf.int32)

            if self._shuffle_training:
                tf_inputs = tf.random_shuffle(tf_inputs, seed=42)
                tf_labels = tf.random_shuffle(tf_labels, seed=42)

            with tf.variable_scope("mlp"):
                tf_logits = self._model.inference(tf_inputs, self._keep_prob, self._num_labels)
                tf_loss = self._model.loss(tf_logits, tf_labels)
                tf_train_op = self._model.optimize(tf_loss, self._learning_rate)

            tf_init_op = tf.global_variables_initializer()
            tf_saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="mlp"))

        session = tf.Session(graph=graph)
        session.run(tf_init_op)

        for epoch in range(self._num_epochs):
            session.run(tf_train_op)

        # timestamped model file
        self._latest_checkpoint = self._checkpoint_dir / "model_{:%Y%m%d%H%M%S%f}".format(datetime.datetime.now())
        tf_saver.save(session, str(self._latest_checkpoint), write_meta_graph=False)

        session.close()

    def predict(self,
                data_set: DataSet):
        # generic parameter checks
        super().predict(data_set)

        if self._latest_checkpoint is None:
            raise RuntimeError("no model has been built yet. Invoke fit before predict")

        graph = tf.Graph()

        with graph.as_default():
            tf_inputs = tf.placeholder(shape=[None, self.num_features], dtype=tf.float32, name="inputs")

            with tf.variable_scope("mlp"):
                tf_logits = self._model.inference(tf_inputs, 1.0, self._num_labels)
                tf_prediction = self._model.prediction(tf_logits)

            tf_saver = tf.train.Saver(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="mlp"))

        with tf.Session(graph=graph) as session:
            tf_saver.restore(session, str(self._latest_checkpoint))

            predictions = session.run(tf_prediction, feed_dict={tf_inputs: data_set.features})

        return predictions
audeep/backend/data/data_set.py
def load(path: Path) -> DataSet:
    """
    Loads a data set from the specified NetCDF4 file.
    
    Parameters
    ----------
    path: pathlib.Path
        Path to the file which should be loaded.

    Returns
    -------
    DataSet
        The data set loaded from the specified file
    """
    log = logging.getLogger(__name__)
    log.info("loading data set from %s", path)

    data = xr.open_dataset(str(path))  # type: xr.Dataset

    # restore data types
    data[_DataVar.FILENAME] = data[_DataVar.FILENAME].astype(np.object).fillna(None)
    data[_DataVar.CHUNK_NR] = data[_DataVar.CHUNK_NR].astype(np.object).fillna(None)
    data[_DataVar.CV_FOLDS] = data[_DataVar.CV_FOLDS].astype(np.object).fillna(None)
    data[_DataVar.PARTITION] = data[_DataVar.PARTITION].astype(np.object).fillna(None)
    data[_DataVar.LABEL_NOMINAL] = data[_DataVar.LABEL_NOMINAL].astype(np.object).fillna(None)
    data[_DataVar.LABEL_NUMERIC] = data[_DataVar.LABEL_NUMERIC].astype(np.object)
    data[_DataVar.FEATURES] = data[_DataVar.FEATURES].astype(np.float32)

    return DataSet(data=data,
                   mutable=False)
audeep/backend/learners.py
class LibLINEARLearner(LearnerBase):
    """
    A learner using LibLINEAR for classification. 
    """

    def __init__(self,
                 complexity: float):
        """
        Create and initialize a new LibLINEARLearner with the specified SVM complexity parameter.
        
        Parameters
        ----------
        complexity: float
            The SVM complexity parameter
        """
        self._model = LinearSVC(C=complexity)

    def fit(self,
            data_set: DataSet):
        # generic parameter checks
        super().fit(data_set)

        self._model.fit(X=data_set.features, y=data_set.labels_numeric)

    def predict(self,
                data_set: DataSet):
        # generic parameter checks
        super().predict(data_set)

        return self._model.predict(data_set.features)
audeep/backend/evaluation.py
class CrossValidatedEvaluation(LoggingMixin):
    """
    Cross-validated evaluation of a learner on some data.
    
    Given a data set and a learner, this class computes cross-validated accuracy and unweighted average recall, and a
    95% confidence interval for both values. Additionally, a confusion matrix is summed over all cross-validation folds.
    """

    def __init__(self,
                 learner: LearnerBase,
                 upsample: bool,
                 majority_vote: bool):
        """
        Creates and initializes a new cross-validated evaluation of the specified learner.
        
        The `run` method has to be invoked on some data before results can be retrieved.
        
        Parameters
        ----------
        learner: LearnerBase
            A learner which should be evaluated
        upsample: bool
            Balance classes in the training splits of each fold by upsampling instances
        """
        super().__init__()

        self._learner = learner
        self._upsample = upsample
        self._majority_vote = majority_vote
        self._accuracy = None
        self._accuracy_confidence_interval = None
        self._uar = None
        self._uar_confidence_interval = None
        self._confusion_matrix = None

    @property
    def accuracy(self) -> float:
        """
        Returns the accuracy of the learner.
        
        This property returns the accuracy on the last data set on which the `run` method has been invoked. As a 
        consequence, this property returns None if the `run` method has not yet been invoked.
        
        Returns
        -------
        float
            Returns the accuracy of the learner on the last data set passed to the `run` method
        """
        return self._accuracy

    @property
    def accuracy_confidence_interval(self) -> float:
        """
        Returns a 95% confidence interval for the accuracy of the learner.
        
        This property returns the confidence interval on the last data set on which the `run` method has been invoked. 
        As a consequence, this property returns None if the `run` method has not yet been invoked.
        
        Returns
        -------
        float
            Returns a 95% confidence interval for the accuracy of the learner on the last data set passed to the `run` 
            method
        """
        return self._accuracy_confidence_interval

    @property
    def uar(self) -> float:
        """
        Returns the unweighted average recall of the learner.
        
        This property returns the unweighted average recall on the last data set on which the `run` method has been 
        invoked. As a consequence, this property returns None if the `run` method has not yet been invoked.
        
        Returns
        -------
        float
            Returns the unweighted average recall of the learner on the last data set passed to the `run` method
        """
        return self._uar

    @property
    def uar_confidence_interval(self) -> float:
        """
        Returns a 95% confidence interval for the unweighted average recall of the learner.
        
        This property returns the confidence interval on the last data set on which the `run` method has been invoked. 
        As a consequence, this property returns None if the `run` method has not yet been invoked.
        
        Returns
        -------
        float
            Returns a 95% confidence interval for the unweighted average recall of the learner on the last data set 
            passed to the `run` method
        """
        return self._uar_confidence_interval

    @property
    def confusion_matrix(self) -> np.ndarray:
        """
        Returns the confusion matrix of the learner.
        
        This property returns the confusion matrix on the last data set on which the `run` method has been invoked. 
        As a consequence, this property returns None if the `run` method has not yet been invoked. The confusion matrix
        is computed as the sum of the confusion matrices on the individual cross-validation folds.
        
        Returns
        -------
        numpy.ndarray
            The confusion matrix of the learner on the last data set passed to the `run` method
        """
        return self._confusion_matrix

    def run(self,
            data_set: DataSet):
        """
        Evaluates the learner on the specified data set using cross-validation.
        
        Sets the various properties of this instance to the values obtained during evaluation on the specified data set.
        
        Parameters
        ----------
        data_set: DataSet
            The data set on which the learner should be evaluated

        Raises
        ------
        ValueError 
            If the specified data set does not have cross-validation information
        """
        if not data_set.has_cv_info:
            raise ValueError("data set does not have cross validation info")

        accuracies = []
        uars = []
        confusion_matrices = []

        # order numeric labels by nominal value
        ordered_labels = sorted(data_set.label_map.items(), key=lambda t: t[0])
        ordered_labels = list(zip(*ordered_labels))[1]

        for fold in range(data_set.num_folds):
            self.log.info("processing cross validation fold %d...", fold + 1)

            learner_wrapper = PreProcessingWrapper(learner=self._learner,
                                                   upsample=self._upsample,
                                                   majority_vote=self._majority_vote)

            train_split = data_set.split(fold=fold,
                                         split=Split.TRAIN)
            valid_split = data_set.split(fold=fold,
                                         split=Split.VALID)

            learner_wrapper.fit(train_split)

            # IMPORTANT: these methods return maps of filename to label, since order may (or most certainly will) be
            # different
            predictions = learner_wrapper.predict(valid_split)
            true_labels = valid_split.filename_labels_numeric

            # sort labels and predictions by filename
            predictions = np.array([item[1] for item in sorted(predictions.items(), key=lambda item: item[0])])
            true_labels = np.array([item[1] for item in sorted(true_labels.items(), key=lambda item: item[0])])

            accuracy = accuracy_score(true_labels, predictions)
            uar = uar_score(true_labels, predictions)

            accuracies.append(accuracy)
            uars.append(uar)
            confusion_matrices.append(confusion_matrix(y_true=true_labels,
                                                       y_pred=predictions,
                                                       labels=ordered_labels))

            self.log.info("fold %d accuracy is %2.2f%% (UAR %2.2f%%)", fold + 1, 100 * accuracy, 100 * uar)

        self._accuracy = np.mean(accuracies)
        self._accuracy_confidence_interval = 2 * np.std(accuracies)
        self._uar = np.mean(uars)
        self._uar_confidence_interval = 2 * np.std(uars)
        self._confusion_matrix = np.sum(confusion_matrices, axis=0)
audeep/backend/learners.py
class LearnerBase:
    """
    Defines a common interface for all classification algorithms.
    """
    __metaclass__ = ABCMeta

    _num_features = None

    @property
    def num_features(self):
        """
        Returns the number of features for which the learner was trained.
        
        This property is updated each time the `fit` method is called on some data. Any subsequent calls to the
        `predict` method then expect the same number of features.
        
        Returns
        -------
        int
            The number of features for which the learner was trained
        """
        return self._num_features

    @abstractmethod
    def fit(self,
            data_set: DataSet):
        """
        Fit the learner to the specified data.
        
        Parameters
        ----------
        data_set: DataSet
            The data set on which to train the learner. Must be fully labelled.

        Raises
        ------
        ValueError
            If the feature matrix has invalid shape
        """
        if len(data_set.feature_shape) != 1:
            raise ValueError("invalid number of feature dimensions: {}".format(len(data_set.feature_shape)))

        self._num_features = data_set.feature_shape[0]

    @abstractmethod
    def predict(self,
                data_set: DataSet):
        """
        Predict the labels of the specified data set.
        
        Parameters
        ----------
        data_set: DataSet
            Data set containing instances for which to predict labels.

        Returns
        -------
        numpy.ndarray
            The predicted labels of the specified data
            
        Raises
        ------
        ValueError
            If the specified feature matrix has invalid shape
        """
        if len(data_set.feature_shape) != 1:
            raise ValueError("invalid number of feature dimensions: {}".format(len(data_set.feature_shape)))
        if data_set.feature_shape[0] != self.num_features:
            raise ValueError("learner was trained with different number of features")
audeep/backend/log.py
class LoggingMixin:
    """
    A logging mixin, which adds a log attribute to a class.
    """

    def __init__(self, *args, **kwargs):
        """
        Create and initialize the LoggingMixin.
        
        Any parameters passed to this constructor will be passed to the base class unchanged.
        
        Parameters
        ----------
        args
            Positional arguments
        kwargs
            Keyword arguments
        """
        # noinspection PyArgumentList
        super().__init__(*args, **kwargs)

        self.log = logging.getLogger(self.__class__.__name__)
audeep/backend/data/data_set.py
class Partition(Enum):
    """
    Identifiers for different data partitions.
    """
    TRAIN = 0
    DEVEL = 1
    TEST = 2
audeep/backend/enum_parser.py
class EnumType:
    """
    Performs case-insensitive parsing of string arguments to enum members.
    
    Consider, for example, an enum type with a member called MEMBER. Then, all string values with lower case 
    equivalent "member" will be parsed to the enum member MEMBER.
    """

    def __init__(self,
                 enum_class: Type[Enum]):
        """
        Create and initialize a new EnumParser for the specified Enum type.
        
        Parameters
        ----------
        enum_class: Type[Enum]
            The enum class for which values should be parsed
        """
        self._enum_class = enum_class
        self._member_map = {x.name.lower(): x.name for x in enum_class}

    def __call__(self, arg: str):
        """
        Try to parse the specified string to an enum member.
        
        Parameters
        ----------
        arg: str
            The string value which should be parsed

        Returns
        -------
        enum
            A member of the enum type passed to this class which has the specified name
            
        Raises
        ------
        argparse.ArgumentTypeError
            If the enum does not have a member with the specified name
        """
        arg_lower = arg.lower()

        if not arg_lower in self._member_map:
            raise ArgumentTypeError("invalid choice %s (choose from %s)" % (arg, ", ".join(self._member_map.values())))

        return self._enum_class[self._member_map[arg_lower]]
audeep/backend/evaluation.py
class PartitionedEvaluation(LoggingMixin):
    """
    Partitioned evaluation of a learner on some data set.
    
    The learner is trained on specific partitions of the data set, and evaluated on some other specific partitions of
    the data set. Typically, a the learner is tuned by training on the train partition and evaluating on the development
    partition of a data set. Once tuning is complete, the learner is trained on the train and development partitions, 
    and evaluated on the test partition.
    """

    def __init__(self,
                 learner: LearnerBase,
                 train_partitions: Sequence[Partition],
                 eval_partitions: Sequence[Partition],
                 upsample: bool,
                 majority_vote: bool):
        """
        Create a new partitioned evaluation of the specified learner.
         
        Evaluation is performed by training the learner on the specified training partitions, and evaluating the 
        learner on the specified evaluation partitions.
        
        Parameters
        ----------
        learner: LearnerBase
            The learner which should be evaluated
        train_partitions: list of Partition
            The partitions on which the learner should be trained
        eval_partitions: list of Partition
            The partitions on which the learner should be evaluted
        upsample: bool
            Balance classes in the training partitions by upsampling instances
        """
        super().__init__()

        self._learner = learner
        self._train_partitions = train_partitions
        self._eval_partitions = eval_partitions
        self._upsample = upsample
        self._majority_vote = majority_vote
        self._accuracy = None
        self._uar = None
        self._confusion_matrix = None

    @property
    def accuracy(self) -> float:
        """
        Returns the accuracy of the learner.
        
        This property returns the accuracy on the last data set on which the `run` method has been invoked. As a 
        consequence, this property returns None if the `run` method has not yet been invoked.
        
        Returns
        -------
        float
            Returns the accuracy of the learner on the last data set passed to the `run` method
        """
        return self._accuracy

    @property
    def uar(self) -> float:
        """
        Returns the unweighted average recall of the learner.
        
        This property returns the unweighted average recall on the last data set on which the `run` method has been 
        invoked. As a consequence, this property returns None if the `run` method has not yet been invoked.
        
        Returns
        -------
        float
            Returns the unweighted average recall of the learner on the last data set passed to the `run` method
        """
        return self._uar

    @property
    def confusion_matrix(self) -> np.ndarray:
        """
        Returns the confusion matrix of the learner.
        
        This property returns the confusion matrix on the last data set on which the `run` method has been invoked. 
        As a consequence, this property returns None if the `run` method has not yet been invoked.
        
        Returns
        -------
        numpy.ndarray
            The confusion matrix of the learner on the last data set passed to the `run` method
        """
        return self._confusion_matrix

    def run(self,
            data_set: DataSet):
        """
        Evaluates the learner on the specified data set.
        
        Sets the various properties of this instance to the values obtained during evaluation on the specified data set.
        
        Parameters
        ----------
        data_set: DataSet
            The data set on which the learner should be evaluated

        Raises
        ------
        ValueError 
            If the specified data set does not have partition information
        """
        if not data_set.has_partition_info:
            raise ValueError("data set does not have partition info")

        self.log.info("training classifier")

        learner_wrapper = PreProcessingWrapper(learner=self._learner,
                                               upsample=self._upsample,
                                               majority_vote=self._majority_vote)

        train_split = data_set.partitions(self._train_partitions)
        eval_split = data_set.partitions(self._eval_partitions)

        learner_wrapper.fit(train_split)

        # IMPORTANT: these methods return maps of filename to label, since order may (or most certainly will) be
        # different
        predictions = learner_wrapper.predict(eval_split)
        true_labels = eval_split.filename_labels_numeric

        # sort labels and predictions by filename
        predictions = np.array([item[1] for item in sorted(predictions.items(), key=lambda item: item[0])])
        true_labels = np.array([item[1] for item in sorted(true_labels.items(), key=lambda item: item[0])])

        self._accuracy = accuracy_score(true_labels, predictions)
        self._uar = uar_score(true_labels, predictions)

        # order numeric labels by nominal value
        ordered_labels = sorted(data_set.label_map.items(), key=lambda t: t[0])
        ordered_labels = list(zip(*ordered_labels))[1]

        self._confusion_matrix = confusion_matrix(y_true=true_labels,
                                                  y_pred=predictions,
                                                  labels=ordered_labels)


Question: import abc
import itertools
import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
from cliff.command import Command
from audeep.backend.data.data_set import load, Partition
from audeep.backend.enum_parser import EnumType
from audeep.backend.evaluation import CrossValidatedEvaluation, PartitionedEvaluation
from audeep.backend.formatters import ConfusionMatrixFormatter
from audeep.backend.learners import TensorflowMLPLearner, LearnerBase, LibLINEARLearner
from audeep.backend.log import LoggingMixin
                            action="store_true",
                            help="Balance classes in the training partitions/splits")
        parser.add_argument("--majority-vote",
                            action="store_true",
                            help="Use majority voting to determine the labels of chunked instances")
        parser.add_argument("--repeat",
                            metavar="N",
                            default=1,
                            type=int,
                            help="Repeat evaluation N times and compute the mean accuracy (default 1)")

        return parser

    def plot_confusion_matrix(self, cm, classes,
                              normalize=False,
                              title='Confusion matrix',
                              cmap="Blues"):
        """
        This function plots the confusion matrix.
        Normalization can be applied by setting `normalize=True`.
        """
        cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        plt.imshow(cm_norm, interpolation='nearest', cmap=cmap)
        plt.title(title)
        plt.colorbar()
        tick_marks = np.arange(len(classes))
        plt.xticks(tick_marks, classes, rotation=90)
        plt.yticks(tick_marks, classes)

        if normalize:
            cm = cm_norm

        thresh = cm.max() / 2.
        for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
            if normalize:
                plt.text(j, i, "%1.2f" % cm[i, j],
                         horizontalalignment="center",
                         color="white" if cm[i, j] > thresh else "black")
            else:
                plt.text(j, i, "%.0f" % cm[i, j],
                         horizontalalignment="center",
                         color="white" if cm[i, j] > thresh else "black")

        plt.tight_layout()
        plt.ylabel('True label')
        plt.xlabel('Predicted label')
        plt.show()

    @abc.abstractmethod
    def _get_learner(self, parsed_args) -> LearnerBase:
        pass

    def take_action(self, parsed_args):
        self._learner = self._get_learner(parsed_args)

        if not parsed_args.input.exists():
            raise IOError("failed to open data file at {}".format(parsed_args.input))

        if parsed_args.train_partitions is None and parsed_args.eval_partitions is None and not parsed_args.cross_validate:
            raise ValueError("must select either cross validated or partitioned evaluation")
        if (parsed_args.train_partitions is not None or parsed_args.eval_partitions is not None) \
                and parsed_args.cross_validate:
            raise ValueError("partitioned evaluation and cross validated evaluation are mutually exclusive")
        if not parsed_args.cross_validate and ((parsed_args.train_partitions is not None)
                                                   ^ (parsed_args.eval_partitions is not None)):
            raise ValueError("at least one train and eval partition required")

        data_set = load(parsed_args.input)

        accuracies = []
        uars = []
        confusion_matrices = []

        if parsed_args.cross_validate:
            accuracy_confidence_intervals = []
            uar_confidence_intervals = []

            for _ in range(parsed_args.repeat):
                evaluation = CrossValidatedEvaluation(learner=self._learner,
                                                      upsample=parsed_args.upsample,
                                                      majority_vote=parsed_args.majority_vote)
                evaluation.run(data_set)

                accuracies.append(evaluation.accuracy)
                uars.append(evaluation.uar)
                accuracy_confidence_intervals.append(evaluation.accuracy_confidence_interval)
                uar_confidence_intervals.append(evaluation.uar_confidence_interval)
                confusion_matrices.append(evaluation.confusion_matrix)

            accuracy = np.mean(accuracies)
            accuracy_confidence_interval = np.mean(accuracy_confidence_intervals)
            uar = np.mean(uars)
            uar_confidence_interval = np.mean(uar_confidence_intervals)

            self.log.info("cross validation accuracy: %2.2f%% (+/- %2.2f%%)", 100 * accuracy,
                          100 * accuracy_confidence_interval)
            self.log.info("cross validation UAR: %2.2f%% (+/- %2.2f%%)", 100 * uar, 100 * uar_confidence_interval)
        else:
            for _ in range(parsed_args.repeat):
                # noinspection PyTypeChecker
                evaluation = PartitionedEvaluation(learner=self._learner,
                                                   train_partitions=parsed_args.train_partitions,
                                                   eval_partitions=parsed_args.eval_partitions,
                                                   upsample=parsed_args.upsample,
                                                   majority_vote=parsed_args.majority_vote)
                evaluation.run(data_set)

                accuracies.append(evaluation.accuracy)
                uars.append(evaluation.uar)
                confusion_matrices.append(evaluation.confusion_matrix)

            accuracy = np.mean(accuracies)
            uar = np.mean(uars)

            # noinspection PyTypeChecker,PyStringFormat
            self.log.info("accuracy on %s: %2.2f%% (UAR %2.2f%%)" % (
                " & ".join([p.name for p in parsed_args.eval_partitions]), 100 * accuracy, 100 * uar))

        confusion_matrix = np.sum(confusion_matrices, axis=0)


Answer:         formatter = ConfusionMatrixFormatter()