Repository: Continvvm/continuum (language: python)

Issue:
metric documentation should at least list all the metrics proposed



Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index b6e47617..03295735 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,8 @@
+# Custom
+/tests/Datasets
+/tests/Samples
+/pytest
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
diff --git a/continuum/__init__.py b/continuum/__init__.py
index aa887b82..26e4b618 100644
--- a/continuum/__init__.py
+++ b/continuum/__init__.py
@@ -9,4 +9,6 @@
 # flake8: noqa
 from continuum import datasets
 from continuum.scenarios import *
+from continuum.tasks import *
+from continuum.metrics import *
 from continuum.viz import plot_samples
diff --git a/continuum/metrics/__init__.py b/continuum/metrics/__init__.py
new file mode 100644
index 00000000..4fdb70d1
--- /dev/null
+++ b/continuum/metrics/__init__.py
@@ -0,0 +1,18 @@
+# pylint: disable=C0401
+# flake8: noqa
+from continuum.metrics.logger import Logger
+from continuum.metrics.metrics import (
+    accuracy, accuracy_A, backward_transfer, positive_backward_transfer,
+    remembering, forward_transfer, forgetting
+)
+
+__all__ = [
+    "Logger",
+    "accuracy",
+    "accuracy_A",
+    "backward_transfer",
+    "positive_backward_transfer",
+    "remembering",
+    "forward_transfer",
+    "forgetting"
+]
diff --git a/continuum/metrics/logger.py b/continuum/metrics/logger.py
new file mode 100644
index 00000000..99d96f43
--- /dev/null
+++ b/continuum/metrics/logger.py
@@ -0,0 +1,200 @@
+
+import collections
+import torch
+import numpy as np
+
+from continuum.metrics.metrics import accuracy, \
+    get_model_size_efficiency, \
+    get_model_size, \
+    forgetting, \
+    accuracy_A, \
+    remembering, \
+    positive_backward_transfer, \
+    forward_transfer, \
+    backward_transfer
+
+def require_subset(subset):
+    def wrapper1(func):
+        def wrapper2(self):
+            if subset not in self._predictions:
+                raise Exception(
+                    f"No {subset} predictions have been logged so far which "
+                    f"{func.__name__} rely on!"
+                )
+            return func(self)
+        return wrapper2
+    return wrapper1
+
+
+def cache(func):
+    def wrapper(self):
+        name = f"__cached_{func.__name__}"
+        v = self.__dict__.get(name)
+        if v is None:
+            v = func(self)
+            self.__dict__[name] = v
+        return v
+    return wrapper
+
+
+class Logger:
+    def __init__(self):
+        self._predictions = collections.defaultdict(list)
+        self._targets = collections.defaultdict(list)
+        self._tasks = collections.defaultdict(list)
+        self._model_sizes = []
+
+        self._batch_predictions = []
+        self._batch_targets = []
+
+    def add_batch(self, predictions, targets):
+        if isinstance(predictions, torch.Tensor):
+            predictions = predictions.cpu().numpy()
+        if isinstance(targets, torch.Tensor):
+            targets = targets.cpu().numpy()
+
+        if not isinstance(predictions, np.ndarray):
+            raise TypeError(f"Provide predictions as np.array, not {type(predictions).__name__}.")
+        if not isinstance(targets, np.ndarray):
+            raise TypeError(f"Provide targets as np.array, not {type(predictions).__name__}.")
+
+        self._batch_predictions.append(predictions)
+        self._batch_targets.append(targets)
+
+    def add_step(self, predictions=None, targets=None, task_ids=None, subset="test", model=None):
+        if subset not in ("train", "test"):
+            raise ValueError(f"Subset must be train, val, or test, not {subset}.")
+
+        if isinstance(predictions, torch.Tensor):
+            predictions = predictions.cpu().numpy()
+        if isinstance(targets, torch.Tensor):
+            targets = targets.cpu().numpy()
+        if isinstance(task_ids, torch.Tensor):
+            task_ids = task_ids.cpu().numpy()
+
+        if predictions is not None and targets is not None and task_ids is not None:
+            self._predictions[subset].append(predictions)
+            self._targets[subset].append(targets)
+            self._tasks[subset].append(task_ids)
+
+        if model is not None:
+            self._model_sizes.append(get_model_size(model))
+
+        # Remove all cached properties
+        for k in list(self.__dict__.keys()):
+            if k.startswith("__cached_"):
+                del self.__dict__[k]
+
+        self._batch_predictions = []
+        self._batch_targets = []
+
+    def log(self):
+        print(f"Task id={self.nb_tasks}, acc={self.accuracy}, avg-acc={self.average_incremental_accuracy}")
+
+    @property
+    def nb_tasks(self):
+        return len(self._predictions[list(self._predictions.keys())[0]])
+
+    @property
+    def online_accuracy(self):
+        if len(self._batch_predictions) == 0:
+            raise Exception(
+                "You need to call <add_batch(preds, targets)> in order to get the online accuracy."
+            )
+
+        if len(self._batch_predictions) > 1:
+            p, t = np.concatenate(self._batch_predictions), np.concatenate(self._batch_targets)
+        else:
+            p, t = self._batch_predictions[0], self._batch_targets[0]
+
+        return accuracy(p, t)
+
+    @property
+    @cache
+    @require_subset("test")
+    def accuracy(self):
+        return accuracy(
+            self._predictions["test"][-1],
+            self._targets["test"][-1]
+        )
+
+    @property
+    @cache
+    @require_subset("test")
+    def accuracy_per_task(self):
+        """Returns all task accuracy individually."""
+        return [
+            _get_R_ij(-1, j, all_preds, all_targets, all_tasks)
+            for j in range(self.nb_tasks)
+        ]
+
+    @property
+    @cache
+    @require_subset("train")
+    def online_cumulative_performance(self):
+        """Computes the accuracy of last task on the train set.
+
+        Reference:
+        * Online Fast Adaptation and Knowledge Accumulation: a New Approach to Continual Learning
+          Caccia et al. NeurIPS 2020
+        """
+        return accuracy(
+            self._predictions["train"][-1],
+            self._targets["train"][-1]
+        )
+
+    @property
+    @cache
+    @require_subset("test")
+    def average_incremental_accuracy(self):
+        """Computes the average of the accuracies computed after each task.
+
+        Reference:
+        * iCaRL: Incremental Classifier and Representation Learning
+          Rebuffi et al. CVPR 2017
+        """
+        return statistics.mean([
+            accuracy(self._predictions["test"][t], self._targets["test"][t])
+            for t in range(len(self._predictions["test"]))
+        ])
+
+    @property
+    @cache
+    @require_subset("test")
+    def backward_transfer(self):
+        return backward_transfer(self._predictions["test"], self._targets["test"], self._tasks["test"])
+
+    @property
+    @cache
+    @require_subset("test")
+    def forward_transfer(self):
+        return forward_transfer(self._predictions["test"], self._targets["test"], self._tasks["test"])
+
+    @property
+    @cache
+    @require_subset("test")
+    def positive_backward_transfer(self):
+        return positive_backward_transfer(self._predictions["test"], self._targets["test"], self._tasks["test"])
+
+    @property
+    @cache
+    @require_subset("test")
+    def remembering(self):
+        return remembering(self._predictions["test"], self._targets["test"], self._tasks["test"])
+
+    @property
+    @cache
+    @require_subset("test")
+    def accuracy_A(self):
+        return accuracy_A(self._predictions["test"], self._targets["test"], self._tasks["test"])
+
+    @property
+    @cache
+    @require_subset("test")
+    def forgetting(self):
+        return forgetting(self._predictions["test"], self._targets["test"], self._tasks["test"])
+
+    @property
+    @cache
+    def model_size_efficiency(self):
+        return get_model_size_efficiency(self._model_sizes)
diff --git a/continuum/metrics/metrics.py b/continuum/metrics/metrics.py
new file mode 100644
index 00000000..74d81092
--- /dev/null
+++ b/continuum/metrics/metrics.py
@@ -0,0 +1,239 @@
+from functools import reduce
+
+
+def accuracy(task_preds, task_targets):
+    """Computes the accuracy of a given task.
+
+    :param task_preds: Predicted labels.
+    :param task_targets: Ground-truth targets.
+    :return: a float metric between 0 and 1.
+    """
+    metric = (task_preds == task_targets).mean()
+    assert 0. <= metric <= 1.0
+    return metric
+
+
+def accuracy_A(all_preds, all_targets, all_tasks):
+    """Accuracy as defined in Diaz-Rodriguez and Lomonaco.
+
+    Note that it is slightly different from the normal accuracy as it considers
+    each task accuracy with equal weight, while the normal accuracy considers
+    the proportion of all targets.
+
+    Example:
+    - Given task 1 with 50,000 images and task 2 with 1,000 images.
+    - With normal accuracy, task 1 has more importance in the average accuracy.
+    - With this accuracy A, task 1 has as much importance as task 2.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+    :param all_preds: All predicted labels up to now.
+    :param all_targets: All targets up to now.
+    :param all_tasks: All task ids up to now.
+    :return: a float metric between 0 and 1.
+    """
+    T = len(all_preds)  # Number of seen tasks so far
+    # TODO if we take in account zeroshot, we should take the max of all_tasks?
+    A = 0.
+
+    for i in range(T):
+        for j in range(i+1):
+            A += _get_R_ij(i, j, all_preds, all_targets, all_tasks)
+
+    metric = A / (T * (T + 1) / 2)
+    assert 0. <= metric <= 1.0
+    return metric
+
+
+def backward_transfer(all_preds, all_targets, all_tasks):
+    """Measures the influence that learning a task has on the performance on previous tasks.
+
+    Reference:
+    * Gradient Episodic Memory for Continual Learning
+      Lopez-paz & ranzato, NeurIPS 2017
+
+    :param all_preds: All predicted labels up to now.
+    :param all_targets: All targets up to now.
+    :param all_tasks: All task ids up to now.
+    :return: a float metric between 0 and 1.
+    """
+    T = len(all_preds)  # Number of seen tasks so far
+    # TODO if we take in account zeroshot, we should take the max of all_tasks?
+    if T <= 1:
+        return 0.
+    bwt = 0.
+
+    for i in range(1, T):
+        for j in range(0, i - 1):
+            r_ij = _get_R_ij(i, j, all_preds, all_targets, all_tasks)
+            r_jj = _get_R_ij(j, j, all_preds, all_targets, all_tasks)
+
+            bwt += (r_ij - r_jj)
+
+    metric = bwt / (T * (T - 1) / 2)
+    assert -1. <= metric <= 1.0
+    return metric
+
+
+def positive_backward_transfer(all_preds, all_targets, all_tasks):
+    """Computes the the positive gain of Backward transfer.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+    :param all_preds: All predicted labels up to now.
+    :param all_targets: All targets up to now.
+    :param all_tasks: All task ids up to now.
+    :return: a float metric between 0 and 1.
+    """
+    bwt = backward_transfer(all_preds, all_targets, all_tasks)
+    metric = 1 - abs(min(bwt, 0.))
+    assert 0. <= metric <= 1.0
+    return metric
+
+
+def remembering(all_preds, all_targets, all_tasks):
+    """Computes the forgetting part of Backward transfer.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+    :param all_preds: All predicted labels up to now.
+    :param all_targets: All targets up to now.
+    :param all_tasks: All task ids up to now.
+    :return: a float metric between 0 and 1.
+    """
+    bwt = backward_transfer(all_preds, all_targets, all_tasks)
+    metric = max(bwt, 0.)
+    assert 0. <= metric <= 1.0
+    return metric
+
+
+def forward_transfer(all_preds, all_targets, all_tasks):
+    """Measures the influence that learning a task has on the performance of future tasks.
+
+    Reference:
+    * Gradient Episodic Memory for Continual Learning
+      Lopez-paz & ranzato, NeurIPS 2017
+
+    :param all_preds: All predicted labels up to now.
+    :param all_targets: All targets up to now.
+    :param all_tasks: All task ids up to now.
+    :return: a float metric between 0 and 1.
+    """
+    T = len(all_preds)  # Number of seen tasks so far
+    # TODO if we take in account zeroshot, we should take the max of all_tasks?
+    if T <= 1:
+        return 0.
+
+    fwt = 0.
+    for i in range(T):
+        for j in range(i):
+            fwt += _get_R_ij(i, j, all_preds, all_targets, all_tasks)
+
+    metric = fwt / (T * (T - 1) / 2)
+    assert -1. <= metric <= 1.0
+    return metric
+
+
+
+def forgetting(all_preds, all_targets, all_tasks):
+    """Measures the average forgetting.
+
+    Reference:
+    * Riemannian Walk for Incremental Learning: Understanding Forgetting and Intransigence
+      Chaudhry et al. ECCV 2018
+    """
+    T = len(all_preds)  # Number of seen tasks so far
+    # TODO if we take in account zeroshot, we should take the max of all_tasks?
+    if T <= 1:
+        return 0.
+
+    f = 0.
+    for j in range(T - 1):
+        r_kj = _get_R_ij(T - 1, j, all_preds, all_targets, all_tasks)
+        r_lj = max(_get_R_ij(l, j, all_preds, all_targets, all_tasks) for l in range(T - 1))
+        f += (r_lj - r_kj)
+
+    metric = f / (T - 1)
+    assert 0. <= metric <= 1.0
+    return metric
+
+
+def _get_R_ij(i, j, all_preds, all_targets, all_tasks):
+    """Computes an accuracy after task i on task j.
+
+    R matrix:
+
+          || T_e1 | T_e2 | T_e3
+    ============================|
+     T_r1 || R*  | R_ij  | R_ij |
+    ----------------------------|
+     T_r1 || R*  | R_ij  | R_ij |
+    ----------------------------|
+     T_r1 || R*  | R_ij  | R_ij |
+    ============================|
+
+    R_13 is the R of the first column and the third row.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+    :param i: Task id after which a model was trained.
+    :param j: Task id of the test data.
+    :param all_preds: All predicted labels up to now.
+    :param all_targets: All targets up to now.
+    :param all_tasks: All task ids up to now.
+    :return: a float metric between 0 and 1.
+    """
+    preds = all_preds[i]
+    targets = all_targets[i]
+    tasks = all_tasks[i]
+
+    indexes = tasks == j
+    return (preds[indexes] == targets[indexes]).mean()
+
+
+def get_model_size(model):
+    """Computes the total number of parameters.
+
+    :param model: A Pytorch's nn.Module model.
+    :return: The number of parameters.
+    """
+    nb_params = 0
+
+    for w in model.parameters():
+        if len(w.shape) > 0:  # Tensor
+            nb_params += reduce(lambda a, b: a * b, w.shape)
+        else:  # Scalar
+            nb_params += 1
+
+    return nb_params
+
+
+def get_model_size_efficiency(model_sizes):
+    """Computes the efficiency of the model sizes.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+    :param model_sizes: A list of number of parameters, each being computed after a task.
+    :return: a float metric between 0 and 1.
+    """
+    T = len(model_sizes)
+    if T <= 1:
+        return 1.0
+
+    ms = 0.
+    for i in range(T):
+        ms += (model_sizes[0] / model_sizes[i])
+
+    metric = min(1., ms / T)
+    assert 0. <= metric <= 1.0
+    return metric
diff --git a/docs/_tutorials/metrics/metrics.rst b/docs/_tutorials/metrics/metrics.rst
new file mode 100644
index 00000000..b02fcbc0
--- /dev/null
+++ b/docs/_tutorials/metrics/metrics.rst
@@ -0,0 +1,173 @@
+Metrics
+-------
+
+Continual Learning has many different metrics due to the nature of the task.
+Continuum proposes a logger module that accumulates the predictions of the model.
+Then the logger can compute various type of continual learning metrics based on the prediction saved.
+
+Pseudo-code
+
+.. code-block:: python
+
+    for task in scenario:
+        for (x,y,t) in tasks:
+            predictions = model(x,y,t)
+
+            logger.add_batch(predictions, y)
+
+        print(f"Metric result: {logger.my_pretty_metric}")
+
+
+
+Here is a list of all implemented metrics:
+
++-------------------------------+-----------------------------+-------+
+|Name                           | Code                        | ↑ / ↓ |
++===============================+=============================+=======+
+| **Accuracy**                  | `accuracy`                  |   ↑   |
++-------------------------------+-----------------------------+-------+
+| **Accuracy A**                | `accuracy_A`                |   ↑   |
++-------------------------------+-----------------------------+-------+
+| **Backward Transfer**         | `backward_transfer`         |   ↑   |
++-------------------------------+-----------------------------+-------+
+| **Positive Backward Transfer**| `positive_backward_transfer`|   ↑   |
++-------------------------------+-----------------------------+-------+
+| **Remembering**               | `remembering`               |   ↑   |
++-------------------------------+-----------------------------+-------+
+| **Forward Transfer**          | `forward_transfer`          |   ↑   |
++-------------------------------+-----------------------------+-------+
+| **Forgetting**                | `forgetting`                |   ↓   |
++-------------------------------+-----------------------------+-------+
+| **Model Size Efficiency**     | `model_size_efficiency`     |   ↓   |
++-------------------------------+-----------------------------+-------+
+
+**Accuracy**::
+
+    Computes the accuracy of a given task.
+
+
+**Accuracy A**::
+
+    Accuracy as defined in Diaz-Rodriguez and Lomonaco.
+
+    Note that it is slightly different from the normal accuracy as it considers
+    each task accuracy with equal weight, while the normal accuracy considers
+    the proportion of all targets.
+
+    Example:
+    - Given task 1 with 50,000 images and task 2 with 1,000 images.
+    - With normal accuracy, task 1 has more importance in the average accuracy.
+    - With this accuracy A, task 1 has as much importance as task 2.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+
+**Backward Transfer**::
+
+    Measures the influence that learning a task has on the performance on previous tasks.
+
+    Reference:
+    * Gradient Episodic Memory for Continual Learning
+      Lopez-paz & ranzato, NeurIPS 2017
+
+
+**Positive Backward Transfer**::
+
+    Computes the the positive gain of Backward transfer.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+
+**Remembering**::
+
+    Computes the forgetting part of Backward transfer.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+
+**Forward Transfer**::
+
+    Measures the influence that learning a task has on the performance of future tasks.
+
+    Reference:
+    * Gradient Episodic Memory for Continual Learning
+      Lopez-paz & ranzato, NeurIPS 2017
+
+
+**Forgetting**::
+
+    Measures the average forgetting.
+
+    Reference:
+    * Riemannian Walk for Incremental Learning: Understanding Forgetting and Intransigence
+      Chaudhry et al. ECCV 2018
+
+
+**Model Size Efficiency**::
+
+    Computes the efficiency of the model sizes.
+
+    Reference:
+    * Don’t forget, there is more than forgetting: newmetrics for Continual Learning
+      Diaz-Rodriguez and Lomonaco et al. NeurIPS Workshop 2018
+
+
+
+Detailed Example
+----------------
+
+.. code-block:: python
+
+	from torch.utils.data import DataLoader
+    import numpy as np
+
+    from continuum import ClassIncremental
+    from continuum.datasets import MNIST
+    from continuum.metrics import Logger
+
+    train_scenario = ClassIncremental(
+        MNIST(data_path="my/data/path", download=True, train=True),
+        increment=2
+     )
+    test_scenario = ClassIncremental(
+        MNIST(data_path="my/data/path", download=True, train=False),
+        increment=2
+     )
+
+    model = ... Initialize your model here ...
+
+    logger = Logger()
+
+    for task_id, (train_taskset, test_taskset) in enumerate(zip(train_scenario, test_scenario)):
+        train_loader = DataLoader(train_taskset)
+        test_loader = DataLoader(test_taskset)
+
+        for x, y, t in train_loader:
+            predictions = model(x)
+
+            # Do here your model training with losses and optimizer...
+
+            logger.add_batch(predictions, y)
+            print(f"Online accuracy: {logger.only_accuracy}")
+
+        preds, targets, task_ids = [], [], []
+        for x, y, t in test_loader:
+            preds.append(model(x).cpu().numpy())
+            targets.append(y.cpu().numpy())
+            task_ids.append(t.cpu().numpy())
+
+        logger.add_step(
+            np.concatenate(preds),
+            np.concatenate(targets),
+            np.concatenate(task_ids),
+            model
+        )
+        print(f"Task: {task_id}, acc: {logger.accuracy}, avg acc: {logger.average_incremental_accuracy}")
+        print(f"BWT: {logger.backward_transfer}, FWT: {logger.forward_transfer}")
+
diff --git a/docs/_tutorials/scenarios/scenarios.rst b/docs/_tutorials/scenarios/scenarios.rst
index a9b6711e..da8efb56 100644
--- a/docs/_tutorials/scenarios/scenarios.rst
+++ b/docs/_tutorials/scenarios/scenarios.rst
@@ -1,5 +1,5 @@
 Continuum Scenarios
------------------
+--------------------
 
 Continual learning (CL) aim is to learn without forgetting in the most "satisfying" way. To evaluate the capacity of different CL algorithms the community use different type of benchmarks scenarios.
 
@@ -175,6 +175,22 @@ with 10 different domains. Each domain represents a new task.
     scenario = InstanceIncremental(dataset=dataset)
 
 
+Another example could be using different dataset with their original classes as for MNISTFellowship:
+
+.. code-block:: python
+
+    from continuum import InstanceIncremental
+    from continuum.datasets import MNISTFellowship
+
+    dataset = MNISTFellowship("/my/path/where/to/download")
+    scenario = InstanceIncremental(dataset=dataset)
+
+In this case, the three dataset MNIST, Fashion-MNIST and KMNIST will be learn with their original labels, e.g. classes 0 of all dataset stay 0.
+
+
+Instance incremental scenarios can also be create with transformation as described in next section.
+
+
 Transformed Incremental
 -----------------------
 
diff --git a/docs/conf.py b/docs/conf.py
index 3cb33ced..4943d139 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -159,7 +159,7 @@
 texinfo_documents = [
     (
         master_doc, 'Continuum', 'Continuum Documentation', author, 'Continuum',
-        'One line description of project.', 'Miscellaneous'
+        'Data Loaders & Metrics for Continual Learning', 'Research'
     ),
 ]
 
diff --git a/docs/index.rst b/docs/index.rst
index 793d48b0..35fccd4c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -22,6 +22,13 @@ The latter is about the different setting you may encounter in Continual Learnin
 
    _tutorials/datasets/*
 
+.. toctree::
+   :glob:
+   :maxdepth: 2
+   :caption: Metrics:
+
+   _tutorials/metrics/*
+
 
 Quick Example
 -------------
diff --git a/setup.py b/setup.py
index 73714fa7..43f0e671 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="continuum",
-    version="1.0.3",
+    version="1.0.5",
     author="Arthur Douillard, Timothée Lesort",
     author_email="ar.douillard@gmail.com",
     description="A clean and simple library for Continual Learning in PyTorch.",


Test patch:
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
new file mode 100644
index 00000000..885b153b
--- /dev/null
+++ b/tests/test_metrics.py
@@ -0,0 +1,216 @@
+import numpy as np
+import pytest
+from torch import nn
+import torch
+
+from continuum.metrics import Logger
+
+# yapf: disable
+
+@pytest.fixture
+def numpy_data():
+    nb_classes = 20
+    nb_tasks = 5
+    nb_data = 100
+    seen_classes = 0
+    inc_classes = nb_classes // nb_tasks
+
+    targets = []
+    tasks = []
+
+    for t in range(nb_tasks):
+        seen_classes += inc_classes
+
+        task_targets = np.concatenate([
+            np.ones(nb_data) * c
+            for c in range(seen_classes)
+        ])
+        task_tasks = np.concatenate([
+            np.ones(nb_data * inc_classes) * tt
+            for tt in range(t + 1)
+        ])  # We also see previous data
+
+        targets.append(task_targets)
+        tasks.append(task_tasks)
+
+    return targets, tasks
+
+
+@pytest.fixture
+def torch_models():
+    class Small(nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.conv = nn.Sequential(nn.Conv2d(2, 3, 1, bias=False), nn.Conv2d(2, 3, 1, bias=False))
+            self.fc = nn.Linear(5, 4)
+            self.scalar = nn.Parameter(torch.tensor(3.))
+
+    class Big(nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.conv = nn.Sequential(nn.Conv2d(2, 3, 1, bias=False), nn.Conv2d(2, 3, 1, bias=False))
+            self.fc = nn.Linear(5, 4)
+            self.fc2 = nn.Linear(5, 4)
+            self.scalar = nn.Parameter(torch.tensor(3.))
+
+    return Small(), Big()
+
+
+@pytest.mark.parametrize("mode,expected", [
+    ("best", 1.), ("worst", 0.), ("random", None)
+])
+def test_metrics(numpy_data, mode, expected):
+    logger = Logger()
+    all_targets, all_tasks = numpy_data
+
+    for targets, task_ids in zip(all_targets, all_tasks):
+        if mode == "best":
+            preds = np.copy(targets)
+        elif mode == "worst":
+            # Trick to never generate the correct predictions
+            # only work for more three classes or more
+            preds = (np.copy(targets) + 1) % np.max(targets)
+        else:
+            preds = np.random.randint(0, np.max(targets) + 1, targets.shape)
+
+        logger.add_step(preds, targets, task_ids, subset="train")
+        logger.add_step(preds, targets, task_ids, subset="test")
+
+        accuracies = [
+            logger.accuracy, logger.online_cumulative_performance,
+            logger.accuracy_A,
+        ]
+        for acc in accuracies:
+            if expected is not None:
+                assert acc == expected, (acc, mode)
+            assert 0. <= acc <= 1.
+        assert -1. <= logger.backward_transfer <= 1.0
+        assert -1. <= logger.forward_transfer <= 1.0
+        assert 0. <= logger.positive_backward_transfer <= 1.0
+        assert 0. <= logger.remembering <= 1.0
+        assert 0. <= logger.forgetting <= 1.0
+
+    assert 0. <= logger.average_incremental_accuracy <= 1.
+
+
+@pytest.mark.parametrize("batch_size", [
+    1, 32, None
+])
+def test_online_accuracy(numpy_data, batch_size):
+    logger = Logger()
+    all_targets, _ = numpy_data
+    targets = all_targets[0]
+
+    check_raised(lambda: logger.online_accuracy)
+
+    batch_size = batch_size or len(targets)
+    for batch_index in range(0, len(targets), batch_size):
+        y = targets[batch_index: batch_index+batch_size]
+        x = np.copy(y)
+
+        logger.add_batch(x, y)
+        logger.online_accuracy
+    logger.online_accuracy
+
+    logger.add_step(targets, np.copy(targets))
+    check_raised(lambda: logger.online_accuracy)
+
+
+def test_require_subset_test(numpy_data):
+    logger = Logger()
+    check_raised(lambda: logger.accuracy)
+
+    logger.add_step(numpy_data[0][0], numpy_data[0][0], numpy_data[0][1], subset="test")
+    logger.accuracy
+
+
+def test_require_subset_train(numpy_data):
+    logger = Logger()
+    check_raised(lambda: logger.online_cumulative_performance)
+
+    logger.add_step(numpy_data[0][0], numpy_data[0][0], numpy_data[0][1], subset="train")
+    logger.online_cumulative_performance
+
+
+def test_model_efficiency(torch_models):
+    small, big = torch_models
+
+    logger1 = Logger()
+    logger1.add_step(model=small)
+    logger1.add_step(model=small)
+    ms1 = logger1.model_size_efficiency
+
+    logger2 = Logger()
+    logger2.add_step(model=small)
+    logger2.add_step(model=big)
+    ms2 = logger2.model_size_efficiency
+
+    logger3 = Logger()
+    logger3.add_step(model=big)
+    logger3.add_step(model=small)
+    ms3 = logger3.model_size_efficiency
+
+    logger4 = Logger()
+    logger4.add_step(model=big)
+    logger4.add_step(model=big)
+    ms4 = logger4.model_size_efficiency
+
+    assert ms1 == ms4 == ms3 == 1.0
+    assert 0. <= ms2 < 1.
+
+
+@pytest.mark.slow
+def test_example_doc():
+    from torch.utils.data import DataLoader
+    import numpy as np
+
+    from continuum import Logger, ClassIncremental
+    from continuum.datasets import MNIST
+
+    train_scenario = ClassIncremental(
+        MNIST(data_path="/tmp", download=True, train=True),
+        increment=2
+     )
+    test_scenario = ClassIncremental(
+        MNIST(data_path="/tmp", download=True, train=False),
+        increment=2
+     )
+
+    logger = Logger()
+
+    for task_id, (train_taskset, test_taskset) in enumerate(zip(train_scenario, test_scenario)):
+        train_loader = DataLoader(train_taskset)
+        test_loader = DataLoader(test_taskset)
+
+        for x, y, t in train_loader:
+            predictions = torch.clone(y)
+
+            logger.add_batch(predictions, y)
+            _ = (f"Online accuracy: {logger.online_accuracy}")
+
+        preds, targets, task_ids = [], [], []
+        for x, y, t in test_loader:
+            preds.append(y.cpu().numpy())
+            targets.append(y.cpu().numpy())
+            task_ids.append(t.cpu().numpy())
+
+        logger.add_step(
+            np.concatenate(preds),
+            np.concatenate(targets),
+            np.concatenate(task_ids)
+        )
+        _ = (f"Task: {task_id}, acc: {logger.accuracy}, avg acc: {logger.average_incremental_accuracy}")
+        _ = (f"BWT: {logger.backward_transfer}, FWT: {logger.forward_transfer}")
+
+
+
+def check_raised(func):
+    has_raised = False
+    try:
+        func()
+    except:
+        has_raised = True
+    finally:
+        assert has_raised
+
+