Repository: pytorch/ignite

Issue:
Add ignite.metrics to add common metrics with the interface (y, y_pred)
See #37 for details


Discussion:
@alykhantejani I'm working on a PR for this now

Resolving patch:
diff --git a/ignite/metrics.py b/ignite/metrics.py
new file mode 100644
--- /dev/null
+++ b/ignite/metrics.py
@@ -0,0 +1,96 @@
+import torch
+
+
+def _convert_history(history, transform):
+    y_preds, ys = zip(*map(transform, history))
+    y_pred = torch.cat(y_preds, dim=0)
+    y = torch.cat(ys, dim=0)
+    return y_pred, y
+
+
+def binary_accuracy(history, transform=lambda x: x):
+    """
+    Calculates the binary accuracy
+
+    Args:
+        history (History): the history used to compute the metric
+        transform (callable, optional): takes a history item and converts it into the form
+            (y_pred, y) (default: identity)
+
+    Returns:
+        float: the accuracy over the entire history
+    """
+    y_pred, y = _convert_history(history, transform)
+    correct = torch.eq(torch.round(y_pred).type(torch.LongTensor), y)
+    return torch.mean(correct.type(torch.FloatTensor))
+
+
+def categorical_accuracy(history, transform=lambda x: x):
+    """
+    Calculates the categorical accuracy
+
+    Args:
+        history (History): the history used to compute the metric
+        transform (callable, optional): takes a history item and converts it into the form
+            (y_pred, y) (default: identity)
+
+    Returns:
+        float: the accuracy over the entire history
+    """
+    y_pred, y = _convert_history(history, transform)
+    indices = torch.max(y_pred, 1)[1]
+    correct = torch.eq(indices, y)
+    return torch.mean(correct.type(torch.FloatTensor))
+
+
+def top_k_categorical_accuracy(history, k=5, transform=lambda x: x):
+    """
+    Calculates the top-k categorical accuracy
+
+    Args:
+        history (History): the history used to compute the metric
+        transform (callable, optional): takes a history item and converts it into the form
+            (y_pred, y) (default: identity)
+
+    Returns:
+        float: the accuracy over the entire history
+    """
+    y_pred, y = _convert_history(history, transform)
+    sorted_indices = torch.topk(y_pred, k, dim=1)[1]
+    expanded_y = y.view(-1, 1).expand(-1, k)
+    correct = torch.sum(torch.eq(sorted_indices, expanded_y), dim=1)
+    return torch.mean(correct.type(torch.FloatTensor))
+
+
+def mean_squared_error(history, transform=lambda x: x):
+    """
+    Calculates the mean squared error
+
+    Args:
+        history (History): the history used to compute the metric
+        transform (callable, optional): takes a history item and converts it into the form
+            (y_pred, y) (default: identity)
+
+    Returns:
+        float: the error over the entire history
+    """
+    y_pred, y = _convert_history(history, transform)
+    squared_error = torch.pow(y_pred - y.view_as(y_pred), 2)
+    return torch.mean(squared_error)
+
+
+def mean_absolute_error(history, transform=lambda x: x):
+    """
+    Calculates the mean absolute error
+
+    Args:
+        history (History): the history used to compute the metric
+        transform (callable, optional): takes a history item and converts it into the form
+            (y_pred, y) (default: identity)
+
+    Returns:
+        float: the error over the entire history
+    """
+    y_pred, y = _convert_history(history, transform)
+    absolute_error = torch.abs(y_pred - y.view_as(y_pred))
+    return torch.mean(absolute_error)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@ def find_version(*file_paths):
 
 VERSION = find_version('ignite', '__init__.py')
 
-requirements = ['enum34']
+requirements = ['enum34', 'torch']
 
 setup(
     # Metadata


Test patch:
diff --git a/tests/ignite/test_metrics.py b/tests/ignite/test_metrics.py
new file mode 100644
--- /dev/null
+++ b/tests/ignite/test_metrics.py
@@ -0,0 +1,111 @@
+import torch
+from pytest import approx
+
+from ignite.trainer.history import History
+from ignite.metrics import (
+    binary_accuracy,
+    categorical_accuracy,
+    top_k_categorical_accuracy,
+    mean_squared_error,
+    mean_absolute_error
+)
+
+
+def test_binary_accuracy():
+    history = History()
+    history.append((torch.FloatTensor([[0.8], [0.6]]), torch.LongTensor([1, 0])))
+    history.append((torch.FloatTensor([[0.4], [0.2]]), torch.LongTensor([1, 0])))
+    result = binary_accuracy(history)
+    assert result == approx(0.5)
+
+    history = History()
+    history.append({'y_pred': torch.FloatTensor([[0.8], [0.6]]), 'y': torch.LongTensor([1, 0])})
+    history.append({'y_pred': torch.FloatTensor([[0.4], [0.2]]), 'y': torch.LongTensor([1, 0])})
+    result = binary_accuracy(history, transform=lambda x: (x['y_pred'], x['y']))
+
+    assert result == approx(0.5)
+
+
+def test_categorical_accuracy():
+    history = History()
+    history.append((torch.FloatTensor([[2, 1], [4, 3]]), torch.LongTensor([1, 0])))
+    history.append((torch.FloatTensor([[6, 5], [8, 7]]), torch.LongTensor([1, 0])))
+    result = categorical_accuracy(history)
+    assert result == approx(0.5)
+
+    history = History()
+    history.append({'y_pred': torch.FloatTensor([[2, 1], [4, 3]]), 'y': torch.LongTensor([1, 0])})
+    history.append({'y_pred': torch.FloatTensor([[6, 5], [8, 7]]), 'y': torch.LongTensor([1, 0])})
+    result = categorical_accuracy(history, transform=lambda x: (x['y_pred'], x['y']))
+    assert result == approx(0.5)
+
+
+def test_top_k_categorical_accuracy():
+    history = History()
+    history.append((torch.FloatTensor([[3, 2, 1], [6, 5, 4]]), torch.LongTensor([1, 0])))
+    history.append((torch.FloatTensor([[9, 8, 7], [12, 11, 10]]), torch.LongTensor([1, 0])))
+    top_2_result = top_k_categorical_accuracy(history, k=2)
+    top_1_result = top_k_categorical_accuracy(history, k=1)
+    assert top_2_result == approx(1.0)
+    assert top_1_result == approx(0.5)
+    assert top_1_result == approx(categorical_accuracy(history))
+
+    history = History()
+    history.append({
+        'y_pred': torch.FloatTensor([[3, 2, 1], [6, 5, 4]]),
+        'y': torch.LongTensor([1, 0])
+    })
+    history.append({
+        'y_pred': torch.FloatTensor([[9, 8, 7], [12, 11, 10]]),
+        'y': torch.LongTensor([1, 0])
+    })
+    top_2_result = top_k_categorical_accuracy(history,
+                                              k=2,
+                                              transform=lambda x: (x['y_pred'], x['y']))
+    top_1_result = top_k_categorical_accuracy(history,
+                                              k=1,
+                                              transform=lambda x: (x['y_pred'], x['y']))
+    assert top_2_result == approx(1.0)
+    assert top_1_result == approx(0.5)
+    assert top_1_result == approx(categorical_accuracy(history,
+                                                       transform=lambda x: (x['y_pred'], x['y'])))
+
+
+def test_mean_squared_error():
+    history = History()
+    history.append((torch.FloatTensor([[4.5], [4.0]]), torch.FloatTensor([5.0, 3.5])))
+    history.append((torch.FloatTensor([[3.5], [3.0]]), torch.FloatTensor([3.0, 3.5])))
+    result = mean_squared_error(history)
+    assert result == approx(0.25)
+
+    history = History()
+    history.append({
+        'y_pred': torch.FloatTensor([[4.5], [4.0]]),
+        'y': torch.FloatTensor([5.0, 3.5])
+    })
+    history.append({
+        'y_pred': torch.FloatTensor([[3.5], [3.0]]),
+        'y': torch.FloatTensor([3.0, 3.5])
+    })
+    result = mean_squared_error(history, transform=lambda x: (x['y_pred'], x['y']))
+    assert result == approx(0.25)
+
+
+def test_mean_absolute_error():
+    history = History()
+    history.append((torch.FloatTensor([[4.5], [4.0]]), torch.FloatTensor([5.0, 3.5])))
+    history.append((torch.FloatTensor([[3.5], [3.0]]), torch.FloatTensor([3.0, 3.5])))
+    result = mean_absolute_error(history)
+    assert result == approx(0.5)
+
+    history = History()
+    history.append({
+        'y_pred': torch.FloatTensor([[4.5], [4.0]]),
+        'y': torch.FloatTensor([5.0, 3.5])
+    })
+    history.append({
+        'y_pred': torch.FloatTensor([[3.5], [3.0]]),
+        'y': torch.FloatTensor([3.0, 3.5])
+    })
+    result = mean_absolute_error(history, transform=lambda x: (x['y_pred'], x['y']))
+    assert result == approx(0.5)