Repository: mars-project/mars

Issue:
Implements `mars.learn.metrics.log_loss`
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Is your feature request related to a problem? Please describe.**

`mars.learn.metrics.log_loss` can be added.


Discussion:

Resolving patch:
diff --git a/mars/learn/metrics/__init__.py b/mars/learn/metrics/__init__.py
--- a/mars/learn/metrics/__init__.py
+++ b/mars/learn/metrics/__init__.py
@@ -13,6 +13,6 @@
 # limitations under the License.
 
 from .pairwise import euclidean_distances, pairwise_distances, pairwise_distances_topk
-from ._classification import accuracy_score
+from ._classification import accuracy_score, log_loss
 from ._ranking import roc_curve, auc
 from ._regresssion import r2_score
diff --git a/mars/learn/metrics/_classification.py b/mars/learn/metrics/_classification.py
--- a/mars/learn/metrics/_classification.py
+++ b/mars/learn/metrics/_classification.py
@@ -21,6 +21,8 @@
 from ...serialization.serializables import AnyField, BoolField, KeyField
 from ...tensor.core import TensorOrder
 from ..operands import LearnOperand, LearnOperandMixin, OutputType
+from ..preprocessing import LabelBinarizer
+from ..utils import check_array, check_consistent_length
 from ._check_targets import _check_targets
 
 
@@ -178,3 +180,129 @@ def accuracy_score(y_true, y_pred, normalize=True, sample_weight=None,
                        sample_weight=sample_weight)
     score = op(y_true, y_pred)
     return score.execute(session=session, **(run_kwargs or dict()))
+
+
+def log_loss(y_true, y_pred, *, eps=1e-15, normalize=True, sample_weight=None,
+             labels=None):
+    r"""Log loss, aka logistic loss or cross-entropy loss.
+
+    This is the loss function used in (multinomial) logistic regression
+    and extensions of it such as neural networks, defined as the negative
+    log-likelihood of a logistic model that returns ``y_pred`` probabilities
+    for its training data ``y_true``.
+    The log loss is only defined for two or more labels.
+    For a single sample with true label :math:`y \in \{0,1\}` and
+    and a probability estimate :math:`p = \operatorname{Pr}(y = 1)`, the log
+    loss is:
+
+    .. math::
+        L_{\log}(y, p) = -(y \log (p) + (1 - y) \log (1 - p))
+
+    Read more in the :ref:`User Guide <log_loss>`.
+
+    Parameters
+    ----------
+    y_true : array-like or label indicator matrix
+        Ground truth (correct) labels for n_samples samples.
+
+    y_pred : array-like of float, shape = (n_samples, n_classes) or (n_samples,)
+        Predicted probabilities, as returned by a classifier's
+        predict_proba method. If ``y_pred.shape = (n_samples,)``
+        the probabilities provided are assumed to be that of the
+        positive class. The labels in ``y_pred`` are assumed to be
+        ordered alphabetically, as done by
+        :class:`preprocessing.LabelBinarizer`.
+
+    eps : float, default=1e-15
+        Log loss is undefined for p=0 or p=1, so probabilities are
+        clipped to max(eps, min(1 - eps, p)).
+
+    normalize : bool, default=True
+        If true, return the mean loss per sample.
+        Otherwise, return the sum of the per-sample losses.
+
+    sample_weight : array-like of shape (n_samples,), default=None
+        Sample weights.
+
+    labels : array-like, default=None
+        If not provided, labels will be inferred from y_true. If ``labels``
+        is ``None`` and ``y_pred`` has shape (n_samples,) the labels are
+        assumed to be binary and are inferred from ``y_true``.
+
+    Returns
+    -------
+    loss : float
+
+    Notes
+    -----
+    The logarithm used is the natural logarithm (base-e).
+
+    Examples
+    --------
+    >>> from mars.learn.metrics import log_loss
+    >>> log_loss(["spam", "ham", "ham", "spam"],
+    ...          [[.1, .9], [.9, .1], [.8, .2], [.35, .65]])
+    0.21616...
+
+    References
+    ----------
+    C.M. Bishop (2006). Pattern Recognition and Machine Learning. Springer,
+    p. 209.
+    """
+    y_pred = check_array(y_pred, ensure_2d=False)
+    check_consistent_length(y_pred, y_true, sample_weight)
+
+    lb = LabelBinarizer()
+
+    if labels is not None:
+        lb.fit(labels)
+    else:
+        lb.fit(y_true)
+
+    if len(lb.classes_) == 1:
+        if labels is None:
+            raise ValueError('y_true contains only one label ({0}). Please '
+                             'provide the true labels explicitly through the '
+                             'labels argument.'.format(lb.classes_[0].fetch()))
+        else:
+            raise ValueError('The labels array needs to contain at least two '
+                             'labels for log_loss, '
+                             'got {0}.'.format(lb.classes_.fetch()))
+
+    transformed_labels = lb.transform(y_true)
+
+    if transformed_labels.shape[1] == 1:
+        transformed_labels = mt.append(1 - transformed_labels,
+                                       transformed_labels, axis=1)
+
+    # Clipping
+    y_pred = mt.clip(y_pred, eps, 1 - eps)
+
+    # If y_pred is of single dimension, assume y_true to be binary
+    # and then check.
+    if y_pred.ndim == 1:  # pragma: no cover
+        y_pred = y_pred[:, mt.newaxis]
+    if y_pred.shape[1] == 1:  # pragma: no cover
+        y_pred = mt.append(1 - y_pred, y_pred, axis=1)
+
+    # Check if dimensions are consistent.
+    transformed_labels = check_array(transformed_labels)
+    if len(lb.classes_) != y_pred.shape[1]:
+        if labels is None:
+            raise ValueError("y_true and y_pred contain different number of "
+                             "classes {0}, {1}. Please provide the true "
+                             "labels explicitly through the labels argument. "
+                             "Classes found in "
+                             "y_true: {2}".format(transformed_labels.shape[1],
+                                                  y_pred.shape[1],
+                                                  lb.classes_.fetch()))
+        else:
+            raise ValueError('The number of classes in labels is different '
+                             'from that in y_pred. Classes found in '
+                             'labels: {0}'.format(lb.classes_.fetch()))
+
+    # Renormalize
+    y_pred /= y_pred.sum(axis=1)[:, mt.newaxis]
+    loss = -(transformed_labels * mt.log(y_pred)).sum(axis=1)
+
+    return _weighted_sum(loss, sample_weight, normalize).execute()


Test patch:
diff --git a/mars/learn/metrics/tests/test_classification.py b/mars/learn/metrics/tests/test_classification.py
--- a/mars/learn/metrics/tests/test_classification.py
+++ b/mars/learn/metrics/tests/test_classification.py
@@ -15,10 +15,12 @@
 import numpy as np
 import pytest
 from sklearn.metrics import accuracy_score as sklearn_accuracy_score
+from sklearn.utils._testing import assert_almost_equal
+from sklearn.utils._mocking import MockDataFrame
 
 from .... import tensor as mt
 from ....lib.sparse import SparseNDArray
-from .. import accuracy_score
+from .. import accuracy_score, log_loss
 from .._classification import _check_targets
 
 
@@ -141,3 +143,87 @@ def test_accuracy_score(setup):
     expected = sklearn_accuracy_score(y_true, y_pred, sample_weight=sample_weight,
                                       normalize=False)
     assert pytest.approx(result) == expected
+
+
+def test_log_loss(setup):
+    # binary case with symbolic labels ("no" < "yes")
+    y_true = ["no", "no", "no", "yes", "yes", "yes"]
+    y_pred = mt.array([[0.5, 0.5], [0.1, 0.9], [0.01, 0.99],
+                       [0.9, 0.1], [0.75, 0.25], [0.001, 0.999]])
+    loss = log_loss(y_true, y_pred).fetch()
+    assert_almost_equal(loss, 1.8817971)
+
+    # multiclass case; adapted from http://bit.ly/RJJHWA
+    y_true = [1, 0, 2]
+    y_pred = [[0.2, 0.7, 0.1], [0.6, 0.2, 0.2], [0.6, 0.1, 0.3]]
+    loss = log_loss(y_true, y_pred, normalize=True).fetch()
+    assert_almost_equal(loss, 0.6904911)
+
+    # check that we got all the shapes and axes right
+    # by doubling the length of y_true and y_pred
+    y_true *= 2
+    y_pred *= 2
+    loss = log_loss(y_true, y_pred, normalize=False).fetch()
+    assert_almost_equal(loss, 0.6904911 * 6, decimal=6)
+
+    # check eps and handling of absolute zero and one probabilities
+    y_pred = np.asarray(y_pred) > .5
+    loss = log_loss(y_true, y_pred, normalize=True, eps=.1).fetch()
+    assert_almost_equal(loss, log_loss(y_true, np.clip(y_pred, .1, .9)).fetch())
+
+    # raise error if number of classes are not equal.
+    y_true = [1, 0, 2]
+    y_pred = [[0.2, 0.7], [0.6, 0.5], [0.4, 0.1]]
+    with pytest.raises(ValueError):
+        log_loss(y_true, y_pred)
+    with pytest.raises(ValueError):
+        log_loss(y_true, y_pred, labels=[0, 1, 2])
+
+    # case when y_true is a string array object
+    y_true = ["ham", "spam", "spam", "ham"]
+    y_pred = [[0.2, 0.7], [0.6, 0.5], [0.4, 0.1], [0.7, 0.2]]
+    loss = log_loss(y_true, y_pred).fetch()
+    assert_almost_equal(loss, 1.0383217, decimal=6)
+
+    # test labels option
+
+    y_true = [2, 2]
+    y_pred = [[0.2, 0.7], [0.6, 0.5]]
+    y_score = np.array([[0.1, 0.9], [0.1, 0.9]])
+    error_str = (r'y_true contains only one label \(2\). Please provide '
+                 r'the true labels explicitly through the labels argument.')
+    with pytest.raises(ValueError, match=error_str):
+        log_loss(y_true, y_pred)
+    error_str = (r'The labels array needs to contain at least two '
+                 r'labels for log_loss, got \[1\].')
+    with pytest.raises(ValueError, match=error_str):
+        log_loss(y_true, y_pred, labels=[1])
+
+    # works when the labels argument is used
+
+    true_log_loss = -np.mean(np.log(y_score[:, 1]))
+    calculated_log_loss = log_loss(y_true, y_score, labels=[1, 2]).fetch()
+    assert_almost_equal(calculated_log_loss, true_log_loss)
+
+    # ensure labels work when len(np.unique(y_true)) != y_pred.shape[1]
+    y_true = [1, 2, 2]
+    y_score2 = [[0.2, 0.7, 0.3], [0.6, 0.5, 0.3], [0.3, 0.9, 0.1]]
+    loss = log_loss(y_true, y_score2, labels=[1, 2, 3]).fetch()
+    assert_almost_equal(loss, 1.0630345, decimal=6)
+
+
+def test_log_loss_pandas_input(setup):
+    # case when input is a pandas series and dataframe gh-5715
+    y_tr = np.array(["ham", "spam", "spam", "ham"])
+    y_pr = np.array([[0.2, 0.7], [0.6, 0.5], [0.4, 0.1], [0.7, 0.2]])
+    types = [(MockDataFrame, MockDataFrame)]
+    try:
+        from pandas import Series, DataFrame
+        types.append((Series, DataFrame))
+    except ImportError:
+        pass
+    for TrueInputType, PredInputType in types:
+        # y_pred dataframe, y_true series
+        y_true, y_pred = TrueInputType(y_tr), PredInputType(y_pr)
+        loss = log_loss(y_true, y_pred).fetch()
+        assert_almost_equal(loss, 1.0383217, decimal=6)