Repository: asreview/asreview

Issue:
Implement Normalized Discounted Cumulative Gain metric
Implementing the Normalized Discounted Cumulative Gain as a performance metric in ASReview https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG would be nice. This algorithm penalizes relevant records lower in the result list.

Hints from issue comments:

Solution patch:
diff --git a/asreview/metrics.py b/asreview/metrics.py
index 2808bbf3..d7b4dcef 100644
--- a/asreview/metrics.py
+++ b/asreview/metrics.py
@@ -14,7 +14,7 @@
 
 """Performance metrics for activate learning results."""
 
-__all__ = ["loss"]
+__all__ = ["loss", "ndcg"]
 
 import numpy as np
 
@@ -39,3 +39,29 @@ def loss(labels: list[int]):
     return float(
         (Ny * (Nx - (Ny - 1) / 2) - np.cumsum(labels).sum()) / (Ny * (Nx - Ny))
     )
+
+
+def ndcg(labels: list[int]):
+    """Compute the Normalized Discounted Cumulative Gain (NDCG)
+
+    Basesd on: https://doi.org/10.48550/arXiv.1304.6480
+
+    Arguments
+    ---------
+    labels: list
+        List of binary labels (0 or 1).
+
+    Returns
+    -------
+    float:
+        The NDCG score.
+    """
+    Ny = sum(labels)
+    Nx = len(labels)
+    if Ny == 0 or Nx == Ny:
+        raise ValueError("Labels must contain two distinct classes.")
+
+    dcg = np.sum(labels / np.log2(np.arange(2, Nx + 2)))
+    idcg = np.sum(1 / np.log2(np.arange(2, Ny + 2)))
+
+    return float(dcg / idcg)
diff --git a/asreview/simulation/simulate.py b/asreview/simulation/simulate.py
index 2fb3e10c..b291bbdf 100644
--- a/asreview/simulation/simulate.py
+++ b/asreview/simulation/simulate.py
@@ -21,6 +21,7 @@ import pandas as pd
 from tqdm import tqdm
 
 from asreview.metrics import loss
+from asreview.metrics import ndcg
 from asreview.models.stoppers import LastRelevant
 from asreview.models.stoppers import NLabeled
 from asreview.state.contextmanager import open_state
@@ -216,7 +217,9 @@ class Simulate:
             )
 
             if self.print_progress:
-                print(f"\nLoss: {loss(padded_results):.3f}")
+                print(
+                    f"\nLoss: {loss(padded_results):.3f}\nNDCG: {ndcg(padded_results):.3f}"
+                )
 
     def label(self, record_ids, cycle=None):
         """Label the records with the given record_ids.


Test patch:
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index b2f497d9..d676b060 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -3,6 +3,7 @@ import pytest
 from numpy.testing import assert_almost_equal
 
 from asreview.metrics import loss
+from asreview.metrics import ndcg
 
 
 @pytest.mark.parametrize(
@@ -78,3 +79,78 @@ def test_random_loss_values():
 
         loss_value = loss(labels)
         assert 0 <= loss_value <= 1
+
+
+@pytest.mark.parametrize(
+    "labels, expected_value",
+    [
+        ([1, 0], 1),
+        ([0, 1], 0.6309297535714575),
+        ([1, 1, 0, 0, 0], 1),
+        ([0, 0, 0, 1, 1], 0.5012658353418871),
+        ([1, 0, 1], 0.9197207891481876),
+        (
+            [
+                0,
+                1,
+                0,
+                0,
+                0,
+                1,
+                0,
+                1,
+                0,
+                1,
+                0,
+                0,
+                1,
+                1,
+                0,
+                0,
+                0,
+                1,
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                0,
+                1,
+                0,
+                1,
+                0,
+                0,
+                1,
+                1,
+            ],
+            0.6627901128094049,
+        ),
+    ],
+)
+def test_ndcg_value_function(labels, expected_value):
+    loss_value = ndcg(labels)
+    assert_almost_equal(loss_value, expected_value)
+
+
+@pytest.mark.parametrize(
+    "labels",
+    [[0, 0, 0], [0], [1]],
+)
+def test_lndcg_value_error_cases(labels):
+    with pytest.raises(ValueError):
+        ndcg(labels)
+
+
+def test_random_ndcg_values():
+    np.random.seed(42)
+    for _ in range(100):
+        length = np.random.randint(2, 100)
+        labels = np.random.randint(0, 2, length)
+
+        # Ensure labels are not all 0 or all 1
+        if np.all(labels == 0) or np.all(labels == 1):
+            labels[np.random.randint(0, length)] = 1 - labels[0]
+
+        loss_value = ndcg(labels)
+        assert 0 <= loss_value <= 1