Repository: sintel-dev/Draco (language: python)

Issue:
Add new False Positive Rate metric
Add a metric that gives the false positive rate for a given/fixed true positive rate for failures. This provides us a single metric to track. 


Discussion / hints:

Fix patch:
diff --git a/greenguard/metrics.py b/greenguard/metrics.py
index 54a151e..ef1c249 100644
--- a/greenguard/metrics.py
+++ b/greenguard/metrics.py
@@ -1,18 +1,55 @@
 # -*- coding: utf-8 -*-
+import logging
 
+import numpy as np
 from sklearn.metrics import (
-    accuracy_score, f1_score, mean_absolute_error, mean_squared_error, r2_score)
+    accuracy_score, f1_score, mean_absolute_error, mean_squared_error, roc_curve, r2_score)
+
+LOGGER = logging.getLogger(__name__)
 
 
 def f1_macro(exp, obs):
     return f1_score(exp, obs, average='macro')
 
 
+def fpr_score(ground_truth, probabilities, tpr=1):
+    """Compute the False Positive Rate associated with the given True Positive Rate.
+
+    This metric computes the False Positive Rate that needs to be assumed in order
+    to achieve the desired True Positive Rate.
+    The metric is computed by finding the minimum necessary threshold to ensure
+    that the TPR is satisfied and then computing the associated FPR. The final output
+    is 1 minus the found FPR to produce a maximization score between 0 and 1.
+
+    Args:
+        ground_truth (numpy.ndarray):
+            ``numpy.ndarray`` of the known values for the given predictions.
+        probabilities (numpy.ndarray):
+            ``numpy.ndarray`` with the generated predictions in probability.
+        tpr (float):
+            ``float`` value representing the percentage of True Positive Rate
+            to be satisfied.
+
+    Returns:
+        float:
+            Value between 0 and 1, where bigger is better.
+    """
+    roc_fpr, roc_tpr, roc_threshold = roc_curve(ground_truth, probabilities, pos_label=1)
+    try:
+        index = np.where(roc_tpr >= tpr)[0][0]
+    except:
+        LOGGER.warn('Could not find a threshold that satisfies the requested True Positive Rate')
+        index = -1
+
+    return 1 - roc_fpr[index]
+
+
 METRICS = {
     'accuracy': (accuracy_score, False),
     'f1': (f1_score, False),
     'f1_macro': (f1_macro, False),
     'r2': (r2_score, False),
     'mse': (mean_squared_error, True),
-    'mae': (mean_absolute_error, True)
+    'mae': (mean_absolute_error, True),
+    'fpr_score': (fpr_score, False)
 }


Test patch:
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
new file mode 100644
index 0000000..ce14132
--- /dev/null
+++ b/tests/test_metrics.py
@@ -0,0 +1,39 @@
+import numpy as np
+
+from greenguard.metrics import fpr_score
+
+
+def test_fpr_score_perfect_scenario():
+    truth = [0, 0, 0, 1, 1, 1]
+    false_probs = [0.2, 0.4, 0.6]
+    true_probs = [0.8, 0.7, 0.9]
+    probs = np.concatenate([false_probs, true_probs])
+    score = fpr_score(truth, probs, tpr=1)
+    assert score == 1
+
+
+def test_fpr_score_predict_over_half():
+    truth = [0, 0, 0, 0, 1, 1, 1, 1]
+    false_probs = [0.1, 0.2, 0.4, 0.6]
+    true_probs = [0.5, 0.7, 0.8, 0.9]
+    probs = np.concatenate([false_probs, true_probs])
+    score = fpr_score(truth, probs, tpr=1)
+    assert score == 0.75
+
+
+def test_fpr_score_predict_half():
+    truth = [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
+    false_probs = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
+    true_probs = [0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
+    probs = np.concatenate([false_probs, true_probs])
+    score = fpr_score(truth, probs, tpr=1)
+    assert score == 0.5
+
+
+def test_fpr_score_predict_one_third():
+    truth = [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
+    false_probs = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
+    true_probs = [0.3, 0.4, 0.5, 0.7, 0.8, 0.9]
+    probs = np.concatenate([false_probs, true_probs])
+    score = fpr_score(truth, probs, tpr=1)
+    assert round(score, 4) == 0.3333