|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""Object detection calibration metrics. |
|
""" |
|
|
|
from __future__ import absolute_import |
|
from __future__ import division |
|
from __future__ import print_function |
|
|
|
import tensorflow as tf |
|
from tensorflow.python.ops import metrics_impl |
|
|
|
|
|
def _safe_div(numerator, denominator): |
|
"""Divides two tensors element-wise, returning 0 if the denominator is <= 0. |
|
|
|
Args: |
|
numerator: A real `Tensor`. |
|
denominator: A real `Tensor`, with dtype matching `numerator`. |
|
|
|
Returns: |
|
0 if `denominator` <= 0, else `numerator` / `denominator` |
|
""" |
|
t = tf.truediv(numerator, denominator) |
|
zero = tf.zeros_like(t, dtype=denominator.dtype) |
|
condition = tf.greater(denominator, zero) |
|
zero = tf.cast(zero, t.dtype) |
|
return tf.where(condition, t, zero) |
|
|
|
|
|
def _ece_from_bins(bin_counts, bin_true_sum, bin_preds_sum, name): |
|
"""Calculates Expected Calibration Error from accumulated statistics.""" |
|
bin_accuracies = _safe_div(bin_true_sum, bin_counts) |
|
bin_confidences = _safe_div(bin_preds_sum, bin_counts) |
|
abs_bin_errors = tf.abs(bin_accuracies - bin_confidences) |
|
bin_weights = _safe_div(bin_counts, tf.reduce_sum(bin_counts)) |
|
return tf.reduce_sum(abs_bin_errors * bin_weights, name=name) |
|
|
|
|
|
def expected_calibration_error(y_true, y_pred, nbins=20): |
|
"""Calculates Expected Calibration Error (ECE). |
|
|
|
ECE is a scalar summary statistic of calibration error. It is the |
|
sample-weighted average of the difference between the predicted and true |
|
probabilities of a positive detection across uniformly-spaced model |
|
confidences [0, 1]. See referenced paper for a thorough explanation. |
|
|
|
Reference: |
|
Guo, et. al, "On Calibration of Modern Neural Networks" |
|
Page 2, Expected Calibration Error (ECE). |
|
https://arxiv.org/pdf/1706.04599.pdf |
|
|
|
This function creates three local variables, `bin_counts`, `bin_true_sum`, and |
|
`bin_preds_sum` that are used to compute ECE. For estimation of the metric |
|
over a stream of data, the function creates an `update_op` operation that |
|
updates these variables and returns the ECE. |
|
|
|
Args: |
|
y_true: 1-D tf.int64 Tensor of binarized ground truth, corresponding to each |
|
prediction in y_pred. |
|
y_pred: 1-D tf.float32 tensor of model confidence scores in range |
|
[0.0, 1.0]. |
|
nbins: int specifying the number of uniformly-spaced bins into which y_pred |
|
will be bucketed. |
|
|
|
Returns: |
|
value_op: A value metric op that returns ece. |
|
update_op: An operation that increments the `bin_counts`, `bin_true_sum`, |
|
and `bin_preds_sum` variables appropriately and whose value matches `ece`. |
|
|
|
Raises: |
|
InvalidArgumentError: if y_pred is not in [0.0, 1.0]. |
|
""" |
|
bin_counts = metrics_impl.metric_variable( |
|
[nbins], tf.float32, name='bin_counts') |
|
bin_true_sum = metrics_impl.metric_variable( |
|
[nbins], tf.float32, name='true_sum') |
|
bin_preds_sum = metrics_impl.metric_variable( |
|
[nbins], tf.float32, name='preds_sum') |
|
|
|
with tf.control_dependencies([ |
|
tf.assert_greater_equal(y_pred, 0.0), |
|
tf.assert_less_equal(y_pred, 1.0), |
|
]): |
|
bin_ids = tf.histogram_fixed_width_bins(y_pred, [0.0, 1.0], nbins=nbins) |
|
|
|
with tf.control_dependencies([bin_ids]): |
|
update_bin_counts_op = tf.assign_add( |
|
bin_counts, tf.to_float(tf.bincount(bin_ids, minlength=nbins))) |
|
update_bin_true_sum_op = tf.assign_add( |
|
bin_true_sum, |
|
tf.to_float(tf.bincount(bin_ids, weights=y_true, minlength=nbins))) |
|
update_bin_preds_sum_op = tf.assign_add( |
|
bin_preds_sum, |
|
tf.to_float(tf.bincount(bin_ids, weights=y_pred, minlength=nbins))) |
|
|
|
ece_update_op = _ece_from_bins( |
|
update_bin_counts_op, |
|
update_bin_true_sum_op, |
|
update_bin_preds_sum_op, |
|
name='update_op') |
|
ece = _ece_from_bins(bin_counts, bin_true_sum, bin_preds_sum, name='value') |
|
return ece, ece_update_op |
|
|
