object_detection/builders/calibration_builder_test.py

# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Tests for calibration_builder."""

import numpy as np
from scipy import interpolate
import tensorflow as tf
from object_detection.builders import calibration_builder
from object_detection.protos import calibration_pb2


class CalibrationBuilderTest(tf.test.TestCase):

  def test_tf_linear_interp1d_map(self):
    """Tests TF linear interpolation mapping to a single number."""
    with self.test_session() as sess:
      tf_x = tf.constant([0., 0.5, 1.])
      tf_y = tf.constant([0.5, 0.5, 0.5])
      new_x = tf.constant([0., 0.25, 0.5, 0.75, 1.])
      tf_map_outputs = calibration_builder._tf_linear_interp1d(
          new_x, tf_x, tf_y)
      tf_map_outputs_np = sess.run([tf_map_outputs])
    self.assertAllClose(tf_map_outputs_np, [[0.5, 0.5, 0.5, 0.5, 0.5]])

  def test_tf_linear_interp1d_interpolate(self):
    """Tests TF 1d linear interpolation not mapping to a single number."""
    with self.test_session() as sess:
      tf_x = tf.constant([0., 0.5, 1.])
      tf_y = tf.constant([0.6, 0.7, 1.0])
      new_x = tf.constant([0., 0.25, 0.5, 0.75, 1.])
      tf_interpolate_outputs = calibration_builder._tf_linear_interp1d(
          new_x, tf_x, tf_y)
      tf_interpolate_outputs_np = sess.run([tf_interpolate_outputs])
    self.assertAllClose(tf_interpolate_outputs_np, [[0.6, 0.65, 0.7, 0.85, 1.]])

  @staticmethod
  def _get_scipy_interp1d(new_x, x, y):
    """Helper performing 1d linear interpolation using SciPy."""
    interpolation1d_fn = interpolate.interp1d(x, y)
    return interpolation1d_fn(new_x)

  def _get_tf_interp1d(self, new_x, x, y):
    """Helper performing 1d linear interpolation using Tensorflow."""
    with self.test_session() as sess:
      tf_interp_outputs = calibration_builder._tf_linear_interp1d(
          tf.convert_to_tensor(new_x, dtype=tf.float32),
          tf.convert_to_tensor(x, dtype=tf.float32),
          tf.convert_to_tensor(y, dtype=tf.float32))
      np_tf_interp_outputs = sess.run(tf_interp_outputs)
    return np_tf_interp_outputs

  def test_tf_linear_interp1d_against_scipy_map(self):
    """Tests parity of TF linear interpolation with SciPy for simple mapping."""
    length = 10
    np_x = np.linspace(0, 1, length)

    # Mapping all numbers to 0.5
    np_y_map = np.repeat(0.5, length)

    # Scipy and TF interpolations
    test_data_np = np.linspace(0, 1, length * 10)
    scipy_map_outputs = self._get_scipy_interp1d(test_data_np, np_x, np_y_map)
    np_tf_map_outputs = self._get_tf_interp1d(test_data_np, np_x, np_y_map)
    self.assertAllClose(scipy_map_outputs, np_tf_map_outputs)

  def test_tf_linear_interp1d_against_scipy_interpolate(self):
    """Tests parity of TF linear interpolation with SciPy."""
    length = 10
    np_x = np.linspace(0, 1, length)

    # Requires interpolation over 0.5 to 1 domain
    np_y_interp = np.linspace(0.5, 1, length)

    # Scipy interpolation for comparison
    test_data_np = np.linspace(0, 1, length * 10)
    scipy_interp_outputs = self._get_scipy_interp1d(test_data_np, np_x,
                                                    np_y_interp)
    np_tf_interp_outputs = self._get_tf_interp1d(test_data_np, np_x,
                                                 np_y_interp)
    self.assertAllClose(scipy_interp_outputs, np_tf_interp_outputs)

  @staticmethod
  def _add_function_approximation_to_calibration_proto(calibration_proto,
                                                       x_array,
                                                       y_array,
                                                       class_label):
    """Adds a function approximation to calibration proto for a class label."""
    # Per-class calibration.
    if class_label:
      label_function_approximation = (calibration_proto
                                      .label_function_approximations
                                      .label_xy_pairs_map[class_label])
    # Class-agnostic calibration.
    else:
      label_function_approximation = (calibration_proto
                                      .function_approximation
                                      .x_y_pairs)
    for x, y in zip(x_array, y_array):
      x_y_pair_message = label_function_approximation.x_y_pair.add()
      x_y_pair_message.x = x
      x_y_pair_message.y = y

  def test_class_agnostic_function_approximation(self):
    """Ensures that calibration appropriate values, regardless of class."""
    # Generate fake calibration proto. For this interpolation, any input on
    # [0.0, 0.5] should be divided by 2 and any input on (0.5, 1.0] should have
    # 0.25 subtracted from it.
    class_agnostic_x = np.asarray([0.0, 0.5, 1.0])
    class_agnostic_y = np.asarray([0.0, 0.25, 0.75])
    calibration_config = calibration_pb2.CalibrationConfig()
    self._add_function_approximation_to_calibration_proto(calibration_config,
                                                          class_agnostic_x,
                                                          class_agnostic_y,
                                                          class_label=None)

    od_graph = tf.Graph()
    with self.test_session(graph=od_graph) as sess:
      calibration_fn = calibration_builder.build(calibration_config)
      # batch_size = 2, num_classes = 2, num_anchors = 2.
      class_predictions_with_background = tf.constant(
          [[[0.1, 0.2, 0.3],
            [0.4, 0.5, 0.0]],
           [[0.6, 0.7, 0.8],
            [0.9, 1.0, 1.0]]], dtype=tf.float32)

      # Everything should map to 0.5 if classes are ignored.
      calibrated_scores = calibration_fn(class_predictions_with_background)
      calibrated_scores_np = sess.run(calibrated_scores)
    self.assertAllClose(calibrated_scores_np, [[[0.05, 0.1, 0.15],
                                                [0.2, 0.25, 0.0]],
                                               [[0.35, 0.45, 0.55],
                                                [0.65, 0.75, 0.75]]])

if __name__ == '__main__':
  tf.test.main()