Spaces:
Sleeping
Sleeping
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. | |
# | |
# Licensed under the Apache License, Version 2.0 (the "License"); | |
# you may not use this file except in compliance with the License. | |
# You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, software | |
# distributed under the License is distributed on an "AS IS" BASIS, | |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
# See the License for the specific language governing permissions and | |
# limitations under the License. | |
import evaluate | |
import datasets | |
import numpy as np | |
from seametrics.horizon.utils import * | |
_CITATION = """\ | |
@InProceedings{huggingface:module, | |
title = {Horizon Metrics}, | |
authors={huggingface, Inc.}, | |
year={2024} | |
} | |
""" | |
# TODO: Add description of the module here | |
_DESCRIPTION = """\ | |
This metric is intended to calculate horizon prediction metrics.""" | |
# TODO: Add description of the arguments of the module here | |
_KWARGS_DESCRIPTION = """ | |
Calculates how good are predictions given some references, using certain scores | |
Args: | |
predictions: list of predictions for each image. Each prediction | |
should be a nested array like this: | |
- [[x1, y1], [x2, y2]] | |
references: list of references for each image. Each reference | |
should be a nested array like this: | |
- [[x1, y1], [x2, y2]] | |
Returns: | |
dict containing following metrics: | |
'average_slope_error': Measures the average difference in slope between the predicted and ground truth horizon. | |
'average_midpoint_error': Calculates the average difference in midpoint position between the predicted and ground truth horizon. | |
'stddev_slope_error': Indicates the variability of errors in slope between the predicted and ground truth horizon. | |
'stddev_midpoint_error': Quantifies the variability of errors in midpoint position between the predicted and ground truth horizon. | |
'max_slope_error': Represents the maximum difference in slope between the predicted and ground truth horizon. | |
'max_midpoint_error': Indicates the maximum difference in midpoint position between the predicted and ground truth horizon. | |
'num_slope_error_jumps': Calculates the differences between errors in successive frames for the slope. It then counts the number of jumps in these errors by comparing the absolute differences to a specified threshold. | |
'num_midpoint_error_jumps': Calculates the differences between errors in successive frames for the midpoint. It then counts the number of jumps in these errors by comparing the absolute differences to a specified threshold. | |
Examples: | |
>>> ground_truth_points = [[[0.0, 0.5384765625], [1.0, 0.4931640625]], | |
[[0.0, 0.53796875], [1.0, 0.4928515625]], | |
[[0.0, 0.5374609375], [1.0, 0.4925390625]], | |
[[0.0, 0.536953125], [1.0, 0.4922265625]], | |
[[0.0, 0.5364453125], [1.0, 0.4919140625]]] | |
>>> prediction_points = [[[0.0, 0.5428930956049597], [1.0, 0.4642497615378973]], | |
[[0.0, 0.5428930956049597], [1.0, 0.4642497615378973]], | |
[[0.0, 0.523573113510805], [1.0, 0.47642688648919496]], | |
[[0.0, 0.5200016849393765], [1.0, 0.4728554579177664]], | |
[[0.0, 0.523573113510805], [1.0, 0.47642688648919496]]] | |
>>> module = evaluate.load("SEA-AI/horizon-metrics", roll_threshold=0.5, pitch_threshold=0.1, vertical_fov_degrees=25.6, height=512) | |
>>> module.add(predictions=ground_truth_points, references=prediction_points) | |
>>> module.compute() | |
>>> {'average_slope_error': 0.014823194839790999, | |
'average_midpoint_error': 0.014285714285714301, | |
'stddev_slope_error': 0.01519178791378349, | |
'stddev_midpoint_error': 0.0022661781575342445, | |
'max_slope_error': 0.033526146567062376, | |
'max_midpoint_error': 0.018161272321428612, | |
'num_slope_error_jumps': 1, | |
'num_midpoint_error_jumps': 1} | |
""" | |
class HorizonMetrics(evaluate.Metric): | |
""" | |
HorizonMetrics is a metric class that calculates horizon prediction metrics. | |
Args: | |
roll_threshold (float, optional): Threshold for roll angle. Defaults to 0.5. | |
pitch_threshold (float, optional): Threshold for pitch angle. Defaults to 0.1. | |
vertical_fov_degrees (float, optional): Vertical field of view in degrees. Defaults to 25.6. | |
**kwargs: Additional keyword arguments. | |
Attributes: | |
slope_threshold (float): Threshold for slope calculated from roll threshold. | |
midpoint_threshold (float): Threshold for midpoint calculated from pitch threshold. | |
predictions (list): List of predicted horizons. | |
ground_truth_det (list): List of ground truth horizons. | |
slope_error_list (list): List of slope errors. | |
midpoint_error_list (list): List of midpoint errors. | |
Methods: | |
_info(): Returns the metric information. | |
add(predictions, references, **kwargs): Updates the predictions and ground truth detections. | |
_compute(predictions, references, **kwargs): Computes the horizon error across the sequence. | |
""" | |
def __init__(self, | |
roll_threshold=0.5, | |
pitch_threshold=0.1, | |
vertical_fov_degrees=25.6, | |
height=512, | |
**kwargs): | |
super().__init__(**kwargs) | |
self.slope_threshold = roll_to_slope(roll_threshold) | |
self.midpoint_threshold = pitch_to_midpoint(pitch_threshold, | |
vertical_fov_degrees) | |
self.predictions = None | |
self.ground_truth_det = None | |
self.slope_error_list = [] | |
self.midpoint_error_list = [] | |
self.height = height | |
self.vertical_fov_degrees = vertical_fov_degrees | |
def _info(self): | |
""" | |
Returns the metric information. | |
Returns: | |
MetricInfo: The metric information. | |
""" | |
return evaluate.MetricInfo( | |
# This is the description that will appear on the modules page. | |
module_type="metric", | |
description=_DESCRIPTION, | |
citation=_CITATION, | |
inputs_description=_KWARGS_DESCRIPTION, | |
# This defines the format of each prediction and reference | |
features=datasets.Features({ | |
'predictions': | |
datasets.Sequence( | |
datasets.Sequence( | |
datasets.Sequence(datasets.Value("float")))), | |
'references': | |
datasets.Sequence( | |
datasets.Sequence( | |
datasets.Sequence(datasets.Value("float")))), | |
}), | |
codebase_urls=["http://github.com/path/to/codebase/of/new_module"]) | |
def add(self, *, predictions, references, **kwargs): | |
""" | |
Updates the predictions and ground truth detections. | |
Parameters: | |
predictions (list): List of predicted horizons. | |
references (list): List of ground truth horizons. | |
**kwargs: Additional keyword arguments. | |
""" | |
super(evaluate.Metric, self).add(prediction=predictions, | |
references=references, | |
**kwargs) | |
self.predictions = predictions | |
self.ground_truth_det = references | |
def _compute(self, *, predictions, references, **kwargs): | |
""" | |
Computes the horizon error across the sequence. | |
Returns: | |
float: The computed horizon error. | |
""" | |
# calculate erros and store values in slope_error_list and midpoint_error_list | |
for annotated_horizon, proposed_horizon in zip(self.ground_truth_det, | |
self.predictions): | |
if annotated_horizon is None or proposed_horizon is None: | |
continue | |
slope_error, midpoint_error = calculate_horizon_error( | |
annotated_horizon, proposed_horizon) | |
self.slope_error_list.append(slope_error) | |
self.midpoint_error_list.append(midpoint_error) | |
# calculate slope errors, midpoint errors and jumps | |
result = calculate_horizon_error_across_sequence( | |
self.slope_error_list, self.midpoint_error_list, | |
self.slope_threshold, self.midpoint_threshold, | |
self.vertical_fov_degrees, self.height) | |
# calulcate detection rate | |
detected_horizon_count = len( | |
self.predictions) - self.predictions.count(None) | |
detected_gt_count = len( | |
self.ground_truth_det) - self.ground_truth_det.count(None) | |
detection_rate = detected_horizon_count / detected_gt_count | |
result['detection_rate'] = detection_rate | |
return result | |