Repository: deepchecks/deepchecks

Issue:
[FEAT][CV] Implement label formatters for common formats
Label Formatter can accept a callable, or a string signifying that the labels come in a common format. The built in converter functions for these string values need implementing.  


Discussion:
@matanper suggested to have a string with meaningful letters:
l - labels
xy - point
wh - width height

Then, we can support lxyxy, lxywh, xyxyl, etc...

Resolving patch:
diff --git a/deepchecks/vision/dataset.py b/deepchecks/vision/dataset.py
--- a/deepchecks/vision/dataset.py
+++ b/deepchecks/vision/dataset.py
@@ -76,8 +76,8 @@ class VisionData:
     or else the callable label_formatter should be able to transform the labels to the desired format.
     """
 
-    label_formatter: BaseLabelFormatter
-    image_formatter: ImageFormatter
+    label_transformer: Optional[BaseLabelFormatter]
+    image_transformer: Optional[ImageFormatter]
     task_type: Optional[TaskType]
     sample_iteration_limit: int
     _data: DataLoader
@@ -172,12 +172,12 @@ def n_of_samples_per_class(self) -> Dict[Any, int]:
 
     def to_display_data(self, batch):
         """Convert a batch of data outputted by the data loader to a format that can be displayed."""
-        return self.image_formatter(batch)
+        return self.image_formatter(batch)  # pylint: disable=not-callable
 
     @property
     def data_dimension(self):
         """Return how many dimensions the image data have."""
-        image = self.image_formatter(next(iter(self)))[0]
+        image = self.image_formatter(next(iter(self)))[0]  # pylint: disable=not-callable
         return ImageInfo(image).get_dimension()
 
     @property
diff --git a/deepchecks/vision/utils/detection_formatters.py b/deepchecks/vision/utils/detection_formatters.py
--- a/deepchecks/vision/utils/detection_formatters.py
+++ b/deepchecks/vision/utils/detection_formatters.py
@@ -9,13 +9,17 @@
 # ----------------------------------------------------------------------------
 #
 """Module for defining detection encoders."""
-from typing import Union, Callable
+from collections import Counter
 from itertools import chain
+from typing import Union, Callable, Sequence, List, Tuple, Iterable
+
 import numpy as np
 import torch
+from PIL.Image import Image
 
-from .base_formatters import BaseLabelFormatter, BasePredictionFormatter
 from deepchecks.core.errors import DeepchecksValueError
+from .base_formatters import BaseLabelFormatter, BasePredictionFormatter
+
 
 __all__ = ['DetectionLabelFormatter', 'DetectionPredictionFormatter']
 
@@ -55,18 +59,21 @@ class DetectionLabelFormatter(BaseLabelFormatter):
 
     """
 
-    label_formatter: Union[str, Callable]
+    label_formatter: Callable
 
     def __init__(self, label_formatter: Union[str, Callable] = lambda x: x[1]):
         super().__init__(label_formatter)
-        self.label_formatter = label_formatter
+        if isinstance(label_formatter, str):
+            self.label_formatter = lambda batch: convert_batch_of_bboxes(
+                zip(*batch),  # batch - expecting to receive tuple[iterable[image], iterable[bboxes]]
+                label_formatter
+            )
+        else:
+            self.label_formatter = label_formatter
 
     def __call__(self, *args, **kwargs):
         """Call the encoder."""
-        if isinstance(self.label_formatter, Callable):
-            return self.label_formatter(*args, **kwargs)
-        elif isinstance(self.label_formatter, str):
-            pass
+        return self.label_formatter(*args, **kwargs)
 
     def get_classes(self, batch_labels):
         """Get a labels batch and return classes inside it."""
@@ -192,3 +199,287 @@ def validate_prediction(self, batch, model, device, n_classes: int = None, eps:
         if batch_predictions[0].shape[1] != 6:
             raise DeepchecksValueError('Check requires detection predictions to be a list of 2D tensors, when '
                                        'each row has 6 columns: [x, y, width, height, class_probability, class_id]')
+
+
+def verify_bbox_format_notation(notation: str) -> Tuple[bool, List[str]]:
+    """Verify and tokenize bbox format notation.
+
+    Parameters
+    ----------
+    notation : str
+        format notation to verify and to tokenize
+
+    Returns
+    -------
+    Tuple[
+        bool,
+        List[Literal['label', 'width', 'height', 'xmin', 'ymin', 'xmax', 'ymax', 'xcenter', 'ycenter']]
+    ]
+        first item indicates whether coordinates are normalized or not,
+        second represents format of the bbox
+    """
+    tokens = []
+    are_coordinates_normalized = False
+    current = notation = notation.strip().lower()
+    current_pos = 0
+
+    while current:
+        if current.startswith('l'):
+            tokens.append('l')
+            current = current[1:]
+            current_pos = current_pos + 1
+        elif current.startswith('wh'):
+            tokens.append('wh')
+            current = current[2:]
+            current_pos = current_pos + 2
+        elif current.startswith('xy'):
+            tokens.append('xy')
+            current = current[2:]
+            current_pos = current_pos + 2
+        elif current.startswith('cxcy'):
+            tokens.append('cxcy')
+            current = current[4:]
+            current_pos = current_pos + 4
+        elif current.startswith('n') and current_pos == 0:
+            are_coordinates_normalized = True
+            current = current[1:]
+            current_pos = current_pos + 1
+        elif current.startswith('n') and (current_pos + 1) == len(notation):
+            are_coordinates_normalized = True
+            current_pos = current_pos + 1
+            break
+        else:
+            raise ValueError(
+                f'Wrong bbox format notation - {notation}. '
+                f'Incorrect or unknown sequence of charecters starting from position {current_pos} '
+                f'(sequence: ...{notation[current_pos:]}'
+            )
+
+    received_combination = Counter(tokens)
+    allowed_combinations = (
+        {'l': 1, 'xy': 2},
+        {'l': 1, 'xy': 1, 'wh': 1},
+        {'l': 1, 'cxcy': 1, 'wh': 1}
+    )
+
+    if sum(c == received_combination for c in allowed_combinations) != 1:
+        raise ValueError(
+            f'Incorrect bbox format notation - {notation}.\n'
+            'Only next combinations of elements are allowed:\n'
+            '+ lxyxy (label, upper-left corner, bottom-right corner)\n'
+            '+ lxywh (label, upper-left corner, bbox width and height)\n'
+            '+ lcxcywh (label, bbox center, bbox width and height)\n'
+            '+ lcxcywhn (label, normalized bbox center, bbox width and height)\n\n'
+            ''
+            'Note:\n'
+            '- notation elements (l, xy, cxcy, wh) can be placed in any order '
+            'but only above combinations of elements are allowed\n'
+            '- "n" at the begining or at the ned of the notation indicates '
+            'normalized coordinates\n'
+        )
+
+    normalized_tokens = []
+
+    for t in tokens:
+        if t == 'l':
+            normalized_tokens.append('label')
+        elif t == 'wh':
+            normalized_tokens.extend(('width', 'height'))
+        elif t == 'cxcy':
+            normalized_tokens.extend(('xcenter', 'ycenter'))
+        elif t == 'xy':
+            if 'xmin' not in normalized_tokens and 'ymin' not in normalized_tokens:
+                normalized_tokens.extend(('xmin', 'ymin'))
+            else:
+                normalized_tokens.extend(('xmax', 'ymax'))
+        else:
+            raise RuntimeError('Internal Error! Unreachable part of code reached')
+
+    return are_coordinates_normalized, normalized_tokens
+
+
+_BatchOfSamples = Iterable[
+    Tuple[
+        Union[Image, np.ndarray, torch.Tensor],  # images
+        Sequence[Sequence[Union[int, float]]]  # bboxes
+    ]
+]
+
+
+def convert_batch_of_bboxes(
+    batch: _BatchOfSamples,
+    notation: str,
+    device: Union[str, torch.device, None] = None
+) -> List[torch.Tensor]:
+    """Convert batch of bboxes to the required format.
+
+    Parameters
+    ----------
+    batch : tuple like object with two items - list if images, list of bboxes
+        batch of images and bboxes
+    notation : str
+        bboxes format notation
+    device : Union[str, torch.device, None], default: None
+        device for use
+
+    Returns
+    -------
+    List[torch.Tensor]
+        list of transformed bboxes
+    """
+    are_coordinates_normalized, notation_tokens = verify_bbox_format_notation(notation)
+    output = []
+
+    for image, bboxes in batch:
+        if len(bboxes) == 0:
+            # image does not have bboxes
+            output.append(torch.tensor([]))
+            continue
+
+        if are_coordinates_normalized is False:
+            image_height = None
+            image_width = None
+        elif isinstance(image, Image):
+            image_height, image_width = image.height, image.width
+        elif isinstance(image, (np.ndarray, torch.Tensor)):
+            image_height, image_width, *_ = image.shape
+        else:
+            raise TypeError(
+                'Do not know how to take dimension sizes of '
+                f'object of type - {type(image)}'
+            )
+
+        r = []
+        for bbox in bboxes:
+            if len(bbox) < 5:
+                raise ValueError('incorrect bbox')  # TODO: better message
+            else:
+                r.append(_convert_bbox(
+                    bbox,
+                    notation_tokens,
+                    device=device,
+                    image_width=image_width,
+                    image_height=image_height,
+                ))
+        output.append(torch.stack(r, dim=0))
+
+    return output
+
+
+def convert_bbox(
+    bbox: Sequence[Union[int, float]],
+    notation: str,
+    image_width: Union[int, float, None] = None,
+    image_height: Union[int, float, None] = None,
+    device: Union[str, torch.device, None] = None
+) -> torch.Tensor:
+    """Convert bbox to the required format.
+
+    Parameters
+    ----------
+    bbox : Sequence[Sequence[Union[int, float]]]
+        bbox to transform
+    notation : str
+        bboxes format notation
+    image_width : Union[int, float, None], default: None
+        width of the image to denormalize bbox coordinates
+    image_height : Union[int, float, None], default: None
+        height of the image to denormalize bbox coordinates
+    device : Union[str, torch.device, None], default: None
+        device for use
+
+    Returns
+    -------
+    torch.Tensor
+        bbox transformed to the required by deepchecks format
+    """
+    if len(bbox) < 5:
+        raise ValueError('incorrect bbox')  # TODO: better message
+
+    are_coordinates_normalized, notation_tokens = verify_bbox_format_notation(notation)
+
+    if (
+        are_coordinates_normalized is True
+        and (image_height is None or image_width is None)
+    ):
+        raise ValueError(
+            'bbox format notation indicates that coordinates of the bbox '
+            'are normalized but \'image_height\' and \'image_width\' parameters '
+            'were not provided. Please pass image height and width parameters '
+            'or remove \'n\' element from the format notation.'
+        )
+
+    if (
+        are_coordinates_normalized is False
+        and (image_height is not None or image_width is not None)
+    ):
+        raise ValueError(
+            'bbox format notation indicates that coordinates of the bbox '
+            'are not normalized but \'image_height\' and \'image_width\' were provided. '
+            'Those parameters are redundant in the case when bbox coordinates are not '
+            'normalized. Please remove those parameters or add \'n\' element to the format '
+            'notation to indicate that coordinates are indeed normalized.'
+        )
+
+    return _convert_bbox(
+        bbox,
+        notation_tokens,
+        image_width,
+        image_height,
+        device,
+    )
+
+
+def _convert_bbox(
+    bbox: Sequence[Union[int, float]],
+    notation_tokens: List[str],
+    image_width: Union[int, float, None] = None,
+    image_height: Union[int, float, None] = None,
+    device: Union[str, torch.device, None] = None
+) -> torch.Tensor:
+    assert \
+        (image_width is not None and image_height is not None) \
+        or (image_width is None and image_height is None)
+
+    data = dict(zip(notation_tokens, bbox[:5]))
+
+    if 'xcenter' in data and 'ycenter' in data:
+        if image_width is not None and image_height is not None:
+            xcenter, ycenter = data['xcenter'] * image_width, data['ycenter'] * image_height
+        else:
+            xcenter, ycenter = data['xcenter'], data['ycenter']
+        return torch.tensor([
+            data['label'],
+            xcenter - (data['width'] / 2),
+            ycenter - (data['height'] / 2),
+            data['width'],
+            data['height'],
+        ], device=device)
+
+    elif 'height' in data and 'width' in data:
+        if image_width is not None and image_height is not None:
+            xmin, ymin = data['xmin'] * image_width, data['ymin'] * image_height
+        else:
+            xmin, ymin = data['xmin'], data['ymin']
+        return torch.tensor([
+            data['label'],
+            xmin,
+            ymin,
+            data['width'],
+            data['height'],
+        ], device=device)
+
+    else:
+        if image_width is not None and image_height is not None:
+            xmin, ymin = data['xmin'] * image_width, data['ymin'] * image_height
+            xmax, ymax = data['xmax'] * image_width, data['ymax'] * image_height
+        else:
+            xmin, ymin = data['xmin'], data['ymin']
+            xmax, ymax = data['xmax'], data['ymax']
+        return torch.tensor([
+            data['label'],
+            xmin,
+            ymin,
+            xmax - xmin,
+            ymax - ymin,
+        ], device=device)


Test patch:
diff --git a/tests/vision/utils_tests/formatters_test.py b/tests/vision/utils_tests/formatters_test.py
--- a/tests/vision/utils_tests/formatters_test.py
+++ b/tests/vision/utils_tests/formatters_test.py
@@ -9,19 +9,24 @@
 # ----------------------------------------------------------------------------
 #
 from typing import Union
+from itertools import permutations
 
 import numpy as np
+import torch
 from hamcrest import assert_that, equal_to, calling, raises, close_to
 from torch.utils.data import DataLoader, Dataset
 
-from deepchecks.vision.utils import ClassificationLabelFormatter
 from deepchecks.core.errors import DeepchecksValueError
+from deepchecks.vision.datasets.detection import coco
+from deepchecks.vision.utils import ClassificationLabelFormatter
+from deepchecks.vision.utils import DetectionLabelFormatter
+from deepchecks.vision.utils.detection_formatters import verify_bbox_format_notation
+from deepchecks.vision.utils.detection_formatters import convert_bbox
+from deepchecks.vision.utils.detection_formatters import convert_batch_of_bboxes
 from deepchecks.vision.utils.image_formatters import ImageFormatter
-# pylint: disable=wildcard-import,redefined-outer-name,unused-wildcard-import
-from tests.vision.vision_conftest import *
 
 
-def numpy_shape_dataloader(shape: tuple = None, value: Union[float, np.array] = 255, collate_fn=None):
+def numpy_shape_dataloader(shape: tuple = None, value: Union[float, np.ndarray] = 255, collate_fn=None):
     if collate_fn is None:
         collate_fn = np.stack
 
@@ -277,3 +282,266 @@ def test_flatten_batch_with_sampling_larger_than_num_pixels():
     res = formatter._flatten_batch(batch)  # pylint: disable=protected-access
 
     assert_that(np.array_equal(res[0], expected_result), equal_to(True))
+
+
+def test_allowed_bbox_format_notations():
+    notations = (
+        '  lxyxy       ',
+        'LxywH',
+        *[''.join(it) for it in set(permutations(['l', 'xy', 'xy'], 3))],
+        *['n'+''.join(it) for it in set(permutations(['l', 'xy', 'xy'], 3))],
+        *[''.join(it) for it in permutations(['l', 'xy', 'wh'], 3)],
+        *[''.join(it) for it in permutations(['l', 'xy', 'wh'], 3)],
+        *[''.join(it) for it in permutations(['l', 'cxcy', 'wh'], 3)],
+        *[''.join(it)+'n' for it in permutations(['l', 'cxcy', 'wh'], 3)]
+    )
+
+    tokens = {
+        'label',
+        'width',
+        'height',
+        'xmin',
+        'ymin',
+        'xmax',
+        'ymax',
+        'xcenter',
+        'ycenter',
+    }
+
+    for notation in notations:
+        are_coordinates_normalized, tokens = verify_bbox_format_notation(notation)
+        assert_that(len(set(tokens).difference(tokens)), equal_to(0))
+        if 'n' in notation:
+            assert_that(are_coordinates_normalized is True)
+
+
+def test_bbox_format_notations_with_forbidden_combination_of_elements():
+    notations = [
+        'l',
+        'xy',
+        'wh',
+        'cxcy',
+        *[''.join(it) for it in permutations(['l', 'xy', 'wh'], 2)],
+        *[''.join(it) for it in permutations(['l', 'xy', 'cxcy'], 3)],
+        *[''.join(it) for it in permutations(['l', 'xy', 'cxcy'], 2)],
+        *[''.join(it) for it in permutations(['wh', 'cxcy', 'xy'], 2)],
+        *[''.join(it) for it in permutations(['wh', 'cxcy', 'xy'], 3)],
+    ]
+
+    for n in notations:
+        assert_that(
+            calling(verify_bbox_format_notation).with_args(n),
+            raises(
+                ValueError,
+                fr'Incorrect bbox format notation - {n}\.\n'
+                r'Only next combinations of elements are allowed:.*\n'
+            )
+        )
+
+
+def test_bbox_format_notations_with_unknown_elements():
+    notations = [
+        'lxyah',
+        'xyxyhw',
+        'cxywhl',
+        'l xy xy',
+        'lxxyy'
+    ]
+    for n in notations:
+        assert_that(
+            calling(verify_bbox_format_notation).with_args(n),
+            raises(
+                ValueError,
+                rf'Wrong bbox format notation - {n}\. '
+                r'Incorrect or unknown sequence of charecters starting from position.*'
+            )
+        )
+
+def test_bbox_format_notation_with_coord_normalization_element_at_wrong_position():
+    notations = [
+        'lnxyxy',
+        'lxynxy',
+        'whnxyl',
+        'lcxcynwh',
+    ]
+
+    for notation in notations:
+        assert_that(
+            calling(verify_bbox_format_notation).with_args(notation),
+            raises(
+                ValueError,
+                rf'Wrong bbox format notation - {notation}\. '
+                r'Incorrect or unknown sequence of charecters starting from position.*'
+            )
+        )
+
+
+def test_batch_of_bboxes_convertion():
+    # Arrange
+    loader = coco.load_dataset()
+    _, input_bboxes = batch = loader.dataset[9]  # it should be always the same sample
+
+    # Act
+    output_bboxes = convert_batch_of_bboxes([batch], 'xywhl')[0]
+
+    # Assert
+    assert_that(len(output_bboxes) == len(input_bboxes))
+
+    for in_bbox, out_bbox in zip(input_bboxes, output_bboxes):
+        assert_that(
+            (out_bbox == torch.tensor([in_bbox[-1], *in_bbox[:-1]])).all(),
+            f'Input bbox: {in_bbox}; Output bbox: {out_bbox}'
+        )
+
+
+def test_batch_of_bboxes_convertion_with_normalized_coordinates():
+    # Arrange
+    loader = coco.load_dataset()
+    image, input_bboxes = loader.dataset[9] # it should be always the same sample
+
+    normilized_input_bboxes = torch.stack([
+        torch.tensor([
+            bbox[0] / image.width,   # x
+            bbox[1] / image.height,  # y
+            bbox[2],  # w
+            bbox[3],  # h
+            bbox[4],  # l
+        ])
+        for bbox in input_bboxes
+    ], dim=0)
+
+    # Act
+    output_bboxes = convert_batch_of_bboxes(
+        [(np.array(image), normilized_input_bboxes)],
+        'nxywhl'
+    )[0]
+
+    # Assert
+    assert_that(len(normilized_input_bboxes) == len(input_bboxes))
+
+    for index, output_bbox in enumerate(output_bboxes):
+        assert_that(
+            (output_bbox == torch.tensor([input_bboxes[index][-1], *input_bboxes[index][:-1]])).all(),
+            f'Original bbox: {input_bboxes[index]}; '
+            f'Normalized bbox: {normilized_input_bboxes[index]}; '
+            f'Output bbox: {output_bbox}'
+        )
+
+
+def test_label_formatter_with_bbox_notation():
+    loader = coco.load_dataset()
+    label_formatter = DetectionLabelFormatter('xywhl')
+
+    for batch in loader:
+        _, bboxes = batch
+
+        # Act
+        transformed_bboxes = label_formatter(batch)
+
+        # Assert
+        assert len(bboxes) == len(transformed_bboxes)
+
+        for input_bboxes, output_bboxes in zip(bboxes, transformed_bboxes):
+            assert_that(input_bboxes.shape == output_bboxes.shape)
+            for input_bbox, output_bbox in zip(input_bboxes, output_bboxes):
+                assert_that(
+                    (output_bbox == torch.tensor([input_bbox[-1], *input_bbox[:-1]])).all(),
+                    f'Input bbox: {input_bbox}; Output bbox: {output_bbox}'
+                )
+
+
+def test_bbox_convertion_to_the_required_format():
+    data = (
+        (
+            dict(notation='xylxy', bbox=torch.tensor([20, 15, 2, 41, 23])), # input
+            torch.tensor([2, 20, 15, 21, 8]), # expected result
+        ),
+        (
+            dict(notation='cxcywhl', bbox=torch.tensor([50, 55, 100, 100, 0])), # input
+            torch.tensor([0, (50 - 100 / 2), (55 - 100 / 2), 100, 100]), # expected result
+        ),
+        (
+            dict(notation='whxyl', bbox=torch.tensor([35, 70, 10, 15, 1])), # input
+            torch.tensor([1, 10, 15, 35, 70,]), # expected result
+        ),
+        (
+            dict( # input
+                notation='nxywhl',
+                bbox=torch.tensor([0.20, 0.20, 20, 40, 1]),
+                image_width=100,
+                image_height=100,
+            ),
+            torch.tensor([1, 20, 20, 20, 40,]), # expected result
+        ),
+        (
+            dict( # input
+                notation='cxcylwhn',
+                bbox=torch.tensor([0.12, 0.17, 0, 50, 100]),
+                image_width=600,
+                image_height=1200,
+            ),
+            torch.tensor([0, 47, 154.00000000000003, 50, 100,]), # expected result
+        )
+    )
+
+    for args, expected_result in data:
+        result = convert_bbox(**args)
+        assert_that(
+            (result == expected_result).all(),
+            f'Arguments: {args}, Result: {result}'
+        )
+
+
+def test_convert_bbox_function_with_ambiguous_combination_of_parameters():
+    image_width, image_height = 100, 100
+    bbox = torch.tensor([35, 70, 10, 15, 1])
+    notation = 'whxyl'
+
+    # format notation indicates that coordinates are not normalized
+    # but image width and height parameters were passed
+    assert_that(
+        calling(convert_bbox).with_args(
+            bbox=bbox, notation=notation,
+            image_width=image_width, image_height=image_height
+        ),
+        raises(
+            ValueError,
+            r'bbox format notation indicates that coordinates of the bbox '
+            r'are not normalized but \'image_height\' and \'image_width\' were provided. '
+            r'Those parameters are redundant in the case when bbox coordinates are not '
+            r'normalized\. Please remove those parameters or add \'n\' element to the format '
+            r'notation to indicate that coordinates are indeed normalized\.'
+        )
+    )
+
+    normalized_bbox = torch.tensor([35, 70, 0.10, 0.15, 1])
+    normalized_notation = 'whxyln'
+
+    # opposite situation
+    # format notation indicates that coordinates are normalized
+    # but image width and height parameters were not provided
+    assert_that(
+        calling(convert_bbox).with_args(
+            bbox=normalized_bbox, notation=normalized_notation
+        ),
+        raises(
+            ValueError,
+            r'bbox format notation indicates that coordinates of the bbox '
+            r'are normalized but \'image_height\' and \'image_width\' parameters '
+            r'were not provided\. Please pass image height and width parameters '
+            r'or remove \'n\' element from the format notation\.'
+        )
+    )
+
+    # verify that in other cases function does not raise an error
+
+    convert_bbox(
+        bbox=bbox,
+        notation=notation,
+    )
+    convert_bbox(
+        bbox=normalized_bbox,
+        notation=normalized_notation,
+        image_width=image_width,
+        image_height=image_height
+    )