Repository: deepchecks/deepchecks

Issue:
Robustness check
Input: dataset & model

Check:
Add corruption (eg. short noise, impulse noise, rotation) to images and compare how much the model predictions changes before & after. Also, compare the model performance.

Output:
Value: dict[corruption, scorer(data)-scorer(corrupt_data)]
Display: 
Table with summary of corruptions and decrease in metric
For each corruption, show 2 sets of images - one where the model was confused by the corruption and one where it was successful.




Discussion:
Remaining tasks:
- [x] Add imgaug augmentations- resolved by #869 
- [x] Add to the notebook an explanation of the requirements to run this check - resolved by #913 
- [x] Fix randomity issues in tests - resolved by #905  
- [ ] Improve the display
Specific ideas for improving the display by @Eranpaz:
1. In the first plot (compare original performance to augmented performance) - it's not clear that the % is "decline compared to origin" and not compared to "perfect performance". Should make it more visually clear or write it.
2. Least affected class is also interesting to know (especially when running on Training data)
3. The name of the augmentation is grouped together with the last plot of the previous augmentation, so the scrolling doesn't work well (see e.g. https://docs.deepchecks.com/en/latest/examples/vision/guides/quickstart_in_5_minutes.html)
4. Show images with most severe degradation. 

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/robustness_report.py b/deepchecks/vision/checks/performance/robustness_report.py
--- a/deepchecks/vision/checks/performance/robustness_report.py
+++ b/deepchecks/vision/checks/performance/robustness_report.py
@@ -29,9 +29,8 @@
 from deepchecks.vision.metrics_utils import calculate_metrics, metric_results_to_df
 from deepchecks.vision.utils.validation import set_seeds
 from deepchecks.vision.metrics_utils import get_scorers_list
-from deepchecks.utils.strings import format_percent
-from deepchecks.vision.utils.image_functions import numpy_to_image_figure, label_bbox_add_to_figure, ImageInfo
-
+from deepchecks.utils.strings import format_percent, split_camel_case
+from deepchecks.vision.utils.image_functions import ImageInfo, numpy_to_html_image
 
 __all__ = ['RobustnessReport']
 
@@ -40,7 +39,7 @@
 
 
 class RobustnessReport(SingleDatasetCheck):
-    """Check several image enhancements for model robustness.
+    """Compare performance of model on original dataset and augmented dataset.
 
     Parameters
     ----------
@@ -101,7 +100,6 @@ def compute(self, context: Context, dataset_kind) -> CheckResult:
         augmentations = self.augmentations or transforms_handler.get_robustness_augmentations(dataset.data_dimension)
         aug_all_data = {}
         for augmentation_func in augmentations:
-            augmentation = augmentation_name(augmentation_func)
             aug_dataset = self._create_augmented_dataset(dataset, augmentation_func, context.random_state)
             # The metrics have saved state, but they are reset inside `calculate_metrics`
             metrics = self._state['metrics']
@@ -114,9 +112,9 @@ def compute(self, context: Context, dataset_kind) -> CheckResult:
             metrics_diff_dict = self._calc_performance_diff(base_mean_results, aug_results)
             # Return dict of metric to list {metric: [{'class': x, 'value': y, 'diff': z, 'samples': w}, ...], ...}
             top_affected_classes = self._calc_top_affected_classes(base_results, aug_results, dataset, 5)
-            # Return list of [(base image, augmented image, class), ...]
+            # Return list of [(base image, augmented image, class, [bbox,...]), ...]
             image_pairs = get_random_image_pairs_from_dataset(dataset, aug_dataset, top_affected_classes)
-            aug_all_data[augmentation] = {
+            aug_all_data[augmentation_name(augmentation_func)] = {
                 'metrics': aug_results,
                 'metrics_diff': metrics_diff_dict,
                 'top_affected': top_affected_classes,
@@ -124,7 +122,10 @@ def compute(self, context: Context, dataset_kind) -> CheckResult:
             }
 
         # Create figures to display
-        figures = self._create_augmentation_figure(dataset, base_mean_results, aug_all_data)
+        aug_names = ', '.join([augmentation_name(aug) for aug in augmentations])
+        info_message = 'Percentage shown are difference between the metric before augmentation and after.<br>' \
+                       f'Augmentations used (separately): {aug_names}'
+        figures = self._create_augmentation_figures(dataset, base_mean_results, aug_all_data)
 
         # Save as result only the metrics diff per augmentation
         result = {aug: data['metrics_diff'] for aug, data in aug_all_data.items()}
@@ -132,7 +133,7 @@ def compute(self, context: Context, dataset_kind) -> CheckResult:
         return CheckResult(
             result,
             header='Robustness Report',
-            display=figures
+            display=[info_message, *figures]
         )
 
     def add_condition_degradation_not_greater_than(self, ratio: float = 0.02):
@@ -235,67 +236,44 @@ def _calc_median_metrics(self, metrics_df) -> dict:
         metrics_df = metrics_df[['Metric', 'Value']].groupby(['Metric']).median()
         return metrics_df.to_dict()['Value']
 
-    def _create_augmentation_figure(self, dataset, base_mean_results, aug_all_data):
+    def _create_augmentation_figures(self, dataset, base_mean_results, aug_all_data):
         figures = []
-        # Iterate augmentation names
-        for augmentation, curr_data in aug_all_data.items():
-            figures.append(f'<h3>Augmentation: {augmentation}</h3>')
+
+        def sort_by_worst_func(aug_data):
+            return sum([m['score'] for m in aug_data[1]['metrics_diff'].values()])
+
+        sorted_by_worst = dict(sorted(aug_all_data.items(), key=sort_by_worst_func))
+
+        # Iterate augmentations
+        for index, (augmentation, curr_data) in enumerate(sorted_by_worst.items()):
+            # Create example figures, return first n_pictures_to_show from original and then n_pictures_to_show from
+            # augmented dataset
+            figures.append(self._create_example_figure(dataset, curr_data['images'], augmentation))
             # Create performance graph
             figures.append(self._create_performance_graph(base_mean_results, curr_data['metrics_diff']))
             # Create top affected graph
             figures.append(self._create_top_affected_graph(curr_data['top_affected'], dataset))
-            # Create example figures, return first n_pictures_to_show from original and then n_pictures_to_show from
-            # augmented dataset
-            figures.append(self._create_example_figure(dataset, curr_data['images']))
-            figures.append('<br>')
-
+            if index < len(aug_all_data) - 1:
+                figures.append('<hr style="background-color:#2a3f5f; height:5px">')
         return figures
 
-    def _create_example_figure(self, dataset: VisionData, images):
-        # Create tuple of ([base images], [aug images], [classes], <[bboxes]>)
-        transposed = list(zip(*images))
-        base_images = transposed[0]
-        aug_images = transposed[1]
-        classes = list(map(dataset.label_id_to_name, transposed[2]))
-
-        # Create image figures
-        fig = make_subplots(rows=2, cols=len(classes), column_titles=classes, row_titles=['Origin', 'Augmented'],
-                            vertical_spacing=0.01, horizontal_spacing=0.01)
-
-        # The width is accumulated and the height is taken by the max image
-        images_width = 0
-        max_height = 0
-        for index, (base_image, aug_image) in enumerate(zip(base_images, aug_images)):
-            # Add image figures
-            fig.append_trace(numpy_to_image_figure(base_image), row=1, col=index + 1)
-            fig.append_trace(numpy_to_image_figure(aug_image), row=2, col=index + 1)
-            # Update sizes
-            img_width, img_height = ImageInfo(base_image).get_size()
-            images_width += img_width
-            max_height = max(max_height, img_height)
-
-        # Add 10 to space between image columns and another 20 for titles
-        width = images_width + 10 * len(base_images) + 20
-        # We have fixed 2 rows, and add 60 for titles space
-        height = max_height * 2 + 60
-        # Set minimum sizes in case of very small images
-        height = max(height, 400)
-        width = max(width, 800)
-        # Font size is annoying and not fixed, but relative to image size, so set the base font relative to height
-        base_font_size = height / 40
-
-        # If length is 4 means we also have bounding boxes to draw
-        if len(transposed) == 4:
-            for index, (base_bbox, aug_bbox) in enumerate(transposed[3]):
-                label_bbox_add_to_figure(base_bbox, fig, row=1, col=index + 1)
-                label_bbox_add_to_figure(aug_bbox, fig, row=2, col=index + 1)
-
-        (fig.update_layout(title=dict(text='Augmentation Samples', font=dict(size=base_font_size * 2)))
-         .update_yaxes(showticklabels=False, visible=True, fixedrange=True, automargin=True)
-         .update_xaxes(showticklabels=False, visible=True, fixedrange=True, automargin=True)
-         .update_annotations(font_size=base_font_size * 1.5))
-
-        return fig.to_image('svg', width=width, height=height).decode('utf-8')
+    def _create_example_figure(self, dataset: VisionData, images, aug_name):
+        html_classes = ''
+        html_base_images = ''
+        html_aug_images = ''
+
+        for sample in images:
+            base_image = sample[0]
+            aug_image = sample[1]
+            class_name = dataset.label_id_to_name(sample[2])
+            bboxes = sample[3] if len(sample) == 4 else (None, None)
+
+            html_base_images += f'<div class="item image-div">{numpy_to_html_image(base_image, labels=bboxes[0])}</div>'
+            html_aug_images += f'<div class="item image-div">{numpy_to_html_image(aug_image, labels=bboxes[1])}</div>'
+            html_classes += f'<h4 class="item class-div">{class_name}</h4>'
+
+        return HTML_TEMPLATE.format(class_names=html_classes, base_images=html_base_images, aug_images=html_aug_images,
+                                    aug_name=aug_name)
 
     def _create_performance_graph(self, base_scores: dict, augmented_scores: dict):
         metrics = sorted(list(base_scores.keys()))
@@ -310,8 +288,9 @@ def _create_performance_graph(self, base_scores: dict, augmented_scores: dict):
             fig.add_trace(go.Bar(x=x, y=y, customdata=diff, texttemplate='%{customdata}',
                                  textposition='auto'), col=index + 1, row=1)
 
+        title = 'Performance Comparison'
         (fig.update_layout(font=dict(size=12), height=300, width=400 * len(metrics), autosize=False,
-                           title=dict(text='Performance Comparison', font=dict(size=20)),
+                           title=dict(text=title, font=dict(size=20)), margin=dict(l=0, b=0),
                            showlegend=False)
          .update_xaxes(title=None, type='category', tickangle=30))
         return fig
@@ -333,12 +312,14 @@ def _create_top_affected_graph(self, top_affected_dict, dataset):
 
             # Plotly have a bug that if all y values are zero text position 'auto' doesn't work
             textposition = 'outside' if sum(y) == 0 else 'auto'
+            hover = 'Metric value: %{y:.2f}<br>Number of samples: %{customdata[1]}'
             fig.add_trace(go.Bar(name=metric, x=x, y=y, customdata=custom_data, texttemplate='%{customdata[0]}',
-                                 textposition=textposition, hovertemplate='Number of samples: %{customdata[1]}'),
+                                 textposition=textposition, hovertemplate=hover),
                           row=1, col=index + 1)
 
+        title = 'Top Affected Classes'
         (fig.update_layout(font=dict(size=12), height=300, width=600 * len(metrics),
-                           title=dict(text='Top Affected Classes', font=dict(size=20)),
+                           title=dict(text=title, font=dict(size=20)), margin=dict(l=0, b=0),
                            showlegend=False)
          .update_xaxes(title=None, type='category', tickangle=30, tickprefix='Class ', automargin=True)
          .update_yaxes(automargin=True))
@@ -348,12 +329,14 @@ def _create_top_affected_graph(self, top_affected_dict, dataset):
 
 def augmentation_name(aug):
     if isinstance(aug, imgaug.augmenters.Augmenter):
-        return aug.name
+        name = aug.name
     elif isinstance(aug, albumentations.BasicTransform):
-        return aug.get_class_fullname()
+        name = aug.get_class_fullname()
     else:
         raise DeepchecksValueError(f'Unsupported augmentation type {type(aug)}')
 
+    return split_camel_case(name)
+
 
 def get_random_image_pairs_from_dataset(original_dataset: VisionData,
                                         augmented_dataset: VisionData,
@@ -419,3 +402,56 @@ def get_random_image_pairs_from_dataset(original_dataset: VisionData,
 
     # Sort by diff but return only the tuple
     return [s for s, _ in sorted(zip(samples, sort_value), key=lambda pair: pair[1])]
+
+
+HTML_TEMPLATE = """
+<style>
+    .container {{
+        overflow-x: auto;
+        display: flex;
+        flex-direction: column;
+    }}
+
+    .row {{
+      display: flex;
+      flex-direction: row;
+      align-items: center;
+    }}
+
+    .item {{
+      flex: 1;
+      min-width: 200px;
+      position: relative;
+      word-wrap: break-word;
+    }}
+
+    .image-div {{
+      min-height: 200px;
+    }}
+
+    .class-div {{
+      text-align: center;
+    }}
+
+    h4 {{
+        font-family: "Open Sans", verdana, arial, sans-serif;
+        color: #2a3f5f
+    }}
+
+</style>
+<h4><b>Augmentation "{aug_name}"</b></h4>
+<div class="container">
+    <div class="row">
+        <h4 class="item">Class</h4>
+        {class_names}
+    </div>
+    <div class="row">
+        <h4 class="item">Base Image</h4>
+        {base_images}
+    </div>
+    <div class="row">
+        <h4 class="item">Augmented Image</h4>
+        {aug_images}
+    </div>
+</div>
+"""
diff --git a/deepchecks/vision/utils/image_functions.py b/deepchecks/vision/utils/image_functions.py
--- a/deepchecks/vision/utils/image_functions.py
+++ b/deepchecks/vision/utils/image_functions.py
@@ -9,6 +9,7 @@
 # ----------------------------------------------------------------------------
 #
 """Module for defining functions related to image data."""
+import base64
 from typing import Tuple
 
 import cv2
@@ -20,7 +21,7 @@
 
 
 __all__ = ['ImageInfo', 'numpy_to_image_figure', 'label_bbox_add_to_figure', 'numpy_grayscale_to_heatmap_figure',
-           'apply_heatmap_image_properties']
+           'apply_heatmap_image_properties', 'numpy_to_html_image']
 
 
 class ImageInfo:
@@ -55,6 +56,35 @@ def numpy_to_image_figure(data: np.ndarray):
     return go.Image(z=data, hoverinfo='skip')
 
 
+def numpy_to_html_image(data: np.ndarray, labels=None):
+    """Use plotly to create PNG image out of numpy data.
+
+    Returns
+    ------
+    str
+        HTML img tag with the embedded picture
+    """
+    dimension = data.shape[2]
+    if dimension == 1:
+        fig = go.Figure(go.Heatmap(z=data.squeeze(), colorscale='gray', hoverinfo='skip'))
+        apply_heatmap_image_properties(fig)
+        fig.update_traces(showscale=False)
+    elif dimension == 3:
+        fig = go.Figure(go.Image(z=data, hoverinfo='skip'))
+    else:
+        raise DeepchecksValueError(f'Don\'t know to plot images with {dimension} dimensions')
+
+    if labels:
+        label_bbox_add_to_figure(labels, fig)
+
+    fig.update_yaxes(showticklabels=False, visible=True, fixedrange=True, automargin=True)
+    fig.update_xaxes(showticklabels=False, visible=True, fixedrange=True, automargin=True)
+    fig.update_layout(margin={'l': 0, 'r': 0, 't': 0, 'b': 0})
+    png = base64.b64encode(fig.to_image('png')).decode('ascii')
+    style = 'position: absolute;margin: auto;top: 0;bottom: 0;left: 0; right: 0;max-width:100%;max-height:100%;'
+    return f'<img style="{style}" src="data:image/png;base64, {png}">'
+
+
 def numpy_grayscale_to_heatmap_figure(data: np.ndarray):
     """Create heatmap graph object from given numpy array data."""
     dimension = data.shape[2]
@@ -71,10 +101,10 @@ def apply_heatmap_image_properties(fig):
     fig.update_xaxes(constrain='domain')
 
 
-def label_bbox_add_to_figure(label: torch.Tensor, figure, row=None, col=None, color='red',
+def label_bbox_add_to_figure(labels: torch.Tensor, figure, row=None, col=None, color='red',
                              prediction=False):
     """Add a bounding box label and rectangle to given figure."""
-    for single in label:
+    for single in labels:
         if prediction:
             x, y, w, h, _, clazz = single.tolist()
         else:


Test patch:
diff --git a/tests/vision/checks/performance/robustness_report_test.py b/tests/vision/checks/performance/robustness_report_test.py
--- a/tests/vision/checks/performance/robustness_report_test.py
+++ b/tests/vision/checks/performance/robustness_report_test.py
@@ -42,11 +42,11 @@ def test_mnist(mnist_dataset_train, trained_mnist, device):
                        device=device)
     # Assert
     assert_that(result.value, has_entries({
-        'RandomBrightnessContrast': has_entries({
+        'Random Brightness Contrast': has_entries({
             'Precision': has_entries(score=close_to(0.984, 0.001), diff=close_to(-0.001, 0.001)),
             'Recall': has_entries(score=close_to(0.986, 0.001), diff=close_to(-0.001, 0.001))
         }),
-        'ShiftScaleRotate': has_entries({
+        'Shift Scale Rotate': has_entries({
             'Precision': has_entries(score=close_to(0.799, 0.001), diff=close_to(-0.189, 0.001)),
             'Recall': has_entries(score=close_to(0.783, 0.001), diff=close_to(-0.206, 0.001))
         }),
@@ -72,7 +72,7 @@ def test_coco_and_condition(coco_train_visiondata, trained_yolov5_object_detecti
                        device=device)
     # Assert
     assert_that(result.value, has_entries({
-        'HueSaturationValue': has_entries({
+        'Hue Saturation Value': has_entries({
             'AP': has_entries(score=close_to(0.308, 0.001), diff=close_to(-0.051, 0.001)),
             'AR': has_entries(score=close_to(0.344, 0.001), diff=close_to(-0.060, 0.001))
         }),
@@ -85,7 +85,7 @@ def test_coco_and_condition(coco_train_visiondata, trained_yolov5_object_detecti
         equal_condition_result(
             is_pass=False,
             name='Metrics degrade by not more than 1%',
-            details='Augmentations not passing: {\'HueSaturationValue\'}'
+            details='Augmentations not passing: {\'Hue Saturation Value\'}'
         )
     ))