Repository: deepchecks/deepchecks

Issue:
[BUG] [Vision] improve numpy_to_html_image performance
**Describe the bug**
results from profiling on quick start full suite:
run time | function
-- | --
185.434 | numpy_to_html_image  deepchecks/vision/utils/image_functions.py:59

which is a large portion of the robustness check. (330.995 seconds)

similar result from cprofiler:

calls | cumulative time | time per call | function
-- | --  | -- | -- 
98 |  175.785 | 1.794 | image_functions.py:59(numpy_to_html_image)





**To Reproduce**
Steps to reproduce the behavior:
1. run quick start until before `full_suite.run`, wrap it in a profiler (used cProfile and pyinstrument) and read the result 

**Expected behavior**
Run time should be shorter.

**Environment (please complete the following information):**
 - OS: Mac 
 - Python Version: 3.8
 - Deepchecks Version:  0.5.0-dev2




Discussion:
I do not see a reason why this function is using plotly to process the data/image instead of a more specialized package like Pillow, I think it is the main reason why it is slow/

@matanper  GitLens is saying that you added this function, is there some reason why you used plotly?

TODO:
- use Pillow instead of plotly to process the data/image
In the start, we used a whole figure to with subplots to create the layout, so we needed plotly traces. Then I changed the layout to be html-based and left plotly only the image creation (which we use also to draw labels & prediction bboxes on the image)

If it's possible to draw in Pillow we can use that in robustness but I don't know about other checks which uses the same functions to display heatmaps for example

Resolving patch:
diff --git a/deepchecks/vision/checks/performance/robustness_report.py b/deepchecks/vision/checks/performance/robustness_report.py
--- a/deepchecks/vision/checks/performance/robustness_report.py
+++ b/deepchecks/vision/checks/performance/robustness_report.py
@@ -29,7 +29,7 @@
 from deepchecks.vision.utils.validation import set_seeds
 from deepchecks.vision.metrics_utils import get_scorers_list
 from deepchecks.utils.strings import format_percent, split_camel_case
-from deepchecks.vision.utils.image_functions import ImageInfo, numpy_to_html_image
+from deepchecks.vision.utils.image_functions import ImageInfo, prepare_thumbnail, draw_bboxes
 
 
 __all__ = ['RobustnessReport']
@@ -51,6 +51,8 @@ class RobustnessReport(SingleDatasetCheck):
         Supported augmentations are of albumentations and imgaug.
     """
 
+    _THUMBNAIL_SIZE = (200, 200)
+
     def __init__(self,
                  alternative_metrics: Optional[Dict[str, Metric]] = None,
                  augmentations: List = None):
@@ -247,23 +249,44 @@ def sort_by_worst_func(aug_data):
                 figures.append('<hr style="background-color:#2a3f5f; height:5px">')
         return figures
 
-    def _create_example_figure(self, dataset: VisionData, images, aug_name):
-        html_classes = ''
-        html_base_images = ''
-        html_aug_images = ''
+    def _create_example_figure(self, dataset: VisionData, images, aug_name: str):
+        classes = []
+        base_images = []
+        aug_images = []
 
         for sample in images:
             base_image = sample[0]
             aug_image = sample[1]
             class_name = dataset.label_id_to_name(sample[2])
-            bboxes = sample[3] if len(sample) == 4 else (None, None)
-
-            html_base_images += f'<div class="item image-div">{numpy_to_html_image(base_image, labels=bboxes[0])}</div>'
-            html_aug_images += f'<div class="item image-div">{numpy_to_html_image(aug_image, labels=bboxes[1])}</div>'
-            html_classes += f'<h4 class="item class-div">{class_name}</h4>'
-
-        return HTML_TEMPLATE.format(class_names=html_classes, base_images=html_base_images, aug_images=html_aug_images,
-                                    aug_name=aug_name)
+            classes.append(f'<h4>{class_name}</h4>')
+
+            if len(sample) == 4:
+                base_image_bboxes, aug_image_bboxes = sample[3]
+                base_image = draw_bboxes(base_image, base_image_bboxes, copy_image=True, border_width=2)
+                aug_image = draw_bboxes(aug_image, aug_image_bboxes, copy_image=True, border_width=2)
+
+            base_images.append(prepare_thumbnail(
+                image=base_image,
+                size=self._THUMBNAIL_SIZE,
+                copy_image=False
+            ))
+            aug_images.append(prepare_thumbnail(
+                image=aug_image,
+                size=self._THUMBNAIL_SIZE,
+                copy_image=False
+            ))
+
+        classes = ''.join(classes)
+        base_images_thumbnails = ''.join(base_images)
+        aug_images_thumbnails = ''.join(aug_images)
+
+        return HTML_TEMPLATE.format(
+            aug_name=aug_name,
+            classes=classes,
+            n_of_images=len(base_images),
+            base_images=base_images_thumbnails,
+            aug_images=aug_images_thumbnails,
+        )
 
     def _create_performance_graph(self, base_scores: dict, augmented_scores: dict):
         metrics = sorted(list(base_scores.keys()))
@@ -369,53 +392,23 @@ def get_random_image_pairs_from_dataset(original_dataset: VisionData,
 
 
 HTML_TEMPLATE = """
-<style>
-    .container {{
+<h3><b>Augmentation "{aug_name}"</b></h3>
+<div
+    style="
         overflow-x: auto;
-        display: flex;
-        flex-direction: column;
-    }}
-
-    .row {{
-      display: flex;
-      flex-direction: row;
-      align-items: center;
-    }}
-
-    .item {{
-      flex: 1;
-      min-width: 200px;
-      position: relative;
-      word-wrap: break-word;
-    }}
-
-    .image-div {{
-      min-height: 200px;
-    }}
-
-    .class-div {{
-      text-align: center;
-    }}
-
-    h4 {{
-        font-family: "Open Sans", verdana, arial, sans-serif;
-        color: #2a3f5f
-    }}
-
-</style>
-<h4><b>Augmentation "{aug_name}"</b></h4>
-<div class="container">
-    <div class="row">
-        <h4 class="item">Class</h4>
-        {class_names}
-    </div>
-    <div class="row">
-        <h4 class="item">Base Image</h4>
-        {base_images}
-    </div>
-    <div class="row">
-        <h4 class="item">Augmented Image</h4>
-        {aug_images}
-    </div>
+        display: grid;
+        grid-template-rows: auto 1fr 1fr;
+        grid-template-columns: auto repeat({n_of_images}, 1fr);
+        grid-gap: 1.5rem;
+        justify-items: center;
+        align-items: center;
+        padding: 2rem;
+        width: max-content;">
+    <h4>Class</h4>
+    {classes}
+    <h4>Base Image</h4>
+    {base_images}
+    <h4>Augmented Image</h4>
+    {aug_images}
 </div>
 """
diff --git a/deepchecks/vision/utils/detection_formatters.py b/deepchecks/vision/utils/detection_formatters.py
--- a/deepchecks/vision/utils/detection_formatters.py
+++ b/deepchecks/vision/utils/detection_formatters.py
@@ -119,7 +119,7 @@ def verify_bbox_format_notation(notation: str) -> Tuple[bool, List[str]]:
 
 _BatchOfSamples = Iterable[
     Tuple[
-        Union[Image, np.ndarray, torch.Tensor],  # images
+        Union[Image, np.ndarray, torch.Tensor],  # image
         Sequence[Sequence[Union[int, float]]]  # bboxes
     ]
 ]
@@ -134,7 +134,7 @@ def convert_batch_of_bboxes(
 
     Parameters
     ----------
-    batch : tuple like object with two items - list if images, list of bboxes
+    batch : iterable of tuple like object with two items - image, list of bboxes
         batch of images and bboxes
     notation : str
         bboxes format notation
@@ -190,7 +190,8 @@ def convert_bbox(
     notation: str,
     image_width: Union[int, float, None] = None,
     image_height: Union[int, float, None] = None,
-    device: Union[str, torch.device, None] = None
+    device: Union[str, torch.device, None] = None,
+    _strict: bool = True  # pylint: disable=invalid-name
 ) -> torch.Tensor:
     """Convert bbox to the required format.
 
@@ -232,13 +233,17 @@ def convert_bbox(
         are_coordinates_normalized is False
         and (image_height is not None or image_width is not None)
     ):
-        raise ValueError(
-            'bbox format notation indicates that coordinates of the bbox '
-            'are not normalized but \'image_height\' and \'image_width\' were provided. '
-            'Those parameters are redundant in the case when bbox coordinates are not '
-            'normalized. Please remove those parameters or add \'n\' element to the format '
-            'notation to indicate that coordinates are indeed normalized.'
-        )
+        if _strict is True:
+            raise ValueError(
+                'bbox format notation indicates that coordinates of the bbox '
+                'are not normalized but \'image_height\' and \'image_width\' were provided. '
+                'Those parameters are redundant in the case when bbox coordinates are not '
+                'normalized. Please remove those parameters or add \'n\' element to the format '
+                'notation to indicate that coordinates are indeed normalized.'
+            )
+        else:
+            image_height = None
+            image_width = None
 
     return _convert_bbox(
         bbox,
diff --git a/deepchecks/vision/utils/image_functions.py b/deepchecks/vision/utils/image_functions.py
--- a/deepchecks/vision/utils/image_functions.py
+++ b/deepchecks/vision/utils/image_functions.py
@@ -9,19 +9,24 @@
 # ----------------------------------------------------------------------------
 #
 """Module for defining functions related to image data."""
+import typing as t
+import io
 import base64
-from typing import Tuple
 
 import cv2
 import numpy as np
-import plotly.graph_objects as go
 import torch
+import PIL.Image as pilimage
+import PIL.ImageDraw as pildraw
+import PIL.ImageOps as pilops
+import plotly.graph_objects as go
 
 from deepchecks.core.errors import DeepchecksValueError
+from .detection_formatters import convert_bbox
 
 
 __all__ = ['ImageInfo', 'numpy_to_image_figure', 'label_bbox_add_to_figure', 'numpy_grayscale_to_heatmap_figure',
-           'apply_heatmap_image_properties', 'numpy_to_html_image', 'crop_image']
+           'apply_heatmap_image_properties', 'draw_bboxes', 'prepare_thumbnail', 'crop_image']
 
 
 class ImageInfo:
@@ -32,7 +37,7 @@ def __init__(self, img):
             raise DeepchecksValueError('Expect image to be numpy array')
         self.img = img
 
-    def get_size(self) -> Tuple[int, int]:
+    def get_size(self) -> t.Tuple[int, int]:
         """Get size of image as (width, height) tuple."""
         return self.img.shape[1], self.img.shape[0]
 
@@ -56,33 +61,143 @@ def numpy_to_image_figure(data: np.ndarray):
     return go.Image(z=data, hoverinfo='skip')
 
 
-def numpy_to_html_image(data: np.ndarray, labels=None):
-    """Use plotly to create PNG image out of numpy data.
+def ensure_image(
+    image: t.Union[pilimage.Image, np.ndarray, torch.Tensor],
+    copy: bool = True
+) -> pilimage.Image:
+    """Transform to `PIL.Image.Image` if possible.
+
+    Parameters
+    ----------
+    image : Union[PIL.Image.Image, numpy.ndarray, torch.Tensor]
+    copy : bool, default True
+        if `image` is an instance of the `PIL.Image.Image` return
+        it as it is or copy it.
 
     Returns
-    ------
-    str
-        HTML img tag with the embedded picture
+    -------
+    `PIL.Image.Image`
     """
-    dimension = data.shape[2]
-    if dimension == 1:
-        fig = go.Figure(go.Heatmap(z=data.squeeze(), colorscale='gray', hoverinfo='skip'))
-        apply_heatmap_image_properties(fig)
-        fig.update_traces(showscale=False)
-    elif dimension == 3:
-        fig = go.Figure(go.Image(z=data, hoverinfo='skip'))
+    if isinstance(image, pilimage.Image):
+        return image.copy() if copy is True else image
+    if isinstance(image, torch.Tensor):
+        image = t.cast(np.ndarray, image.numpy())
+    if isinstance(image, np.ndarray):
+        image = image.squeeze()
+        if image.ndim == 3:
+            return pilimage.fromarray(image)
+        elif image.ndim == 2:
+            image = image.astype(np.uint8)
+            return pilops.colorize(
+                pilimage.fromarray(image),
+                black='black',
+                white='white',
+                blackpoint=image.min(),
+                whitepoint=image.max(),
+            )
+        else:
+            raise ValueError(f'Do not know how to work with {image.ndim} dimensional images')
     else:
-        raise DeepchecksValueError(f'Don\'t know to plot images with {dimension} dimensions')
+        raise TypeError(f'cannot convert {type(image)} to the PIL.Image.Image')
+
+
+def draw_bboxes(
+    image: t.Union[pilimage.Image, np.ndarray, torch.Tensor],
+    bboxes: np.ndarray,
+    bbox_notation: t.Optional[str] = None,
+    copy_image: bool = True,
+    border_width: int = 1,
+    color: t.Union[str, t.Dict[np.number, str]] = 'red',
+) -> pilimage.Image:
+    """Draw bboxes on the image.
+
+    Parameters
+    ----------
+    image : Union[PIL.Image.Image, numpy.ndarray, torch.Tensor]
+        image to draw on
+    bboxes : numpy.ndarray
+        array of bboxes
+    bbox_notation : Optional[str], default None
+        format of the provided bboxes
+    copy_image : bool, default True
+        copy imagge before drawing or not
+    border_width : int, default 1
+        width of the bbox outline
+    color: Union[str, Dict[number, str]], default "red"
+        color of the bbox outline. It could be a map mapping class id to the color
 
-    if labels:
-        label_bbox_add_to_figure(labels, fig)
+    Returns
+    -------
+    PIL.Image.Image : image instance with drawen bboxes on it
+    """
+    image = ensure_image(image, copy=copy_image)
+
+    if bbox_notation is not None:
+        bboxes = np.array([
+            convert_bbox(
+                bbox,
+                notation=bbox_notation,
+                image_width=image.width,
+                image_height=image.height,
+                _strict=False
+            ).tolist()
+            for bbox in bboxes
+        ])
+
+    draw = pildraw.ImageDraw(image)
+
+    for bbox in bboxes:
+        clazz, x0, y0, w, h = bbox
+        x1, y1 = x0 + w, y0 + h
+
+        if isinstance(color, str):
+            color_to_use = color
+        elif isinstance(color, dict):
+            color_to_use = color[clazz]
+        else:
+            raise TypeError('color must be of type - Union[str, Dict[int, str]]')
+
+        draw.rectangle(xy=(x0, y0, x1, y1), width=border_width, outline=color_to_use)
+        draw.text(xy=(x0 + (w * 0.5), y0 + (h * 0.2)), text=str(clazz), fill=color_to_use)
+
+    return image
+
+
+def prepare_thumbnail(
+    image: t.Union[pilimage.Image, np.ndarray, torch.Tensor],
+    size: t.Optional[t.Tuple[int, int]] = None,
+    copy_image: bool = True,
+) -> str:
+    """Prepare html image tag with the provided image.
+
+    Parameters
+    ----------
+    image : Union[PIL.Image.Image, numpy.ndarray, torch.Tensor]
+        image to use
+    size : Optional[Tuple[int, int]], default None
+        size to which image should be rescaled
+    copy_image : bool, default True
+        to rescale the image to the provided size this function uses
+        `PIL.Image.Image.thumbnail` method that modified image instance
+        in-place. If `copy_image` is set to True image will be copied
+        before rescaling.
+
+    Returns
+    -------
+    str : html '<img>' tag with embedded image
+    """
+    if size is not None:
+        image = ensure_image(image, copy=copy_image)
+        image.thumbnail(size=size)
+    else:
+        image = ensure_image(image, copy=False)
 
-    fig.update_yaxes(showticklabels=False, visible=True, fixedrange=True, automargin=True)
-    fig.update_xaxes(showticklabels=False, visible=True, fixedrange=True, automargin=True)
-    fig.update_layout(margin={'l': 0, 'r': 0, 't': 0, 'b': 0})
-    png = base64.b64encode(fig.to_image('png')).decode('ascii')
-    style = 'position: absolute;margin: auto;top: 0;bottom: 0;left: 0; right: 0;max-width:100%;max-height:100%;'
-    return f'<img style="{style}" src="data:image/png;base64, {png}">'
+    img_bytes = io.BytesIO()
+    image.save(fp=img_bytes, format='PNG')
+    img_bytes.seek(0)
+    png = base64.b64encode(img_bytes.read()).decode('ascii')
+    img_bytes.close()
+    return f'<img src="data:image/png;base64,{png}"/>'
 
 
 def numpy_grayscale_to_heatmap_figure(data: np.ndarray):
@@ -114,7 +229,7 @@ def label_bbox_add_to_figure(labels: torch.Tensor, figure, row=None, col=None, c
                               font=dict(color=color))
 
 
-def crop_image(img: np.array, x, y, w, h) -> np.array:
+def crop_image(img: np.ndarray, x, y, w, h) -> np.ndarray:
     """Return the cropped numpy array image by x, y, w, h coordinates (top left corner, width and height."""
     # Convert x, y, w, h to integers if not integers already:
     x, y, w, h = [round(n) for n in [x, y, w, h]]


Test patch: