imaginaire/utils/visualization/common.py

# Copyright (C) 2021 NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
#
# This work is made available under the Nvidia Source Code License-NC.
# To view a copy of this license, check out LICENSE.md
import cv2
import numpy as np
import PIL
from PIL import Image
import torch
import torchvision
import os


def save_tensor_image(
        filename, image, minus1to1_normalized=False):
    r"""Convert a 3 dimensional torch tensor to a PIL image with the desired
    width and height.

    Args:
        filename (str): Image filename to be saved to.
        image (3 x W1 x H1 tensor): Image tensor
        minus1to1_normalized (bool): True if the tensor values are in [-1,
        1]. Otherwise, we assume the values are in [0, 1].

    Returns:
        (PIL image): The resulting PIL image.
    """
    if len(image.size()) != 3:
        raise ValueError('Image tensor dimension does not equal = 3.')
    if image.size(0) != 3:
        raise ValueError('Image has more than 3 channels.')
    if minus1to1_normalized:
        # Normalize back to [0, 1]
        image = (image + 1) * 0.5
    dirname = os.path.dirname(filename)
    os.makedirs(dirname, exist_ok=True)
    image_grid = torchvision.utils.make_grid(
        image, nrow=1, padding=0, normalize=False)
    torchvision.utils.save_image(image_grid, filename, nrow=1)
    return


def tensor2pilimage(image, width=None, height=None, minus1to1_normalized=False):
    r"""Convert a 3 dimensional torch tensor to a PIL image with the desired
    width and height.

    Args:
        image (3 x W1 x H1 tensor): Image tensor
        width (int): Desired width for the result PIL image.
        height (int): Desired height for the result PIL image.
        minus1to1_normalized (bool): True if the tensor values are in [-1,
        1]. Otherwise, we assume the values are in [0, 1].

    Returns:
        (PIL image): The resulting PIL image.
    """
    if len(image.size()) != 3:
        raise ValueError('Image tensor dimension does not equal = 3.')
    if image.size(0) != 3:
        raise ValueError('Image has more than 3 channels.')
    if minus1to1_normalized:
        # Normalize back to [0, 1]
        image = (image + 1) * 0.5
    image = image.detach().cpu().squeeze().numpy()
    image = np.transpose(image, (1, 2, 0)) * 255
    output_img = Image.fromarray(np.uint8(image))
    if width is not None and height is not None:
        output_img = output_img.resize((width, height), Image.BICUBIC)
    return output_img


def tensor2im(image_tensor, imtype=np.uint8, normalize=True,
              three_channel_output=True):
    r"""Convert tensor to image.

    Args:
        image_tensor (torch.tensor or list of torch.tensor): If tensor then
            (NxCxHxW) or (NxTxCxHxW) or (CxHxW).
        imtype (np.dtype): Type of output image.
        normalize (bool): Is the input image normalized or not?
            three_channel_output (bool): Should single channel images be made 3
            channel in output?

    Returns:
        (numpy.ndarray, list if case 1, 2 above).
    """
    if image_tensor is None:
        return None
    if isinstance(image_tensor, list):
        return [tensor2im(x, imtype, normalize) for x in image_tensor]
    if image_tensor.dim() == 5 or image_tensor.dim() == 4:
        return [tensor2im(image_tensor[idx], imtype, normalize)
                for idx in range(image_tensor.size(0))]

    if image_tensor.dim() == 3:
        image_numpy = image_tensor.cpu().float().numpy()
        if normalize:
            image_numpy = (np.transpose(
                image_numpy, (1, 2, 0)) + 1) / 2.0 * 255.0
        else:
            image_numpy = np.transpose(image_numpy, (1, 2, 0)) * 255.0
        image_numpy = np.clip(image_numpy, 0, 255)
        if image_numpy.shape[2] == 1 and three_channel_output:
            image_numpy = np.repeat(image_numpy, 3, axis=2)
        elif image_numpy.shape[2] > 3:
            image_numpy = image_numpy[:, :, :3]
        return image_numpy.astype(imtype)


def tensor2label(segmap, n_label=None, imtype=np.uint8,
                 colorize=True, output_normalized_tensor=False):
    r"""Convert segmentation mask tensor to color image.
    Args:
        segmap (tensor) of
        If tensor then (NxCxHxW) or (NxTxCxHxW) or (CxHxW).
        n_label (int): If None, then segmap.size(0).
        imtype (np.dtype): Type of output image.
        colorize (bool): Put colors in.

    Returns:
        (numpy.ndarray or normalized torch image).
    """
    if segmap is None:
        return None
    if isinstance(segmap, list):
        return [tensor2label(x, n_label,
                             imtype, colorize,
                             output_normalized_tensor) for x in segmap]
    if segmap.dim() == 5 or segmap.dim() == 4:
        return [tensor2label(segmap[idx], n_label,
                             imtype, colorize,
                             output_normalized_tensor)
                for idx in range(segmap.size(0))]

    segmap = segmap.float()
    if not output_normalized_tensor:
        segmap = segmap.cpu()
    if n_label is None:
        n_label = segmap.size(0)
    if n_label > 1:
        segmap = segmap.max(0, keepdim=True)[1]

    if output_normalized_tensor:
        if n_label == 0:
            segmap = Colorize(256)(segmap).to('cuda')
        else:
            segmap = Colorize(n_label)(segmap).to('cuda')
        return 2 * (segmap.float() / 255) - 1
    else:
        if colorize:
            segmap = Colorize(n_label)(segmap)
            segmap = np.transpose(segmap.numpy(), (1, 2, 0))
        else:
            segmap = segmap.cpu().numpy()
        return segmap.astype(imtype)


def tensor2flow(tensor, imtype=np.uint8):
    r"""Convert flow tensor to color image.

    Args:
        tensor (tensor) of
        If tensor then (NxCxHxW) or (NxTxCxHxW) or (CxHxW).
        imtype (np.dtype): Type of output image.

    Returns:
        (numpy.ndarray or normalized torch image).
    """
    if tensor is None:
        return None
    if isinstance(tensor, list):
        tensor = [t for t in tensor if t is not None]
        if not tensor:
            return None
        return [tensor2flow(t, imtype) for t in tensor]
    if tensor.dim() == 5 or tensor.dim() == 4:
        return [tensor2flow(tensor[b]) for b in range(tensor.size(0))]

    tensor = tensor.detach().cpu().float().numpy()
    tensor = np.transpose(tensor, (1, 2, 0))

    hsv = np.zeros((tensor.shape[0], tensor.shape[1], 3), dtype=imtype)
    hsv[:, :, 0] = 255
    hsv[:, :, 1] = 255
    mag, ang = cv2.cartToPolar(tensor[..., 0], tensor[..., 1])
    hsv[..., 0] = ang * 180 / np.pi / 2
    hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
    rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
    return rgb


def plot_keypoints(image, keypoints, normalize=True):
    r"""Plot keypoints on image.

    Args:
       image (PIL.Image, or numpy.ndarray, or torch.Tensor): Input image.
       keypoints (np.ndarray or torch.Tensor, Nx2): Keypoint locations.
       normalize (bool): Whether to normalize the image or not.
    """
    if isinstance(image, PIL.Image.Image):
        image = np.array(image)
    if isinstance(image, torch.Tensor):
        image = tensor2im(image, normalize=normalize)
    if isinstance(image, np.ndarray):
        assert image.ndim == 3
        assert image.shape[-1] == 1 or image.shape[-1] == 3
    if isinstance(keypoints, torch.Tensor):
        keypoints = keypoints.cpu().numpy()
    assert keypoints.ndim == 2 and keypoints.shape[1] == 2

    cv2_image = np.ascontiguousarray(image[:, :, ::-1])  # RGB to BGR.
    for idx in range(keypoints.shape[0]):
        keypoint = np.round(keypoints[idx]).astype(np.int)
        cv2_image = cv2.circle(cv2_image, tuple(keypoint),
                               5, (0, 255, 0), -1)
    image = np.ascontiguousarray(cv2_image[:, :, ::-1])
    return image


def labelcolormap(N):
    r"""Create colors for segmentation label ids.

    Args:
        N (int): Number of labels.
    """
    if N == 35:  # GTA/cityscape train
        cmap = np.array([(0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0),
                         (111, 74, 0), (81, 0, 81), (128, 64, 128),
                         (244, 35, 232), (250, 170, 160), (230, 150, 140),
                         (70, 70, 70), (102, 102, 156), (190, 153, 153),
                         (180, 165, 180), (150, 100, 100), (150, 120, 90),
                         (153, 153, 153), (153, 153, 153), (250, 170, 30),
                         (220, 220, 0), (107, 142, 35), (152, 251, 152),
                         (70, 130, 180), (220, 20, 60), (255, 0, 0),
                         (0, 0, 142), (0, 0, 70), (0, 60, 100), (0, 0, 90),
                         (0, 0, 110), (0, 80, 100), (0, 0, 230), (119, 11, 32),
                         (0, 0, 142)],
                        dtype=np.uint8)
    elif N == 20:  # GTA/cityscape eval
        cmap = np.array([(128, 64, 128), (244, 35, 232), (70, 70, 70),
                         (102, 102, 156), (190, 153, 153), (153, 153, 153),
                         (250, 170, 30), (220, 220, 0), (107, 142, 35),
                         (152, 251, 152), (220, 20, 60), (255, 0, 0),
                         (0, 0, 142), (0, 0, 70), (0, 60, 100), (0, 80, 100),
                         (0, 0, 230), (119, 11, 32), (70, 130, 180), (0, 0, 0)],
                        dtype=np.uint8)
    else:
        cmap = np.zeros([N, 3]).astype(np.uint8)
        for i in range(N):
            r, g, b = np.zeros(3)
            for j in range(8):
                r = r + (1 << (7 - j)) * ((i & (1 << (3 * j))) >> (3 * j))
                g = g + (1 << (7 - j)) * \
                    ((i & (1 << (3 * j + 1))) >> (3 * j + 1))
                b = b + (1 << (7 - j)) * \
                    ((i & (1 << (3 * j + 2))) >> (3 * j + 2))
            cmap[i, :] = np.array([r, g, b])
    return cmap


class Colorize(object):
    """Class to colorize segmentation maps."""

    def __init__(self, n=35):
        self.cmap = labelcolormap(n)
        self.cmap = torch.from_numpy(self.cmap[:n])

    def __call__(self, seg_map):
        r"""

        Args:
            seg_map (tensor): Input Segmentation maps to be colorized.
        """
        size = seg_map.size()
        color_image = torch.ByteTensor(3, size[1], size[2]).fill_(0)
        for label in range(0, len(self.cmap)):
            mask = (label == seg_map[0]).cpu()
            color_image[0][mask] = self.cmap[label][0]
            color_image[1][mask] = self.cmap[label][1]
            color_image[2][mask] = self.cmap[label][2]
        return color_image


def plot_keypoints_on_black(resize_h, resize_w, crop_h, crop_w, is_flipped,
                            cfgdata, keypoints):
    r"""Plot keypoints on black image.

    Args:
        resize_h (int): Height to be resized to.
        resize_w (int): Width to be resized to.
        crop_h (int): Height of the cropping.
        crop_w (int): Width of the cropping.
        is_flipped (bool): If image is a flipped version.
        cfgdata (obj): Data configuration object.
        keypoints (np.ndarray): Keypoint locations. Shape of
            (Nx2) or (TxNx2).

    Returns:
        (list of np.ndarray): List of images (output_h, output_w, 3).
    """
    if keypoints.ndim == 2 and keypoints.shape[1] == 2:
        keypoints = keypoints[np.newaxis, ...]

    outputs = []
    for t_idx in range(keypoints.shape[0]):
        cv2_image = np.zeros((crop_h, crop_w, 3)).astype(np.uint8)
        for idx in range(keypoints[t_idx].shape[0]):
            keypoint = np.round(keypoints[t_idx][idx]).astype(np.int)
            cv2_image = cv2.circle(cv2_image, tuple(keypoint),
                                   5, (0, 255, 0), -1)
        image = np.ascontiguousarray(cv2_image[:, :, ::-1])  # BGR to RGB.
        outputs.append(image)

    return outputs