from packaging import version import random import numpy as np from PIL import Image, ImageFilter, ImageOps from torchvision.transforms.transforms import Lambda, Compose from torchvision.transforms import functional as F from collections.abc import Iterable import torch, torchvision import numbers import copy if version.parse(torchvision.__version__) <= version.parse('0.7.0'): from torchvision.transforms.transforms import _get_image_size def check_input_type_perform_action(input, type, action, *args, **kwargs): output = input if isinstance(input, list): for i in range(0, len(input)): if type is None: if input[i] is not None: # do not combine with last line, to avoid calling isinstance on None. output[i] = action(input[i], *args, **kwargs) elif isinstance(input[i], type): output[i] = action(input[i], *args, **kwargs) elif type is None: if input is not None: output = action(input, *args, **kwargs) elif isinstance(input, type): output = action(input, *args, **kwargs) return output """ Most of these functions are imported from torchvision.transforms.transforms and edited to support 2 or more inputs. """ class JointCompose(object): """ Composes several transforms together. """ def __init__(self, transforms): self.transforms = transforms def __call__(self, input1, input2): for t in self.transforms: input1, input2 = t(input1, input2) return input1, input2 class Grayscale(object): def __init__(self, input1_output_channels=1, input2_output_channels=1): self.input1_output_channels = input1_output_channels self.input2_output_channels = input2_output_channels def __call__(self, input1, input2): output1 = F.to_grayscale(input1, num_output_channels=self.input1_output_channels) if self.input1_output_channels == 1 else input1 output2 = check_input_type_perform_action(input2, Image.Image, F.to_grayscale, num_output_channels=self.input2_output_channels) \ if self.input2_output_channels == 1 else input2 return output1, output2 class Resize(object): def __init__(self, size, interpolation=Image.BILINEAR): assert isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2) self.size = size self.interpolation = interpolation def __call__(self, input1, input2): output1 = F.resize(input1, self.size, self.interpolation) output2 = check_input_type_perform_action(input2, Image.Image, F.resize, self.size, self.interpolation) return output1, output2 class ScaleWidth: def __init__(self, target_size, method=Image.BICUBIC): self.target_size = target_size self.method = method def scalewidth(self, img): ow, oh = img.size w = self.target_size h = int(self.target_size * oh / ow) img_resized = img.resize((w, h), self.method) if h > w: # if resized image's height is larger than its width, crop the center left = 0 top = h // 2 - self.target_size // 2 right = self.target_size bottom = top + self.target_size img_resized = img_resized.crop((left, top, right, bottom)) elif h < w: # pad the heights delta_w = self.target_size - w delta_h = self.target_size - h padding = (delta_w // 2, delta_h // 2, delta_w - (delta_w // 2), delta_h - (delta_h // 2)) img_resized = ImageOps.expand(img_resized, padding) return img_resized def __call__(self, input1, input2): output1 = self.scalewidth(input1) output2 = check_input_type_perform_action(input2, Image.Image, self.scalewidth) return output1, output2 class RandomCrop(object): def __init__(self, size, padding=None, pad_if_needed=False, fill=0, padding_mode='constant'): if isinstance(size, numbers.Number): self.size = (int(size), int(size)) else: self.size = size self.padding = padding self.pad_if_needed = pad_if_needed self.fill = fill self.padding_mode = padding_mode @staticmethod def get_params(img, output_size): if version.parse(torchvision.__version__) <= version.parse('0.7.0'): w, h = _get_image_size(img) else: w, h = F._get_image_size(img) th, tw = output_size if w == tw and h == th: return 0, 0, h, w i = random.randint(0, h - th) j = random.randint(0, w - tw) return i, j, th, tw def pad(self, img): if self.padding is not None: img = F.pad(img, self.padding, self.fill, self.padding_mode) # pad the width if needed if self.pad_if_needed and img.size[0] < self.size[1]: img = F.pad(img, (self.size[1] - img.size[0], 0), self.fill, self.padding_mode) # pad the height if needed if self.pad_if_needed and img.size[1] < self.size[0]: img = F.pad(img, (0, self.size[0] - img.size[1]), self.fill, self.padding_mode) return img def get_crop_range(self, img): return self.get_params(img, self.size) def pad_and_crop(self, input, i, j, h, w): return F.crop(self.pad(input), i, j, h, w) def __call__(self, input1, input2): output1 = self.pad(input1) i, j, h, w = self.get_crop_range(output1) output1 = F.crop(output1, i, j, h, w) output2 = check_input_type_perform_action(input2, Image.Image, self.pad_and_crop, i, j, h, w) return output1, output2 class Crop: def __init__(self, pos, size): self.pos = pos self.size = size def crop(self, img): ow, oh = img.size x1, y1 = self.pos tw = th = self.size if (ow > tw or oh > th): return img.crop((x1, y1, x1 + tw, y1 + th)) return img def __call__(self, input1, input2): output1 = self.crop(input1) output2 = check_input_type_perform_action(input2, Image.Image, self.crop) return output1, output2 class ColorJitter(object): def __init__(self, brightness=0, contrast=0, saturation=0, hue=0): self.brightness = self._check_input(brightness, 'brightness') self.contrast = self._check_input(contrast, 'contrast') self.saturation = self._check_input(saturation, 'saturation') self.hue = self._check_input(hue, 'hue', center=0, bound=(-0.5, 0.5), clip_first_on_zero=False) def _check_input(self, value, name, center=1, bound=(0, float('inf')), clip_first_on_zero=True): if isinstance(value, numbers.Number): if value < 0: raise ValueError("If {} is a single number, it must be non negative.".format(name)) value = [center - value, center + value] if clip_first_on_zero: value[0] = max(value[0], 0) elif isinstance(value, (tuple, list)) and len(value) == 2: if not bound[0] <= value[0] <= value[1] <= bound[1]: raise ValueError("{} values should be between {}".format(name, bound)) else: raise TypeError("{} should be a single number or a list/tuple with lenght 2.".format(name)) # if value is 0 or (1., 1.) for brightness/contrast/saturation # or (0., 0.) for hue, do nothing if value[0] == value[1] == center: value = None return value @staticmethod def get_params(brightness, contrast, saturation, hue): transforms = [] if brightness is not None: brightness_factor = random.uniform(brightness[0], brightness[1]) transforms.append(Lambda(lambda img: F.adjust_brightness(img, brightness_factor))) if contrast is not None: contrast_factor = random.uniform(contrast[0], contrast[1]) transforms.append(Lambda(lambda img: F.adjust_contrast(img, contrast_factor))) if saturation is not None: saturation_factor = random.uniform(saturation[0], saturation[1]) transforms.append(Lambda(lambda img: F.adjust_saturation(img, saturation_factor))) if hue is not None: hue_factor = random.uniform(hue[0], hue[1]) transforms.append(Lambda(lambda img: F.adjust_hue(img, hue_factor))) random.shuffle(transforms) transform = Compose(transforms) return transform def __call__(self, input1, input2): transform = self.get_params(self.brightness, self.contrast, self.saturation, self.hue) output1 = transform(input1) output2 = check_input_type_perform_action(input2, Image.Image, transform) return output1, output2 class RandomAffine(object): def __init__(self, degrees, translate=None, scale=None, shear=None, resample=False, fillcolor=0): if isinstance(degrees, numbers.Number): if degrees < 0: raise ValueError("If degrees is a single number, it must be positive.") self.degrees = (-degrees, degrees) else: assert isinstance(degrees, (tuple, list)) and len(degrees) == 2, \ "degrees should be a list or tuple and it must be of length 2." self.degrees = degrees if translate is not None: assert isinstance(translate, (tuple, list)) and len(translate) == 2, \ "translate should be a list or tuple and it must be of length 2." for t in translate: if not (0.0 <= t <= 1.0): raise ValueError("translation values should be between 0 and 1") self.translate = translate if scale is not None: assert isinstance(scale, (tuple, list)) and len(scale) == 2, \ "scale should be a list or tuple and it must be of length 2." for s in scale: if s <= 0: raise ValueError("scale values should be positive") self.scale = scale if shear is not None: if isinstance(shear, numbers.Number): if shear < 0: raise ValueError("If shear is a single number, it must be positive.") self.shear = (-shear, shear) else: assert isinstance(shear, (tuple, list)) and \ (len(shear) == 2 or len(shear) == 4), \ "shear should be a list or tuple and it must be of length 2 or 4." # X-Axis shear with [min, max] if len(shear) == 2: self.shear = [shear[0], shear[1], 0., 0.] elif len(shear) == 4: self.shear = [s for s in shear] else: self.shear = shear self.resample = resample self.fillcolor = fillcolor @staticmethod def get_params(degrees, translate, scale_ranges, shears, img_size): angle = random.uniform(degrees[0], degrees[1]) if translate is not None: max_dx = translate[0] * img_size[0] max_dy = translate[1] * img_size[1] translations = (np.round(random.uniform(-max_dx, max_dx)), np.round(random.uniform(-max_dy, max_dy))) else: translations = (0, 0) if scale_ranges is not None: scale = random.uniform(scale_ranges[0], scale_ranges[1]) else: scale = 1.0 if shears is not None: if len(shears) == 2: shear = [random.uniform(shears[0], shears[1]), 0.] elif len(shears) == 4: shear = [random.uniform(shears[0], shears[1]), random.uniform(shears[2], shears[3])] else: shear = 0.0 return angle, translations, scale, shear def __call__(self, input1, input2): params = self.get_params(self.degrees, self.translate, self.scale, self.shear, input1.size) output1 = F.affine(input1, *params, resample=self.resample, fillcolor=self.fillcolor) output2 = check_input_type_perform_action(input2, Image.Image, F.affine, *params, resample=self.resample, fillcolor=self.fillcolor) return output1, output2 class RandomRotation(object): def __init__(self, degrees, resample=False, expand=False, center=None, fill=None): if isinstance(degrees, numbers.Number): if degrees < 0: raise ValueError("If degrees is a single number, it must be positive.") self.degrees = (-degrees, degrees) else: if len(degrees) != 2: raise ValueError("If degrees is a sequence, it must be of len 2.") self.degrees = degrees self.resample = resample self.expand = expand self.center = center self.fill = fill @staticmethod def get_params(degrees): angle = random.uniform(degrees[0], degrees[1]) return angle def __call__(self, input1, input2): angle = self.get_params(self.degrees) output1 = F.rotate(input1, angle, self.resample, self.expand, self.center, self.fill) output2 = check_input_type_perform_action(input2, Image.Image, F.rotate, angle, self.resample, self.expand, self.center, self.fill) return output1, output2 class RandomBlur: def __init__(self, blur_chance): self.blur_chance = blur_chance def get_params(self): if self.blur_chance > random.random(): kernel = random.randint(3, 12) while kernel % 2 != 1: kernel = random.randint(3, 12) else: kernel = None return kernel def blur(self, image, kernel): image = image.filter(ImageFilter.GaussianBlur(radius=kernel)) return image def __call__(self, input1, input2): kernel = self.get_params() if kernel is None: return input1, input2 else: output1 = self.blur(input1, kernel) output2 = check_input_type_perform_action(input2, Image.Image, self.blur, kernel) return output1, output2 class NoiseTransform: """code is partly from http://www.xiaoliangbai.com/2016/09/09/more-on-image-noise-generation and edited by Oliver.""" def __init__(self, noise_type): self.noise_type = noise_type def get_params(self, image): params = [] image_np = np.array(image) row, col, ch = image_np.shape if random.random() < 0.5: return None if self.noise_type == "gauss": mean = 0.0 std = random.uniform(0.001, 0.3) gauss = np.random.normal(mean, std, (row, col, ch)) gauss = gauss.reshape(row, col, ch) params.append(gauss) return params elif self.noise_type == "s&p": s_vs_p = 0.5 amount = random.uniform(0.001, 0.01) # Generate Salt '1' noise num_salt = np.ceil(amount * image_np.size * s_vs_p) coords = [np.random.randint(0, i - 1, int(num_salt)) for i in image_np.shape] coords[2] = np.random.randint(0, 3, int(num_salt)) params.append(copy.deepcopy(coords)) # Generate Pepper '0' noise num_pepper = np.ceil(amount * image_np.size * (1. - s_vs_p)) coords = [np.random.randint(0, i - 1, int(num_pepper)) for i in image_np.shape] params.append(copy.deepcopy(coords)) return params elif self.noise_type == "poisson": noisy = np.random.poisson(image_np) params.append(noisy) return params elif self.noise_type == "speckle": factor = random.uniform(0.01, 0.4) gauss = np.random.randn(row, col, ch) gauss = gauss.reshape(row, col, ch) * factor params.append(gauss) return params elif self.noise_type == "band": smaller_dim = min(col, row) num_bands = random.randrange(smaller_dim // 2, smaller_dim) scale = random.uniform(1.0, 10.0) offset = np.zeros(image_np.shape).astype(np.float64) # horizontal branding num_list = list(range(image.width)) # list of integers from 0 to image width-1 # adjust this boundaries to fit your needs random.shuffle(num_list) horizontal_bands = num_list[:num_bands] for w in horizontal_bands: offset[w, :, :] += random.uniform(-1, 1) * scale # vertical branding num_list = list(range(image.height)) # list of integers from 0 to image height-1 # adjust this boundaries to fit your needs random.shuffle(num_list) vertical_bands = num_list[:num_bands] for h in vertical_bands: offset[:, h, :] += random.uniform(-1, 1) * scale params.append(offset) return params else: return params def apply(self, image, params): """ image: ndarray (input image data. It will be converted to float) """ if params is None: return image image_np = np.array(image) if self.noise_type == "gauss": gauss = params[0] noisy = image_np + image_np * gauss noisy = np.clip(noisy, 0, 255) return Image.fromarray(noisy.astype('uint8')) elif self.noise_type == "s&p": out = image_np # Generate Salt '1' noise coords = params[0] out[tuple(coords)] = 255 # Generate Pepper '0' noise coords = params[1] out[tuple(coords)] = 0 out = np.clip(out, 0, 255) return Image.fromarray(out.astype('uint8')) elif self.noise_type == "poisson": noisy = params[0] noisy = np.clip(noisy, 0, 255) return Image.fromarray(noisy.astype('uint8')) elif self.noise_type == "speckle": gauss = params[0] noisy = image_np + image_np * gauss noisy = np.clip(noisy, 0, 255) return Image.fromarray(noisy.astype('uint8')) elif self.noise_type == "band": offset = params[0] noisy = image_np + offset noisy = np.clip(noisy, 0, 255) return Image.fromarray(noisy.astype('uint8')) else: return image def __call__(self, input1, input2): params = self.get_params(input1) output1 = self.apply(input1, params) output2 = check_input_type_perform_action(input2, Image.Image, self.apply, params) return output1, output2 class MakePower2: def __init__(self, base, method=Image.BICUBIC): self.base = base self.method = method self.print_size_warning = PrintSizeWarning() def apply(self, img): ow, oh = img.size h = int(round(oh / self.base) * self.base) w = int(round(ow / self.base) * self.base) if h == oh and w == ow: return img self.print_size_warning(ow, oh, w, h) return img.resize((w, h), self.method) def __call__(self, input1, input2): output1 = self.apply(input1) output2 = check_input_type_perform_action(input2, Image.Image, self.apply) return output1, output2 class RandomHorizontalFlip(object): """Horizontally flip the given PIL Image randomly with a given probability. Args: p (float): probability of the image being flipped. Default value is 0.5 """ def __init__(self, p=0.5): self.p = p def get_params(self): if random.random() < self.p: return True else: return False def __call__(self, input1, input2): flip = self.get_params() if flip: output1 = F.hflip(input1) output2 = check_input_type_perform_action(input2, Image.Image, F.hflip) else: output1, output2 = input1, input2 return output1, output2 class Flip: def __init__(self, flip): self.flip = flip def transpose(self, input): return input.transpose(Image.FLIP_LEFT_RIGHT) def __call__(self, input1, input2): if self.flip: output1 = input1.transpose(Image.FLIP_LEFT_RIGHT) output2 = check_input_type_perform_action(input2, Image.Image, self.transpose) else: output1, output2 = input1, input2 return output1, output2 class ToTensor(object): """Convert a ``PIL Image`` or ``numpy.ndarray`` to tensor. Converts a PIL Image or numpy.ndarray (H x W x C) in the range [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0] if the PIL Image belongs to one of the modes (L, LA, P, I, F, RGB, YCbCr, RGBA, CMYK, 1) or if the numpy.ndarray has dtype = np.uint8 In the other cases, tensors are returned without scaling. """ def __call__(self, input1, input2): output1 = F.to_tensor(input1) output2 = check_input_type_perform_action(input2, None, F.to_tensor) return output1, output2 class Normalize(object): """Normalize a tensor image with mean and standard deviation. Given mean: ``(M1,...,Mn)`` and std: ``(S1,..,Sn)`` for ``n`` channels, this transform will normalize each channel of the input ``torch.*Tensor`` i.e. ``output[channel] = (input[channel] - mean[channel]) / std[channel]`` .. note:: This transform acts out of place, i.e., it does not mutate the input tensor. Args: mean (sequence): Sequence of means for each channel. std (sequence): Sequence of standard deviations for each channel. inplace(bool,optional): Bool to make this operation in-place. """ def __init__(self, first_input_mean, first_input_std, second_input_mean=None, second_input_std=None, inplace=False): self.first_input_mean = first_input_mean self.first_input_std = first_input_std self.second_input_mean = second_input_mean if second_input_mean is not None else first_input_mean self.second_input_std = second_input_std if second_input_std is not None else first_input_std self.inplace = inplace def __call__(self, tensor1, tensor2): """ Args: tensor (Tensor): Tensor image of size (C, H, W) to be normalized. Returns: Tensor: Normalized Tensor image. """ output1 = F.normalize(tensor1, self.first_input_mean, self.first_input_std, self.inplace) output2 = check_input_type_perform_action(tensor2, None, F.normalize, self.second_input_mean, self.second_input_std, self.inplace) return output1, output2 class PrintSizeWarning: def __init__(self): self.has_printed = False def __call__(self, ow, oh, w, h): if not self.has_printed: print("The image size needs to be a multiple of 4. " "The loaded image size was (%d, %d), so it was adjusted to " "(%d, %d). This adjustment will be done to all images " "whose sizes are not multiples of 4" % (ow, oh, w, h)) self.has_printed = True class ImagePathToImage: """Convert an image path to an image. Parameters: filename -- the input file path. """ def load_img(self, path): return Image.open(path).convert('RGB') def __call__(self, filename1, filename2): img1 = self.load_img(filename1) img2 = check_input_type_perform_action(filename2, None, self.load_img) return img1, img2 class NumpyToTensor: """Convert a numpy array to a tensor. Parameters: filename -- the input file path. """ def load_numpy(self, filename): npy = np.load(filename) if isinstance(npy, np.lib.npyio.NpzFile): npy = npy['data'] if len(npy.shape) == 2: npy = np.tile(npy, (1, 1, 1)) else: npy = np.transpose(npy, (2, 0, 1)) return torch.from_numpy(npy).float() def __call__(self, filename1, filename2): tensor1 = self.load_numpy(filename1) tensor2 = check_input_type_perform_action(filename2, None, self.load_numpy) return tensor1, tensor2