#!/usr/bin/env python # -*- coding:utf-8 -*- # Power by Zongsheng Yue 2021-12-07 21:37:58 import sys import math import torch import numpy as np import scipy.ndimage as snd from scipy.special import softmax from scipy.interpolate import interp2d import torch.nn.functional as F from . import util_image from ResizeRight.resize_right import resize def modcrop(im, sf): h, w = im.shape[:2] h -= (h % sf) w -= (w % sf) return im[:h, :w,] #--------------------------------------------Kernel----------------------------------------------- def sigma2kernel(sigma, k_size=21, sf=3, shift=False): ''' Generate Gaussian kernel according to cholesky decomposion. Input: sigma: N x 1 x 2 x 2 torch tensor, covariance matrix k_size: integer, kernel size sf: scale factor Output: kernel: N x 1 x k x k torch tensor ''' try: sigma_inv = torch.inverse(sigma) except: sigma_disturb = sigma + torch.eye(2, dtype=sigma.dtype, device=sigma.device).unsqueeze(0).unsqueeze(0) * 1e-5 sigma_inv = torch.inverse(sigma_disturb) # Set expectation position (shifting kernel for aligned image) if shift: center = k_size // 2 + 0.5 * (sf - k_size % 2) # + 0.5 * (sf - k_size % 2) else: center = k_size // 2 # Create meshgrid for Gaussian X, Y = torch.meshgrid(torch.arange(k_size), torch.arange(k_size)) Z = torch.stack((X, Y), dim=2).to(device=sigma.device, dtype=sigma.dtype).view(1, -1, 2, 1) # 1 x k^2 x 2 x 1 # Calcualte Gaussian for every pixel of the kernel ZZ = Z - center # 1 x k^2 x 2 x 1 ZZ_t = ZZ.permute(0, 1, 3, 2) # 1 x k^2 x 1 x 2 ZZZ = -0.5 * ZZ_t.matmul(sigma_inv).matmul(ZZ).squeeze(-1).squeeze(-1) # N x k^2 kernel = F.softmax(ZZZ, dim=1) # N x k^2 return kernel.view(-1, 1, k_size, k_size) # N x 1 x k x k def shifted_anisotropic_Gaussian(k_size=21, sf=4, lambda_1=1.2, lambda_2=5., theta=0, shift=True): ''' # modified version of https://github.com/cszn/USRNet/blob/master/utils/utils_sisr.py ''' # set covariance matrix Lam = np.diag([lambda_1, lambda_2]) U = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]]) sigma = U @ Lam @ U.T # 2 x 2 inv_sigma = np.linalg.inv(sigma)[None, None, :, :] # 1 x 1 x 2 x 2 # set expectation position (shifting kernel for aligned image) if shift: center = k_size // 2 + 0.5*(sf - k_size % 2) else: center = k_size // 2 # Create meshgrid for Gaussian X, Y = np.meshgrid(range(k_size), range(k_size)) Z = np.stack([X, Y], 2).astype(np.float32)[:, :, :, None] # k x k x 2 x 1 # Calcualte Gaussian for every pixel of the kernel ZZ = Z - center ZZ_t = ZZ.transpose(0,1,3,2) ZZZ = -0.5 * np.squeeze(ZZ_t @ inv_sigma @ ZZ).reshape([1, -1]) kernel = softmax(ZZZ, axis=1).reshape([k_size, k_size]) # k x k # The convariance of the marginal distributions along x and y axis s1, s2 = sigma[0, 0], sigma[1, 1] # Pearson corrleation coefficient rho = sigma[0, 1] / (math.sqrt(s1) * math.sqrt(s2)) kernel_infos = np.array([s1, s2, rho]) # (3,) return kernel, kernel_infos #------------------------------------------Degradation------------------------------------------- def imconv_np(im, kernel, padding_mode='reflect', correlate=False): ''' Image convolution or correlation. Input: im: h x w x c numpy array kernel: k x k numpy array padding_mode: 'reflect', 'constant' or 'wrap' ''' if kernel.ndim != im.ndim: kernel = kernel[:, :, np.newaxis] if correlate: out = snd.correlate(im, kernel, mode=padding_mode) else: out = snd.convolve(im, kernel, mode=padding_mode) return out def conv_multi_kernel_tensor(im_hr, kernel, sf, downsampler): ''' Degradation model by Pytorch. Input: im_hr: N x c x h x w kernel: N x 1 x k x k sf: scale factor ''' im_hr_pad = F.pad(im_hr, (kernel.shape[-1] // 2,)*4, mode='reflect') im_blur = F.conv3d(im_hr_pad.unsqueeze(0), kernel.unsqueeze(1), groups=im_hr.shape[0]) if downsampler.lower() == 'direct': im_blur = im_blur[0, :, :, ::sf, ::sf] # N x c x ... elif downsampler.lower() == 'bicubic': im_blur = resize(im_blur, scale_factors=1/sf) else: sys.exit('Please input the corrected downsampler: Direct or Bicubic!') return im_blur def tidy_kernel(kernel, expect_size=21): ''' Input: kernel: p x p numpy array ''' k_size = kernel.shape[-1] kernel_new = np.zeros([expect_size, expect_size], dtype=kernel.dtype) if expect_size >= k_size: start_ind = expect_size // 2 - k_size // 2 end_ind = start_ind + k_size kernel_new[start_ind:end_ind, start_ind:end_ind] = kernel elif expect_size < k_size: start_ind = k_size // 2 - expect_size // 2 end_ind = start_ind + expect_size kernel_new = kernel[start_ind:end_ind, start_ind:end_ind] kernel_new /= kernel_new.sum() return kernel_new def shift_pixel(x, sf, upper_left=True): """shift pixel for super-resolution with different scale factors Args: x: WxHxC or WxH sf: scale factor upper_left: shift direction """ h, w = x.shape[:2] shift = (sf-1)*0.5 xv, yv = np.arange(0, w, 1.0), np.arange(0, h, 1.0) if upper_left: x1 = xv + shift y1 = yv + shift else: x1 = xv - shift y1 = yv - shift x1 = np.clip(x1, 0, w-1) y1 = np.clip(y1, 0, h-1) if x.ndim == 2: x = interp2d(xv, yv, x)(x1, y1) if x.ndim == 3: for i in range(x.shape[-1]): x[:, :, i] = interp2d(xv, yv, x[:, :, i])(x1, y1) return x #-----------------------------------------Transform-------------------------------------------- class Bicubic: def __init__(self, scale=0.25): self.scale = scale def __call__(self, im, scale=None, out_shape=None): scale = self.scale if scale is None else scale out = resize(im, scale_factors=scale, out_shape=None) return out