import copy import glob import math import os import random import sys import cv2 import matplotlib.pyplot as plt import numpy as np import scipy.io as sio import torch from imgaug import augmenters as iaa from PIL import Image from scipy import interpolate from skimage import io from skimage import transform as ski_transform from skimage.color import rgb2gray from torch.utils.data import DataLoader from torch.utils.data import Dataset from torchvision import transforms from torchvision import utils from torchvision.transforms import Compose from torchvision.transforms import Lambda from torchvision.transforms.functional import adjust_brightness from torchvision.transforms.functional import adjust_contrast from torchvision.transforms.functional import adjust_hue from torchvision.transforms.functional import adjust_saturation from utils.utils import cv_crop from utils.utils import cv_rotate from utils.utils import draw_gaussian from utils.utils import fig2data from utils.utils import generate_weight_map from utils.utils import power_transform from utils.utils import shuffle_lr from utils.utils import transform class AddBoundary(object): def __init__(self, num_landmarks=68): self.num_landmarks = num_landmarks def __call__(self, sample): landmarks_64 = np.floor(sample["landmarks"] / 4.0) if self.num_landmarks == 68: boundaries = {} boundaries["cheek"] = landmarks_64[0:17] boundaries["left_eyebrow"] = landmarks_64[17:22] boundaries["right_eyebrow"] = landmarks_64[22:27] boundaries["uper_left_eyelid"] = landmarks_64[36:40] boundaries["lower_left_eyelid"] = np.array([landmarks_64[i] for i in [36, 41, 40, 39]]) boundaries["upper_right_eyelid"] = landmarks_64[42:46] boundaries["lower_right_eyelid"] = np.array([landmarks_64[i] for i in [42, 47, 46, 45]]) boundaries["noise"] = landmarks_64[27:31] boundaries["noise_bot"] = landmarks_64[31:36] boundaries["upper_outer_lip"] = landmarks_64[48:55] boundaries["upper_inner_lip"] = np.array([landmarks_64[i] for i in [60, 61, 62, 63, 64]]) boundaries["lower_outer_lip"] = np.array([landmarks_64[i] for i in [48, 59, 58, 57, 56, 55, 54]]) boundaries["lower_inner_lip"] = np.array([landmarks_64[i] for i in [60, 67, 66, 65, 64]]) elif self.num_landmarks == 98: boundaries = {} boundaries["cheek"] = landmarks_64[0:33] boundaries["left_eyebrow"] = landmarks_64[33:38] boundaries["right_eyebrow"] = landmarks_64[42:47] boundaries["uper_left_eyelid"] = landmarks_64[60:65] boundaries["lower_left_eyelid"] = np.array([landmarks_64[i] for i in [60, 67, 66, 65, 64]]) boundaries["upper_right_eyelid"] = landmarks_64[68:73] boundaries["lower_right_eyelid"] = np.array([landmarks_64[i] for i in [68, 75, 74, 73, 72]]) boundaries["noise"] = landmarks_64[51:55] boundaries["noise_bot"] = landmarks_64[55:60] boundaries["upper_outer_lip"] = landmarks_64[76:83] boundaries["upper_inner_lip"] = np.array([landmarks_64[i] for i in [88, 89, 90, 91, 92]]) boundaries["lower_outer_lip"] = np.array([landmarks_64[i] for i in [76, 87, 86, 85, 84, 83, 82]]) boundaries["lower_inner_lip"] = np.array([landmarks_64[i] for i in [88, 95, 94, 93, 92]]) elif self.num_landmarks == 19: boundaries = {} boundaries["left_eyebrow"] = landmarks_64[0:3] boundaries["right_eyebrow"] = landmarks_64[3:5] boundaries["left_eye"] = landmarks_64[6:9] boundaries["right_eye"] = landmarks_64[9:12] boundaries["noise"] = landmarks_64[12:15] elif self.num_landmarks == 29: boundaries = {} boundaries["upper_left_eyebrow"] = np.stack([landmarks_64[0], landmarks_64[4], landmarks_64[2]], axis=0) boundaries["lower_left_eyebrow"] = np.stack([landmarks_64[0], landmarks_64[5], landmarks_64[2]], axis=0) boundaries["upper_right_eyebrow"] = np.stack([landmarks_64[1], landmarks_64[6], landmarks_64[3]], axis=0) boundaries["lower_right_eyebrow"] = np.stack([landmarks_64[1], landmarks_64[7], landmarks_64[3]], axis=0) boundaries["upper_left_eye"] = np.stack([landmarks_64[8], landmarks_64[12], landmarks_64[10]], axis=0) boundaries["lower_left_eye"] = np.stack([landmarks_64[8], landmarks_64[13], landmarks_64[10]], axis=0) boundaries["upper_right_eye"] = np.stack([landmarks_64[9], landmarks_64[14], landmarks_64[11]], axis=0) boundaries["lower_right_eye"] = np.stack([landmarks_64[9], landmarks_64[15], landmarks_64[11]], axis=0) boundaries["noise"] = np.stack([landmarks_64[18], landmarks_64[21], landmarks_64[19]], axis=0) boundaries["outer_upper_lip"] = np.stack([landmarks_64[22], landmarks_64[24], landmarks_64[23]], axis=0) boundaries["inner_upper_lip"] = np.stack([landmarks_64[22], landmarks_64[25], landmarks_64[23]], axis=0) boundaries["outer_lower_lip"] = np.stack([landmarks_64[22], landmarks_64[26], landmarks_64[23]], axis=0) boundaries["inner_lower_lip"] = np.stack([landmarks_64[22], landmarks_64[27], landmarks_64[23]], axis=0) functions = {} for key, points in boundaries.items(): temp = points[0] new_points = points[0:1, :] for point in points[1:]: if point[0] == temp[0] and point[1] == temp[1]: continue else: new_points = np.concatenate((new_points, np.expand_dims(point, 0)), axis=0) temp = point points = new_points if points.shape[0] == 1: points = np.concatenate((points, points + 0.001), axis=0) k = min(4, points.shape[0]) functions[key] = interpolate.splprep([points[:, 0], points[:, 1]], k=k - 1, s=0) boundary_map = np.zeros((64, 64)) fig = plt.figure(figsize=[64 / 96.0, 64 / 96.0], dpi=96) ax = fig.add_axes([0, 0, 1, 1]) ax.axis("off") ax.imshow(boundary_map, interpolation="nearest", cmap="gray") # ax.scatter(landmarks[:, 0], landmarks[:, 1], s=1, marker=',', c='w') for key in functions.keys(): xnew = np.arange(0, 1, 0.01) out = interpolate.splev(xnew, functions[key][0], der=0) plt.plot(out[0], out[1], ",", linewidth=1, color="w") img = fig2data(fig) plt.close() sigma = 1 temp = 255 - img[:, :, 1] temp = cv2.distanceTransform(temp, cv2.DIST_L2, cv2.DIST_MASK_PRECISE) temp = temp.astype(np.float32) temp = np.where(temp < 3 * sigma, np.exp(-(temp * temp) / (2 * sigma * sigma)), 0) fig = plt.figure(figsize=[64 / 96.0, 64 / 96.0], dpi=96) ax = fig.add_axes([0, 0, 1, 1]) ax.axis("off") ax.imshow(temp, cmap="gray") plt.close() boundary_map = fig2data(fig) sample["boundary"] = boundary_map[:, :, 0] return sample class AddWeightMap(object): def __call__(self, sample): heatmap = sample["heatmap"] boundary = sample["boundary"] heatmap = np.concatenate((heatmap, np.expand_dims(boundary, axis=0)), 0) weight_map = np.zeros_like(heatmap) for i in range(heatmap.shape[0]): weight_map[i] = generate_weight_map(weight_map[i], heatmap[i]) sample["weight_map"] = weight_map return sample class ToTensor(object): """Convert ndarrays in sample to Tensors.""" def __call__(self, sample): image, heatmap, landmarks, boundary, weight_map = ( sample["image"], sample["heatmap"], sample["landmarks"], sample["boundary"], sample["weight_map"], ) # swap color axis because # numpy image: H x W x C # torch image: C X H X W if len(image.shape) == 2: image = np.expand_dims(image, axis=2) image_small = np.expand_dims(image_small, axis=2) image = image.transpose((2, 0, 1)) boundary = np.expand_dims(boundary, axis=2) boundary = boundary.transpose((2, 0, 1)) return { "image": torch.from_numpy(image).float().div(255.0), "heatmap": torch.from_numpy(heatmap).float(), "landmarks": torch.from_numpy(landmarks).float(), "boundary": torch.from_numpy(boundary).float().div(255.0), "weight_map": torch.from_numpy(weight_map).float(), } class FaceLandmarksDataset(Dataset): """Face Landmarks dataset.""" def __init__( self, img_dir, landmarks_dir, num_landmarks=68, gray_scale=False, detect_face=False, enhance=False, center_shift=0, transform=None, ): """ Args: landmark_dir (string): Path to the mat file with landmarks saved. img_dir (string): Directory with all the images. transform (callable, optional): Optional transform to be applied on a sample. """ self.img_dir = img_dir self.landmarks_dir = landmarks_dir self.num_lanmdkars = num_landmarks self.transform = transform self.img_names = glob.glob(self.img_dir + "*.jpg") + glob.glob(self.img_dir + "*.png") self.gray_scale = gray_scale self.detect_face = detect_face self.enhance = enhance self.center_shift = center_shift if self.detect_face: self.face_detector = MTCNN(thresh=[0.5, 0.6, 0.7]) def __len__(self): return len(self.img_names) def __getitem__(self, idx): img_name = self.img_names[idx] pil_image = Image.open(img_name) if pil_image.mode != "RGB": # if input is grayscale image, convert it to 3 channel image if self.enhance: pil_image = power_transform(pil_image, 0.5) temp_image = Image.new("RGB", pil_image.size) temp_image.paste(pil_image) pil_image = temp_image image = np.array(pil_image) if self.gray_scale: image = rgb2gray(image) image = np.expand_dims(image, axis=2) image = np.concatenate((image, image, image), axis=2) image = image * 255.0 image = image.astype(np.uint8) if not self.detect_face: center = [450 // 2, 450 // 2 + 0] if self.center_shift != 0: center[0] += int(np.random.uniform(-self.center_shift, self.center_shift)) center[1] += int(np.random.uniform(-self.center_shift, self.center_shift)) scale = 1.8 else: detected_faces = self.face_detector.detect_image(image) if len(detected_faces) > 0: box = detected_faces[0] left, top, right, bottom, _ = box center = [right - (right - left) / 2.0, bottom - (bottom - top) / 2.0] center[1] = center[1] - (bottom - top) * 0.12 scale = (right - left + bottom - top) / 195.0 else: center = [450 // 2, 450 // 2 + 0] scale = 1.8 if self.center_shift != 0: shift = self.center * self.center_shift / 450 center[0] += int(np.random.uniform(-shift, shift)) center[1] += int(np.random.uniform(-shift, shift)) base_name = os.path.basename(img_name) landmarks_base_name = base_name[:-4] + "_pts.mat" landmarks_name = os.path.join(self.landmarks_dir, landmarks_base_name) if os.path.isfile(landmarks_name): mat_data = sio.loadmat(landmarks_name) landmarks = mat_data["pts_2d"] elif os.path.isfile(landmarks_name[:-8] + ".pts.npy"): landmarks = np.load(landmarks_name[:-8] + ".pts.npy") else: landmarks = [] heatmap = [] if landmarks != []: new_image, new_landmarks = cv_crop(image, landmarks, center, scale, 256, self.center_shift) tries = 0 while self.center_shift != 0 and tries < 5 and (np.max(new_landmarks) > 240 or np.min(new_landmarks) < 15): center = [450 // 2, 450 // 2 + 0] scale += 0.05 center[0] += int(np.random.uniform(-self.center_shift, self.center_shift)) center[1] += int(np.random.uniform(-self.center_shift, self.center_shift)) new_image, new_landmarks = cv_crop(image, landmarks, center, scale, 256, self.center_shift) tries += 1 if np.max(new_landmarks) > 250 or np.min(new_landmarks) < 5: center = [450 // 2, 450 // 2 + 0] scale = 2.25 new_image, new_landmarks = cv_crop(image, landmarks, center, scale, 256, 100) assert np.min(new_landmarks) > 0 and np.max(new_landmarks) < 256, "Landmarks out of boundary!" image = new_image landmarks = new_landmarks heatmap = np.zeros((self.num_lanmdkars, 64, 64)) for i in range(self.num_lanmdkars): if landmarks[i][0] > 0: heatmap[i] = draw_gaussian(heatmap[i], landmarks[i] / 4.0 + 1, 1) sample = {"image": image, "heatmap": heatmap, "landmarks": landmarks} if self.transform: sample = self.transform(sample) return sample def get_dataset( val_img_dir, val_landmarks_dir, batch_size, num_landmarks=68, rotation=0, scale=0, center_shift=0, random_flip=False, brightness=0, contrast=0, saturation=0, blur=False, noise=False, jpeg_effect=False, random_occlusion=False, gray_scale=False, detect_face=False, enhance=False, ): val_transforms = transforms.Compose([AddBoundary(num_landmarks), AddWeightMap(), ToTensor()]) val_dataset = FaceLandmarksDataset( val_img_dir, val_landmarks_dir, num_landmarks=num_landmarks, gray_scale=gray_scale, detect_face=detect_face, enhance=enhance, transform=val_transforms, ) val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=6) data_loaders = {"val": val_dataloader} dataset_sizes = {} dataset_sizes["val"] = len(val_dataset) return data_loaders, dataset_sizes