MakeItTalk/thirdparty/face_of_art/menpo_functions.py

import os
from scipy.io import loadmat
from menpo.shape.pointcloud import PointCloud
from menpo.transform import ThinPlateSplines
import menpo.transform as mt

import menpo.io as mio
from glob import glob
from thirdparty.face_of_art.deformation_functions import *

# landmark indices by facial feature
jaw_indices = np.arange(0, 17)
lbrow_indices = np.arange(17, 22)
rbrow_indices = np.arange(22, 27)
upper_nose_indices = np.arange(27, 31)
lower_nose_indices = np.arange(31, 36)
leye_indices = np.arange(36, 42)
reye_indices = np.arange(42, 48)
outer_mouth_indices = np.arange(48, 60)
inner_mouth_indices = np.arange(60, 68)

# flipped landmark indices
mirrored_parts_68 = np.hstack([
    jaw_indices[::-1], rbrow_indices[::-1], lbrow_indices[::-1],
    upper_nose_indices, lower_nose_indices[::-1],
    np.roll(reye_indices[::-1], 4), np.roll(leye_indices[::-1], 4),
    np.roll(outer_mouth_indices[::-1], 7),
    np.roll(inner_mouth_indices[::-1], 5)
])


def load_bb_files(bb_file_dirs):
    """load bounding box mat file for challenging, common, full & training datasets"""

    bb_files_dict = {}
    for bb_file in bb_file_dirs:
        bb_mat = loadmat(bb_file)['bounding_boxes']
        num_imgs = np.max(bb_mat.shape)
        for i in range(num_imgs):
            name = bb_mat[0][i][0][0][0][0]
            bb_init = bb_mat[0][i][0][0][1] - 1  # matlab indicies
            bb_gt = bb_mat[0][i][0][0][2] - 1  # matlab indicies
            if str(name) in bb_files_dict.keys():
                print (str(name) + ' already exists')
            else:
                bb_files_dict[str(name)] = (bb_init, bb_gt)
    return bb_files_dict


def load_bb_dictionary(bb_dir, mode, test_data='full'):
    """create bounding box dictionary of input dataset: train/common/full/challenging"""

    if mode == 'TRAIN':
        bb_dirs = \
            ['bounding_boxes_afw.mat', 'bounding_boxes_helen_trainset.mat', 'bounding_boxes_lfpw_trainset.mat']
    else:
        if test_data == 'common':
            bb_dirs = \
                ['bounding_boxes_helen_testset.mat', 'bounding_boxes_lfpw_testset.mat']
        elif test_data == 'challenging':
            bb_dirs = ['bounding_boxes_ibug.mat']
        elif test_data == 'full':
            bb_dirs = \
                ['bounding_boxes_ibug.mat', 'bounding_boxes_helen_testset.mat', 'bounding_boxes_lfpw_testset.mat']
        elif test_data == 'training':
            bb_dirs = \
                ['bounding_boxes_afw.mat', 'bounding_boxes_helen_trainset.mat', 'bounding_boxes_lfpw_trainset.mat']
        else:
            bb_dirs = None

    if mode == 'TEST' and test_data not in ['full', 'challenging', 'common', 'training']:
        bb_files_dict = None
    else:
        bb_dirs = [os.path.join(bb_dir, dataset) for dataset in bb_dirs]
        bb_files_dict = load_bb_files(bb_dirs)

    return bb_files_dict


def center_margin_bb(bb, img_bounds, margin=0.25):
    """create new bounding box with input margin"""

    bb_size = ([bb[0, 2] - bb[0, 0], bb[0, 3] - bb[0, 1]])
    margins = (np.max(bb_size) * (1 + margin) - bb_size) / 2
    bb_new = np.zeros_like(bb)
    bb_new[0, 0] = np.maximum(bb[0, 0] - margins[0], 0)
    bb_new[0, 2] = np.minimum(bb[0, 2] + margins[0], img_bounds[1])
    bb_new[0, 1] = np.maximum(bb[0, 1] - margins[1], 0)
    bb_new[0, 3] = np.minimum(bb[0, 3] + margins[1], img_bounds[0])
    return bb_new


def crop_to_face_image(img, bb_dictionary=None, gt=True, margin=0.25, image_size=256, normalize=True,
                       return_transform=False):
    """crop face image using bounding box dictionary, or GT landmarks"""

    name = img.path.name
    img_bounds = img.bounds()[1]

    # if there is no bounding-box dict and GT landmarks are available, use it to determine the bounding box
    if bb_dictionary is None and img.has_landmarks:
        grp_name = img.landmarks.group_labels[0]
        bb_menpo = img.landmarks[grp_name].bounding_box().points
        bb = np.array([[bb_menpo[0, 1], bb_menpo[0, 0], bb_menpo[2, 1], bb_menpo[2, 0]]])
    elif bb_dictionary is not None:
        if gt:
            bb = bb_dictionary[name][1]  # ground truth
        else:
            bb = bb_dictionary[name][0]  # init from face detector
    else:
        bb = None

    if bb is not None:
        # add margin to bounding box
        bb = center_margin_bb(bb, img_bounds, margin=margin)
        bb_pointcloud = PointCloud(np.array([[bb[0, 1], bb[0, 0]],
                                             [bb[0, 3], bb[0, 0]],
                                             [bb[0, 3], bb[0, 2]],
                                             [bb[0, 1], bb[0, 2]]]))
        if return_transform:
            face_crop, bb_transform = img.crop_to_pointcloud(bb_pointcloud, return_transform=True)
        else:
            face_crop = img.crop_to_pointcloud(bb_pointcloud)
    else:
        # if there is no bounding box/gt landmarks, use entire image
        face_crop = img.copy()
        bb_transform = None

    # if face crop is not a square - pad borders with mean pixel value
    h, w = face_crop.shape
    diff = h - w
    if diff < 0:
        face_crop.pixels = np.pad(face_crop.pixels, ((0, 0), (0, -1 * diff), (0, 0)), 'mean')
    elif diff > 0:
        face_crop.pixels = np.pad(face_crop.pixels, ((0, 0), (0, 0), (0, diff)), 'mean')

    if return_transform:
        face_crop, rescale_transform = face_crop.resize([image_size, image_size], return_transform=True)
        if bb_transform is None:
            transform_chain = rescale_transform
        else:
            transform_chain = mt.TransformChain(transforms=(rescale_transform, bb_transform))
    else:
        face_crop = face_crop.resize([image_size, image_size])

    if face_crop.n_channels == 4:
        face_crop.pixels = face_crop.pixels[:3, :, :]

    if normalize:
        face_crop.pixels = face_crop.rescale_pixels(0., 1.).pixels

    if return_transform:
        return face_crop, transform_chain
    else:
        return face_crop


def augment_face_image(img, image_size=256, crop_size=248, angle_range=30, flip=True):
    """basic image augmentation: random crop, rotation and horizontal flip"""

    # taken from MDM: https://github.com/trigeorgis/mdm
    def mirror_landmarks_68(lms, im_size):
        return PointCloud(abs(np.array([0, im_size[1]]) - lms.as_vector(
        ).reshape(-1, 2))[mirrored_parts_68])

    # taken from MDM: https://github.com/trigeorgis/mdm
    def mirror_image(im):
        im = im.copy()
        im.pixels = im.pixels[..., ::-1].copy()

        for group in im.landmarks:
            lms = im.landmarks[group]
            if lms.points.shape[0] == 68:
                im.landmarks[group] = mirror_landmarks_68(lms, im.shape)

        return im

    flip_rand = np.random.random() > 0.5
    #     rot_rand = np.random.random() > 0.5
    #     crop_rand = np.random.random() > 0.5
    rot_rand = True  # like ECT: https://github.com/HongwenZhang/ECT-FaceAlignment
    crop_rand = True  # like ECT: https://github.com/HongwenZhang/ECT-FaceAlignment

    if crop_rand:
        lim = image_size - crop_size
        min_crop_inds = np.random.randint(0, lim, 2)
        max_crop_inds = min_crop_inds + crop_size
        img = img.crop(min_crop_inds, max_crop_inds)

    if flip and flip_rand:
        img = mirror_image(img)

    if rot_rand:
        rot_angle = 2 * angle_range * np.random.random_sample() - angle_range
        img = img.rotate_ccw_about_centre(rot_angle)

    img = img.resize([image_size, image_size])

    return img


def augment_menpo_img_ns(img, img_dir_ns, p_ns=0.):
    """texture style image augmentation using stylized copies in *img_dir_ns*"""

    img = img.copy()
    if p_ns > 0.5:
        ns_augs = glob(os.path.join(img_dir_ns, img.path.name.split('.')[0] + '_ns*'))
        num_augs = len(ns_augs)
        if num_augs > 0:
            ns_ind = np.random.randint(0, num_augs)
            ns_aug = mio.import_image(ns_augs[ns_ind])
            ns_pixels = ns_aug.pixels
            img.pixels = ns_pixels
    return img


def augment_menpo_img_geom(img, p_geom=0.):
    """geometric style image augmentation using random face deformations"""

    img = img.copy()
    if p_geom > 0.5:
        grp_name = img.landmarks.group_labels[0]
        lms_geom_warp = deform_face_geometric_style(img.landmarks[grp_name].points.copy(), p_scale=p_geom, p_shift=p_geom)
        img = warp_face_image_tps(img, PointCloud(lms_geom_warp), grp_name)
    return img


def warp_face_image_tps(img, new_shape, lms_grp_name='PTS', warp_mode='constant'):
    """warp image to new landmarks using TPS interpolation"""

    tps = ThinPlateSplines(new_shape, img.landmarks[lms_grp_name])
    try:
        img_warp = img.warp_to_shape(img.shape, tps, mode=warp_mode)
        img_warp.landmarks[lms_grp_name] = new_shape
        return img_warp
    except np.linalg.linalg.LinAlgError as err:
        print ('Error:'+str(err)+'\nUsing original landmarks for:\n'+str(img.path))
        return img


def load_menpo_image_list(
    img_dir, train_crop_dir, img_dir_ns, mode, bb_dictionary=None, image_size=256, margin=0.25,
    bb_type='gt', test_data='full', augment_basic=True, augment_texture=False, p_texture=0,
    augment_geom=False, p_geom=0, verbose=False, return_transform=False):

    """load images from image dir to create menpo-type image list"""

    def crop_to_face_image_gt(img):
        return crop_to_face_image(img, bb_dictionary, gt=True, margin=margin, image_size=image_size,
                                  return_transform=return_transform)

    def crop_to_face_image_init(img):
        return crop_to_face_image(img, bb_dictionary, gt=False, margin=margin, image_size=image_size,
                                  return_transform=return_transform)

    def crop_to_face_image_test(img):
        return crop_to_face_image(img, bb_dictionary=None, margin=margin, image_size=image_size,
                                  return_transform=return_transform)

    def augment_menpo_img_ns_rand(img):
        return augment_menpo_img_ns(img, img_dir_ns, p_ns=1. * (np.random.rand() < p_texture)[0])

    def augment_menpo_img_geom_rand(img):
        return augment_menpo_img_geom(img, p_geom=1. * (np.random.rand() < p_geom)[0])

    if mode is 'TRAIN':
        if train_crop_dir is None:
            img_set_dir = os.path.join(img_dir, 'training')
            out_image_list = mio.import_images(img_set_dir, verbose=verbose, normalize=False)
            if bb_type is 'gt':
                out_image_list = out_image_list.map(crop_to_face_image_gt)
            elif bb_type is 'init':
                out_image_list = out_image_list.map(crop_to_face_image_init)
        else:
            img_set_dir = os.path.join(img_dir, train_crop_dir)
            out_image_list = mio.import_images(img_set_dir, verbose=verbose)

        # perform image augmentation
        if augment_texture and p_texture > 0:
            out_image_list = out_image_list.map(augment_menpo_img_ns_rand)
        if augment_geom and p_geom > 0:
            out_image_list = out_image_list.map(augment_menpo_img_geom_rand)
        if augment_basic:
            out_image_list = out_image_list.map(augment_face_image)

    else:  # if mode is 'TEST', load test data
        if test_data in ['full', 'challenging', 'common', 'training', 'test']:
            img_set_dir = os.path.join(img_dir, test_data)
            out_image_list = mio.import_images(img_set_dir, verbose=verbose, normalize=False)
            if bb_type is 'gt':
                out_image_list = out_image_list.map(crop_to_face_image_gt)
            elif bb_type is 'init':
                out_image_list = out_image_list.map(crop_to_face_image_init)
        else:
            img_set_dir = os.path.join(img_dir, test_data+'*')
            out_image_list = mio.import_images(img_set_dir, verbose=verbose, normalize=False)
            out_image_list = out_image_list.map(crop_to_face_image_test)

    return out_image_list