MakeItTalk/thirdparty/face_of_art/pdm_clm_functions.py

from thirdparty.face_of_art.logging_functions import *
import os
import numpy as np
from menpo.shape import PointCloud
from menpofit.clm import GradientDescentCLMFitter
import pickle
import math

jaw_line_inds = np.arange(0, 17)
nose_inds = np.arange(27, 36)
left_eye_inds = np.arange(36, 42)
right_eye_inds = np.arange(42, 48)
left_brow_inds = np.arange(17, 22)
right_brow_inds = np.arange(22, 27)
mouth_inds = np.arange(48, 68)


def sigmoid(x, rate, offset):
    return 1 / (1 + math.exp(-rate * (x - offset)))


def calculate_evidence(patch_responses, rate=0.25, offset=20):
    # from ECT: https://github.com/HongwenZhang/ECT-FaceAlignment

    rspmapShape = patch_responses[0, 0, ...].shape
    n_points = patch_responses.shape[0]

    y_weight = [np.sum(patch_responses[i, 0, ...], axis=1) for i in range(n_points)]
    x_weight = [np.sum(patch_responses[i, 0, ...], axis=0) for i in range(n_points)]

    # y_weight /= y_weight.sum()
    # x_weight /= x_weight.sum()

    y_coordinate = range(0, rspmapShape[0])
    x_coordinate = range(0, rspmapShape[1])

    varList = [(np.abs(
        np.average((y_coordinate - np.average(y_coordinate, weights=y_weight[i])) ** 2, weights=y_weight[i])),
                np.abs(np.average((x_coordinate - np.average(x_coordinate, weights=x_weight[i])) ** 2,
                                  weights=x_weight[i])))
               for i in range(n_points)]

    # patch_responses[patch_responses<0.001] = 0
    prpList = [
        (np.sum(patch_responses[i, 0, ...], axis=(-1, -2)), np.sum(patch_responses[i, 0, ...], axis=(-1, -2)))
        for i in range(n_points)]

    var = np.array(varList).flatten()
    var[var == 0] = np.finfo(float).eps
    var = np.sqrt(var)
    var = 1 / var

    weight = np.array(prpList).flatten()
    weight *= var

    # offset = np.average(weight) - 20
    weight = [sigmoid(i, rate, offset) for i in weight]

    weight = np.array(weight)

    return weight


def get_patches_around_landmarks(heat_maps, menpo_shape, patch_size=(30,30), image_shape=256):
    # from ECT: https://github.com/HongwenZhang/ECT-FaceAlignment

    padH = int(image_shape / 2)
    padW = int(image_shape / 2)

    rps_zeros = np.zeros((1, 2 * image_shape, 2 * image_shape, menpo_shape.n_points))
    rps_zeros[0, padH:padH + image_shape, padW:padW + image_shape, :] = heat_maps

    rOffset = np.floor(patch_size[0] / 2).astype(int)
    lOffset = patch_size[0] - rOffset

    rspList = [rps_zeros[0, y - rOffset:y + lOffset, x - rOffset:x + lOffset, i] for i in range(menpo_shape.n_points)
               for y in [np.around(menpo_shape.points[i][0] + 1 + padH).astype(int)]
               for x in [np.around(menpo_shape.points[i][1] + 1 + padW).astype(int)]]
    patches = np.array(rspList)[:, None, :, :]
    return patches


def pdm_correct(init_shape, pdm_model, part_inds=None):
    """ correct landmarks using pdm (point distribution model)"""
    pdm_model.set_target(PointCloud(init_shape))
    if part_inds is None:
        return pdm_model.target.points
    else:
        return pdm_model.target.points[part_inds]


def weighted_pdm_transform(input_pdm_model, patches, shape, inirho=20):
    # from ECT: https://github.com/HongwenZhang/ECT-FaceAlignment
    weight = calculate_evidence(patches, rate=0.5, offset=10).reshape((1, -1))
    pdm_model = input_pdm_model.copy()

    # write project_weight
    ini_rho2_inv_prior = np.hstack((np.zeros((4,)), inirho / pdm_model.model.eigenvalues))
    J = np.rollaxis(pdm_model.d_dp(None), -1, 1)
    J = J.reshape((-1, J.shape[-1]))

    initial_shape_mean = shape.points.ravel() - pdm_model.model._mean
    iniJe = - J.T.dot(initial_shape_mean * weight[0])
    iniJWJ = J.T.dot(np.diag(weight[0]).dot(J))
    inv_JJ = np.linalg.inv(iniJWJ + np.diag(ini_rho2_inv_prior))
    initial_p = -inv_JJ.dot(iniJe)

    # Update pdm
    pdm_model._from_vector_inplace(initial_p)
    return pdm_model.target.points


def w_pdm_correct(init_shape, patches, pdm_model, part_inds=None):
    """ correct landmarks using weighted pdm"""

    points = weighted_pdm_transform(input_pdm_model=pdm_model, patches=patches, shape=PointCloud(init_shape))

    if (part_inds is not None and pdm_model.n_points < 68) or part_inds is None:
        return points
    else:
        return points[part_inds]


def feature_based_pdm_corr(lms_init, models_dir, train_type='basic', patches=None):
    """ correct landmarks using part-based pdm"""

    jaw_line_inds = np.arange(0, 17)
    nose_inds = np.arange(27, 36)
    left_eye_inds = np.arange(36, 42)
    right_eye_inds = np.arange(42, 48)
    left_brow_inds = np.arange(17, 22)
    right_brow_inds = np.arange(22, 27)
    mouth_inds = np.arange(48, 68)

    '''
    selected number of PCs:
    jaw:7
    eye:3
    brow:2
    nose:5
    mouth:7
    '''

    new_lms = np.zeros((68, 2))

    parts = ['l_brow', 'r_brow', 'l_eye', 'r_eye', 'mouth', 'nose', 'jaw']
    part_inds_opt = [left_brow_inds, right_brow_inds, left_eye_inds, right_eye_inds, mouth_inds, nose_inds,
                     jaw_line_inds]
    pc_opt = [2, 2, 3, 3, 7, 5, 7]

    for i, part in enumerate(parts):
        part_inds = part_inds_opt[i]
        pc = pc_opt[i]
        temp_model = os.path.join(models_dir, train_type + '_' + part + '_' + str(pc))
        filehandler = open(temp_model, "rb")
        try:
            pdm_temp = pickle.load(filehandler)
        except UnicodeDecodeError:
            pdm_temp = pickle.load(filehandler, fix_imports=True, encoding="latin1")
        filehandler.close()

        if patches is None:
            part_lms_pdm = pdm_correct(lms_init[part_inds], pdm_temp)
        else:
            part_lms_pdm = w_pdm_correct(
                init_shape=lms_init[part_inds], patches=patches, pdm_model=pdm_temp, part_inds=part_inds)

        new_lms[part_inds] = part_lms_pdm
    return new_lms


def clm_correct(clm_model_path, image, map, lms_init):
    """ tune landmarks using clm (constrained local model)"""

    filehandler = open(os.path.join(clm_model_path), "rb")
    try:
        part_model = pickle.load(filehandler)
    except UnicodeDecodeError:
        part_model = pickle.load(filehandler, fix_imports=True, encoding="latin1")
    filehandler.close()

    # from ECT: https://github.com/HongwenZhang/ECT-FaceAlignment
    part_model.opt = dict()
    part_model.opt['numIter'] = 5
    part_model.opt['kernel_covariance'] = 10
    part_model.opt['sigOffset'] = 25
    part_model.opt['sigRate'] = 0.25
    part_model.opt['pdm_rho'] = 20
    part_model.opt['verbose'] = False
    part_model.opt['rho2'] = 20
    part_model.opt['ablation'] = (True, True)
    part_model.opt['ratio1'] = 0.12
    part_model.opt['ratio2'] = 0.08
    part_model.opt['smooth'] = True

    fitter = GradientDescentCLMFitter(part_model, n_shape=30)

    image.rspmap_data = np.swapaxes(np.swapaxes(map, 1, 3), 2, 3)

    fr = fitter.fit_from_shape(image=image, initial_shape=PointCloud(lms_init), gt_shape=PointCloud(lms_init))
    w_pdm_clm = fr.final_shape.points

    return w_pdm_clm