face-swap / face_swap.py
shuhulhandoo's picture
Upload 6 files
5664dc5
#! /usr/bin/env python
import cv2
import numpy as np
import scipy.spatial as spatial
import logging
## 3D Transform
def bilinear_interpolate(img, coords):
""" Interpolates over every image channel
http://en.wikipedia.org/wiki/Bilinear_interpolation
:param img: max 3 channel image
:param coords: 2 x _m_ array. 1st row = xcoords, 2nd row = ycoords
:returns: array of interpolated pixels with same shape as coords
"""
int_coords = np.int32(coords)
x0, y0 = int_coords
dx, dy = coords - int_coords
# 4 Neighour pixels
q11 = img[y0, x0]
q21 = img[y0, x0 + 1]
q12 = img[y0 + 1, x0]
q22 = img[y0 + 1, x0 + 1]
btm = q21.T * dx + q11.T * (1 - dx)
top = q22.T * dx + q12.T * (1 - dx)
inter_pixel = top * dy + btm * (1 - dy)
return inter_pixel.T
def grid_coordinates(points):
""" x,y grid coordinates within the ROI of supplied points
:param points: points to generate grid coordinates
:returns: array of (x, y) coordinates
"""
xmin = np.min(points[:, 0])
xmax = np.max(points[:, 0]) + 1
ymin = np.min(points[:, 1])
ymax = np.max(points[:, 1]) + 1
return np.asarray([(x, y) for y in range(ymin, ymax)
for x in range(xmin, xmax)], np.uint32)
def process_warp(src_img, result_img, tri_affines, dst_points, delaunay):
"""
Warp each triangle from the src_image only within the
ROI of the destination image (points in dst_points).
"""
roi_coords = grid_coordinates(dst_points)
# indices to vertices. -1 if pixel is not in any triangle
roi_tri_indices = delaunay.find_simplex(roi_coords)
for simplex_index in range(len(delaunay.simplices)):
coords = roi_coords[roi_tri_indices == simplex_index]
num_coords = len(coords)
out_coords = np.dot(tri_affines[simplex_index],
np.vstack((coords.T, np.ones(num_coords))))
x, y = coords.T
result_img[y, x] = bilinear_interpolate(src_img, out_coords)
return None
def triangular_affine_matrices(vertices, src_points, dst_points):
"""
Calculate the affine transformation matrix for each
triangle (x,y) vertex from dst_points to src_points
:param vertices: array of triplet indices to corners of triangle
:param src_points: array of [x, y] points to landmarks for source image
:param dst_points: array of [x, y] points to landmarks for destination image
:returns: 2 x 3 affine matrix transformation for a triangle
"""
ones = [1, 1, 1]
for tri_indices in vertices:
src_tri = np.vstack((src_points[tri_indices, :].T, ones))
dst_tri = np.vstack((dst_points[tri_indices, :].T, ones))
mat = np.dot(src_tri, np.linalg.inv(dst_tri))[:2, :]
yield mat
def warp_image_3d(src_img, src_points, dst_points, dst_shape, dtype=np.uint8):
rows, cols = dst_shape[:2]
result_img = np.zeros((rows, cols, 3), dtype=dtype)
delaunay = spatial.Delaunay(dst_points)
tri_affines = np.asarray(list(triangular_affine_matrices(
delaunay.simplices, src_points, dst_points)))
process_warp(src_img, result_img, tri_affines, dst_points, delaunay)
return result_img
## 2D Transform
def transformation_from_points(points1, points2):
points1 = points1.astype(np.float64)
points2 = points2.astype(np.float64)
c1 = np.mean(points1, axis=0)
c2 = np.mean(points2, axis=0)
points1 -= c1
points2 -= c2
s1 = np.std(points1)
s2 = np.std(points2)
points1 /= s1
points2 /= s2
U, S, Vt = np.linalg.svd(np.dot(points1.T, points2))
R = (np.dot(U, Vt)).T
return np.vstack([np.hstack([s2 / s1 * R,
(c2.T - np.dot(s2 / s1 * R, c1.T))[:, np.newaxis]]),
np.array([[0., 0., 1.]])])
def warp_image_2d(im, M, dshape):
output_im = np.zeros(dshape, dtype=im.dtype)
cv2.warpAffine(im,
M[:2],
(dshape[1], dshape[0]),
dst=output_im,
borderMode=cv2.BORDER_TRANSPARENT,
flags=cv2.WARP_INVERSE_MAP)
return output_im
## Generate Mask
def mask_from_points(size, points,erode_flag=1):
radius = 10 # kernel size
kernel = np.ones((radius, radius), np.uint8)
mask = np.zeros(size, np.uint8)
cv2.fillConvexPoly(mask, cv2.convexHull(points), 255)
if erode_flag:
mask = cv2.erode(mask, kernel,iterations=1)
return mask
## Color Correction
def correct_colours(im1, im2, landmarks1):
COLOUR_CORRECT_BLUR_FRAC = 0.75
LEFT_EYE_POINTS = list(range(42, 48))
RIGHT_EYE_POINTS = list(range(36, 42))
blur_amount = COLOUR_CORRECT_BLUR_FRAC * np.linalg.norm(
np.mean(landmarks1[LEFT_EYE_POINTS], axis=0) -
np.mean(landmarks1[RIGHT_EYE_POINTS], axis=0))
blur_amount = int(blur_amount)
if blur_amount % 2 == 0:
blur_amount += 1
im1_blur = cv2.GaussianBlur(im1, (blur_amount, blur_amount), 0)
im2_blur = cv2.GaussianBlur(im2, (blur_amount, blur_amount), 0)
# Avoid divide-by-zero errors.
im2_blur = im2_blur.astype(int)
im2_blur += 128*(im2_blur <= 1)
result = im2.astype(np.float64) * im1_blur.astype(np.float64) / im2_blur.astype(np.float64)
result = np.clip(result, 0, 255).astype(np.uint8)
return result
## Copy-and-paste
def apply_mask(img, mask):
""" Apply mask to supplied image
:param img: max 3 channel image
:param mask: [0-255] values in mask
:returns: new image with mask applied
"""
masked_img=cv2.bitwise_and(img,img,mask=mask)
return masked_img
## Alpha blending
def alpha_feathering(src_img, dest_img, img_mask, blur_radius=15):
mask = cv2.blur(img_mask, (blur_radius, blur_radius))
mask = mask / 255.0
result_img = np.empty(src_img.shape, np.uint8)
for i in range(3):
result_img[..., i] = src_img[..., i] * mask + dest_img[..., i] * (1-mask)
return result_img
def check_points(img,points):
# Todo: I just consider one situation.
if points[8,1]>img.shape[0]:
logging.error("Jaw part out of image")
else:
return True
return False
def face_swap(src_face, dst_face, src_points, dst_points, dst_shape, dst_img, args, end=48):
h, w = dst_face.shape[:2]
## 3d warp
warped_src_face = warp_image_3d(src_face, src_points[:end], dst_points[:end], (h, w))
## Mask for blending
mask = mask_from_points((h, w), dst_points)
mask_src = np.mean(warped_src_face, axis=2) > 0
mask = np.asarray(mask * mask_src, dtype=np.uint8)
## Correct color
if args == "correct color":
warped_src_face = apply_mask(warped_src_face, mask)
dst_face_masked = apply_mask(dst_face, mask)
warped_src_face = correct_colours(dst_face_masked, warped_src_face, dst_points)
## 2d warp
if args == "warp_2d":
unwarped_src_face = warp_image_3d(warped_src_face, dst_points[:end], src_points[:end], src_face.shape[:2])
warped_src_face = warp_image_2d(unwarped_src_face, transformation_from_points(dst_points, src_points),
(h, w, 3))
mask = mask_from_points((h, w), dst_points)
mask_src = np.mean(warped_src_face, axis=2) > 0
mask = np.asarray(mask * mask_src, dtype=np.uint8)
## Shrink the mask
kernel = np.ones((10, 10), np.uint8)
mask = cv2.erode(mask, kernel, iterations=1)
##Poisson Blending
r = cv2.boundingRect(mask)
center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)))
output = cv2.seamlessClone(warped_src_face, dst_face, mask, center, cv2.NORMAL_CLONE)
x, y, w, h = dst_shape
dst_img_cp = dst_img.copy()
dst_img_cp[y:y + h, x:x + w] = output
return dst_img_cp