Spaces:
Running
Running
import numpy as np | |
import cv2 | |
from skimage import io | |
import matplotlib.pyplot as plt | |
from os import listdir | |
from os.path import join, isfile, splitext | |
from scipy import misc | |
import sys | |
import dlib | |
import os | |
from PIL import Image | |
import urllib.request | |
import imageio | |
width_ratio = 1.5 | |
top_ratio = 1.5 | |
gap_ratio = 0.1 | |
down_ratio = 4.5 | |
chin_width_ratio = 2.8 | |
forehead_ratio = 0.3 | |
verb = False | |
BASE_DIR = os.path.dirname(os.path.abspath(__file__)) | |
PREDICTOR_PATH = os.path.join(BASE_DIR, "shape_predictor_68_face_landmarks.dat") | |
eye_cascade = cv2.CascadeClassifier(os.path.join(BASE_DIR, "haarcascade_eye.xml")) | |
assert not eye_cascade.empty() | |
SCALE_FACTOR = 1 | |
FEATHER_AMOUNT = 11 | |
FACE_POINTS = list(range(17, 68)) | |
MOUTH_POINTS = list(range(48, 61)) | |
RIGHT_BROW_POINTS = list(range(17, 22)) | |
LEFT_BROW_POINTS = list(range(22, 27)) | |
RIGHT_EYE_POINTS = list(range(36, 42)) | |
LEFT_EYE_POINTS = list(range(42, 48)) | |
NOSE_POINTS = list(range(27, 35)) | |
JAW_POINTS = list(range(0, 17)) | |
OVERLAY_POINTS = [ | |
LEFT_EYE_POINTS + RIGHT_EYE_POINTS + LEFT_BROW_POINTS + RIGHT_BROW_POINTS, | |
NOSE_POINTS + MOUTH_POINTS, | |
] | |
detector = dlib.get_frontal_face_detector() | |
predictor = dlib.shape_predictor(PREDICTOR_PATH) | |
class TooManyFaces(Exception): | |
pass | |
class NoFaces(Exception): | |
pass | |
def get_landmarks(im): | |
rects = detector(im, 1) | |
if len(rects) > 1: | |
raise TooManyFaces | |
if len(rects) == 0: | |
raise NoFaces | |
return np.matrix([[p.x, p.y] for p in predictor(im, rects[0]).parts()]) | |
def read_imgURL(URL): | |
with urllib.request.urlopen(URL) as url: | |
with open('temp.jpg', 'wb') as f: | |
f.write(url.read()) | |
img = Image.open('temp.jpg') | |
img = np.array(img) | |
return img | |
def draw_convex_hull(im, points, color): | |
points = cv2.convexHull(points) | |
cv2.fillConvexPoly(im, points, color=color) | |
def get_face_mask(im, landmarks): | |
im = np.zeros(im.shape[:2], dtype=np.float64) | |
for group in OVERLAY_POINTS: | |
draw_convex_hull(im, | |
landmarks[group], | |
color=1) | |
im = np.array([im, im, im]).transpose((1, 2, 0)) | |
im = (cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0) * 1.0 | |
im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) | |
return im | |
def read_im_and_landmarks(fname): | |
im = np.array(fname) | |
im = cv2.resize(im, (im.shape[1] * SCALE_FACTOR, | |
im.shape[0] * SCALE_FACTOR)) | |
s = get_landmarks(im) | |
return im, s | |
def warp_im(im, M, dshape): | |
output_im = np.zeros(dshape, dtype=im.dtype) | |
cv2.warpAffine(im, | |
M[:2], | |
(dshape[1], dshape[0]), | |
dst=output_im, | |
borderMode=cv2.BORDER_TRANSPARENT, | |
flags=cv2.WARP_INVERSE_MAP) | |
return output_im | |
def infer_chin_region(eye, width_ratio, down_ratio, left_or_right): | |
region1 = [0] * 4 | |
if left_or_right == 'right': #assuming it is the absolute right chin | |
region1[0] = int(max(0, int(eye[0] - 0.5 * eye[2]))) #chin region should go lefwards | |
region1[2] = int(0.5 * eye[2]) | |
else: # assuming it is the absolute left chin | |
region1[0] = int(eye[0] + eye[2]) # chin region should go rightwards | |
region1[2] = int(0.5 * eye[2]) | |
region1[1] = int(eye[1] + eye[3]) | |
region1[3] = int(1.5 * eye[3]) | |
return region1 | |
def detect_face_direction(gray, face, eye, down_ratio, chin_width_ratio): | |
region1 = [0] * 4 # assuming this is the left eye, forhead should go rightward | |
region2 = [0] * 4 # assuming this is the right eye, forhead should go leftward | |
print(eye[0]) | |
region1 = infer_chin_region(eye[0], chin_width_ratio, down_ratio, 'left') #region1 is from eye to right | |
region2 = infer_chin_region(eye[0], chin_width_ratio, down_ratio, 'right') # region2 is from eye to left | |
std1 = np.std(gray[region1[1]:(region1[1]+region1[3]), region1[0]:(region1[0]+region1[2])]) | |
std2 = np.std(gray[region2[1]:(region2[1]+region2[3]), region2[0]:(region2[0]+region2[2])]) | |
face_direction = "" | |
if std1 > std2: #eye right has higher variance than eye left | |
face_direction = "right" | |
else: | |
face_direction = "left" | |
return face_direction | |
def extract_cheek_region(face_x_min, face_x_max, face_y_max, eye_landmarks, left_or_right): | |
if left_or_right == "Left": | |
cheek_region_min_x = eye_landmarks[0,0] | |
cheek_region_max_x = int(face_x_max - 0.05 * (face_x_max - min(eye_landmarks[:,0]))) | |
else: | |
cheek_region_max_x = max(eye_landmarks[:,0])[0,0] | |
#print (max(eye_landmarks[:,0])[0,0]) | |
#cheek_region_max_x = max(eye_landmarks[:, 0]) | |
cheek_region_min_x = int(face_x_min + 0.1 * (cheek_region_max_x - face_x_min)) | |
cheek_region_min_y = int(max(eye_landmarks[:,1]) + 0.2 * (max(eye_landmarks[:,1]) - min(eye_landmarks[:,1]))) | |
cheek_region_max_y = int(face_y_max - 0.1 * (face_y_max - max(eye_landmarks[:,1]))) | |
return [cheek_region_min_x, cheek_region_min_y, cheek_region_max_x, cheek_region_max_y] | |
def extract_patches(imagefile, dimension_dict, face_loc_dict, image_dim, croppedFaces_Dir): | |
imageName = "temp" | |
img, landmarks = read_im_and_landmarks(imagefile) | |
face_detected = True | |
img_height, img_width = img.shape[0:2] | |
image_dim = [img_height, img_width] | |
min_dim = min(img_height, img_width) | |
min_face_size = min(min_dim * 0.2, min_dim * 0.2) | |
min_eye = min_face_size * 0.2 | |
min_eye_area = min_eye ** 2 | |
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) | |
if face_detected: | |
mask = get_face_mask(img, landmarks) | |
face_x_min = int(max(0, np.asarray(min(landmarks[:,0])).flatten()[0])) | |
face_x_max = int(min(img_width, np.asarray(max(landmarks[:,0])).flatten()[0])) | |
face_y_min = int(max(0, np.asarray(min(landmarks[:,1])).flatten()[0])) | |
face_y_max = int(min(img_height, np.asarray(max(landmarks[:,1])).flatten()[0])) | |
face_loc_dict['face_loc'] = [face_x_min, face_x_max, face_y_min, face_y_max] | |
face_height = face_y_max - face_y_min | |
forehead_height = int(face_height * forehead_ratio) | |
new_face_y_min = max(0, face_y_min - forehead_height) | |
right_brow_landmarks = landmarks[RIGHT_BROW_POINTS,:] | |
left_brow_landmarks = landmarks[LEFT_BROW_POINTS,:] | |
right_eye_landmarks = landmarks[RIGHT_EYE_POINTS,:] | |
left_eye_landmarks = landmarks[LEFT_EYE_POINTS,:] | |
mouse_landmarks = landmarks[MOUTH_POINTS,:] | |
######################## | |
# Get the forehead patch | |
######################## | |
[right_brow_min_x, left_brow_max_x] = \ | |
[max(0, np.min(np.array(right_brow_landmarks[:,0]))), min(img_width, np.max(np.array(left_brow_landmarks[:,0])))] | |
brow_min_y = min(np.min(np.array(right_brow_landmarks[:,1])),np.min(np.array(left_brow_landmarks[:,1]))) | |
forehead_x_min = right_brow_min_x | |
forehead_x_max = left_brow_max_x | |
forehead_y_min = max(0, brow_min_y - forehead_height) | |
forehead_y_max = min(brow_min_y, forehead_y_min + forehead_height) | |
forehead_region = img[forehead_y_min:forehead_y_max, forehead_x_min:forehead_x_max, :] | |
#print ('forehead dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (forehead_x_min, forehead_x_max, forehead_y_min, forehead_y_max)) | |
key_name = 'landmark_fh' | |
dimension_dict[key_name] = [forehead_x_min, forehead_x_max, forehead_y_min, forehead_y_max] | |
forehead_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
#forehead_region = cv2.cvtColor(forehead_region, cv2.COLOR_BGR2RGB) | |
imageio.imwrite(forehead_file_name, forehead_region) | |
chin_x_min = np.max(np.array(right_eye_landmarks[:,0])) | |
chin_x_max = np.min(np.array(left_eye_landmarks[:,0])) | |
chin_y_min = np.max(np.array(mouse_landmarks[:,1])) | |
chin_y_max = face_y_max | |
chin_region = img[chin_y_min:chin_y_max, chin_x_min:chin_x_max, :] | |
#print ('chin dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (chin_x_min, chin_x_max, chin_y_min, chin_y_max)) | |
key_name = 'landmark_chin' | |
dimension_dict[key_name] = [chin_x_min, chin_x_max, chin_y_min, chin_y_max] | |
chin_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
#chin_region = cv2.cvtColor(chin_region, cv2.COLOR_BGR2RGB) | |
imageio.imwrite(chin_file_name, chin_region) | |
########################## | |
# Get the cheeks patch | |
########################## | |
# Decide whether it is a side view or not | |
left_eye_width = np.max(np.array(left_eye_landmarks[:,0])) - np.min(np.array(left_eye_landmarks[:,0])) | |
right_eye_width = np.max(np.array(right_eye_landmarks[:,0])) - np.min(np.array(right_eye_landmarks[:,0])) | |
right_face = True | |
left_face = True | |
if float(right_eye_width) / float(left_eye_width) >= 1.15: # right eye is bigger than left eye, showing the right face | |
left_face = False | |
elif float(left_eye_width) / float(right_eye_width) >= 1.15: # left eye is bigger than right eye, showing the left face | |
right_face = False | |
if right_face: | |
right_cheek_region = extract_cheek_region(face_x_min, face_x_max, face_y_max, right_eye_landmarks, "Right") | |
cheek_region = img[right_cheek_region[1]:right_cheek_region[3], right_cheek_region[0]:right_cheek_region[2], :] | |
#print ('right cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (right_cheek_region[0], right_cheek_region[2], right_cheek_region[1], right_cheek_region[3])) | |
key_name = 'landmark_rc' | |
dimension_dict[key_name] = [right_cheek_region[0], right_cheek_region[2], right_cheek_region[1], right_cheek_region[3]] | |
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
#cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB) | |
imageio.imwrite(cheek_file_name, cheek_region) | |
if left_face: | |
left_cheek_region = extract_cheek_region(face_x_min, face_x_max, face_y_max, left_eye_landmarks, "Left") | |
cheek_region = img[left_cheek_region[1]:left_cheek_region[3], left_cheek_region[0]:left_cheek_region[2], :] | |
#print ('left cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (left_cheek_region[0], left_cheek_region[2], left_cheek_region[1], left_cheek_region[3])) | |
key_name = 'landmark_lc' | |
dimension_dict[key_name] = [left_cheek_region[0], left_cheek_region[2], left_cheek_region[1], left_cheek_region[3]] | |
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
#cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB) | |
imageio.imwrite(cheek_file_name, cheek_region) | |
if not face_detected: | |
print("Face not detected by landmarks model...") | |
# Use the OneEye model to detect one eye, and infer the face region based on the eye location | |
eye_detected = False | |
roi_gray = gray | |
roi_color = img | |
roi_color = cv2.cvtColor(roi_color, cv2.COLOR_BGR2RGB) | |
eyes = eye_cascade.detectMultiScale(roi_gray, 1.1, 5) | |
max_area = 0 | |
eye_count = 0 | |
max_index = 0 | |
for (ex,ey,ew,eh) in eyes: # there might be multiple eyes detected. Choose the biggest one | |
if ew*eh >= max_area and ex >= img_width * 0.1 and ex <= img_width * 0.9: | |
max_area = ew*eh | |
max_index = eye_count | |
eye_count += 1 | |
if max_area >= min_eye_area: | |
eye_detected = True | |
(ex, ey, ew, eh) = eyes[max_index] | |
if float(ew) / float(img_width) > 0.15 or float(eh) / float(img_height) > 0.15: # detected eye too large | |
# resize the detected eye | |
center_x = ex + ew/2 | |
center_y = ey + eh/2 | |
resized_w = min(img_width * 0.15, img_height * 0.15) | |
ex = int(center_x - resized_w/2) | |
ey = int(center_y - resized_w/2) | |
ew = int(resized_w) | |
eh = int(resized_w) | |
eyes1 = np.array([ex, ey, resized_w, resized_w]).reshape((1,4)) | |
else: | |
eyes1 = np.array(eyes[max_index]).reshape((1,4)) | |
face1 = np.array(()) | |
face_direction = detect_face_direction(gray, face1, eyes1, down_ratio, chin_width_ratio) | |
if face_direction == "left": | |
print("Left eye detected") | |
face_min_x = eyes1[0, 0] | |
face_max_x = min(img_width, int(eyes1[0,0] + (chin_width_ratio + 0.5) * eyes1[0, 2])) | |
forehead_max_x = min(img_width, int(eyes1[0,0] + width_ratio * eyes1[0, 2])) | |
forehead_min_x = face_min_x | |
cheek_min_x = int(eyes1[0, 0] + 0.5 * eyes1[0,2]) | |
cheek_max_x = face_max_x | |
else: | |
print("Right eye detected") | |
face_min_x = max(0, int(eyes1[0, 0] - chin_width_ratio * eyes1[0, 2])) | |
face_max_x = eyes1[0, 0] + eyes1[0, 2] | |
forehead_min_x = max(0, int(eyes1[0, 0] - width_ratio * eyes1[0, 2])) | |
forehead_max_x = min(img_width, int(eyes1[0, 0] + width_ratio * eyes1[0, 2])) | |
cheek_max_x = int(eyes1[0,0] + 0.5*eyes1[0,2]) | |
cheek_min_x = face_min_x | |
forehead_min_y = max(0, int(eyes1[0, 1] - top_ratio * eyes1[0,3])) | |
forehead_max_y = max(0, int(eyes1[0, 1] - 0.5 * eyes1[0, 3])) | |
forehead_ok = False | |
# Get the forehead region | |
if forehead_max_y - forehead_min_y >= 0.7 * eyes1[0, 3]: | |
forehead_ok = True | |
forehead_region = img[forehead_min_y:forehead_max_y, forehead_min_x: forehead_max_x, :] | |
#print ('forehead dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (forehead_min_x, forehead_max_x, forehead_min_y, forehead_max_y)) | |
key_name = 'oneeye_fh' | |
dimension_dict[key_name] = [forehead_min_x, forehead_max_x, forehead_min_y, forehead_max_y] | |
forehead_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
imageio.imwrite(forehead_file_name, forehead_region) | |
# Get the cheek region | |
cheek_min_y = int(eyes1[0, 1] + eyes1[0, 3]) | |
cheek_max_y = min(img_height, int(eyes1[0, 1] + down_ratio * eyes1[0, 3])) | |
cheek_region = img[cheek_min_y: cheek_max_y, cheek_min_x: cheek_max_x, :] | |
#print ('cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (cheek_min_x, cheek_max_x, cheek_min_y, cheek_max_y)) | |
key_name = 'oneeye_cheek' | |
dimension_dict[key_name] = [cheek_min_x, cheek_max_x, cheek_min_y, cheek_max_y] | |
face_loc_dict['face_loc'] = [face_min_x, face_max_x, forehead_min_y, cheek_max_y] | |
#cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB) | |
if face_direction == "left": | |
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
elif face_direction == "right": | |
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
else: | |
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg") | |
imageio.imwrite(cheek_file_name, cheek_region) | |
if (not face_detected) and (not eye_detected): | |
print("No chin or forehead detected, output the original file %s.jpg"%imageName) | |
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) | |
outfile = join(croppedFaces_Dir, imageName+".jpg") | |
imageio.imwrite(outfile, img) | |
return dimension_dict, face_loc_dict, image_dim |