Spaces:
Runtime error
Runtime error
add files
Browse files- download.py +4 -0
- run.py +23 -0
- run_sdk.py +11 -0
- source/__init__.py +0 -0
- source/cartoonize.py +120 -0
- source/facelib/LICENSE +4 -0
- source/facelib/LK/__init__.py +0 -0
- source/facelib/LK/lk.py +97 -0
- source/facelib/__init__.py +0 -0
- source/facelib/config.py +23 -0
- source/facelib/face_detector.py +116 -0
- source/facelib/face_landmark.py +154 -0
- source/facelib/facer.py +150 -0
- source/mtcnn_pytorch/LICENSE +21 -0
- source/mtcnn_pytorch/README.md +26 -0
- source/mtcnn_pytorch/__init__.py +0 -0
- source/mtcnn_pytorch/src/__init__.py +0 -0
- source/mtcnn_pytorch/src/align_trans.py +187 -0
- source/mtcnn_pytorch/src/matlab_cp2tform.py +339 -0
- source/utils.py +107 -0
download.py
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from modelscope.hub.snapshot_download import snapshot_download
|
2 |
+
model_dir = snapshot_download('damo/cv_unet_person-image-cartoon_compound-models', cache_dir='.')
|
3 |
+
|
4 |
+
|
run.py
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
import cv2
|
3 |
+
from source.cartoonize import Cartoonizer
|
4 |
+
import os
|
5 |
+
|
6 |
+
def process():
|
7 |
+
|
8 |
+
algo = Cartoonizer(dataroot='damo/cv_unet_person-image-cartoon_compound-models')
|
9 |
+
img = cv2.imread('input.png')[...,::-1]
|
10 |
+
|
11 |
+
result = algo.cartoonize(img)
|
12 |
+
|
13 |
+
cv2.imwrite('res.png', result)
|
14 |
+
print('finished!')
|
15 |
+
|
16 |
+
|
17 |
+
|
18 |
+
|
19 |
+
if __name__ == '__main__':
|
20 |
+
process()
|
21 |
+
|
22 |
+
|
23 |
+
|
run_sdk.py
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import cv2
|
2 |
+
from modelscope.outputs import OutputKeys
|
3 |
+
from modelscope.pipelines import pipeline
|
4 |
+
from modelscope.utils.constant import Tasks
|
5 |
+
|
6 |
+
img_cartoon = pipeline(Tasks.image_portrait_stylization,
|
7 |
+
model='damo/cv_unet_person-image-cartoon_compound-models')
|
8 |
+
result = img_cartoon('input.png')
|
9 |
+
|
10 |
+
cv2.imwrite('result.png', result[OutputKeys.OUTPUT_IMG])
|
11 |
+
print('finished!')
|
source/__init__.py
ADDED
File without changes
|
source/cartoonize.py
ADDED
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import cv2
|
3 |
+
import tensorflow as tf
|
4 |
+
import numpy as np
|
5 |
+
from source.facelib.facer import FaceAna
|
6 |
+
import source.utils as utils
|
7 |
+
from source.mtcnn_pytorch.src.align_trans import warp_and_crop_face, get_reference_facial_points
|
8 |
+
|
9 |
+
if tf.__version__ >= '2.0':
|
10 |
+
tf = tf.compat.v1
|
11 |
+
tf.disable_eager_execution()
|
12 |
+
|
13 |
+
|
14 |
+
class Cartoonizer():
|
15 |
+
def __init__(self, dataroot):
|
16 |
+
|
17 |
+
self.facer = FaceAna(dataroot)
|
18 |
+
self.sess_head = self.load_sess(
|
19 |
+
os.path.join(dataroot, 'cartoon_anime_h.pb'), 'model_head')
|
20 |
+
self.sess_bg = self.load_sess(
|
21 |
+
os.path.join(dataroot, 'cartoon_anime_bg.pb'), 'model_bg')
|
22 |
+
|
23 |
+
self.box_width = 288
|
24 |
+
global_mask = cv2.imread(os.path.join(dataroot, 'alpha.jpg'))
|
25 |
+
global_mask = cv2.resize(
|
26 |
+
global_mask, (self.box_width, self.box_width),
|
27 |
+
interpolation=cv2.INTER_AREA)
|
28 |
+
self.global_mask = cv2.cvtColor(
|
29 |
+
global_mask, cv2.COLOR_BGR2GRAY).astype(np.float32) / 255.0
|
30 |
+
|
31 |
+
def load_sess(self, model_path, name):
|
32 |
+
config = tf.ConfigProto(allow_soft_placement=True)
|
33 |
+
config.gpu_options.allow_growth = True
|
34 |
+
sess = tf.Session(config=config)
|
35 |
+
print(f'loading model from {model_path}')
|
36 |
+
with tf.gfile.FastGFile(model_path, 'rb') as f:
|
37 |
+
graph_def = tf.GraphDef()
|
38 |
+
graph_def.ParseFromString(f.read())
|
39 |
+
sess.graph.as_default()
|
40 |
+
tf.import_graph_def(graph_def, name=name)
|
41 |
+
sess.run(tf.global_variables_initializer())
|
42 |
+
print(f'load model {model_path} done.')
|
43 |
+
return sess
|
44 |
+
|
45 |
+
|
46 |
+
def detect_face(self, img):
|
47 |
+
src_h, src_w, _ = img.shape
|
48 |
+
src_x = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
49 |
+
boxes, landmarks, _ = self.facer.run(src_x)
|
50 |
+
if boxes.shape[0] == 0:
|
51 |
+
return None
|
52 |
+
else:
|
53 |
+
return landmarks
|
54 |
+
|
55 |
+
|
56 |
+
def cartoonize(self, img):
|
57 |
+
# img: RGB input
|
58 |
+
ori_h, ori_w, _ = img.shape
|
59 |
+
img = utils.resize_size(img, size=720)
|
60 |
+
|
61 |
+
img_brg = img[:, :, ::-1]
|
62 |
+
|
63 |
+
# background process
|
64 |
+
pad_bg, pad_h, pad_w = utils.padTo16x(img_brg)
|
65 |
+
|
66 |
+
bg_res = self.sess_bg.run(
|
67 |
+
self.sess_bg.graph.get_tensor_by_name(
|
68 |
+
'model_bg/output_image:0'),
|
69 |
+
feed_dict={'model_bg/input_image:0': pad_bg})
|
70 |
+
res = bg_res[:pad_h, :pad_w, :]
|
71 |
+
|
72 |
+
landmarks = self.detect_face(img_brg)
|
73 |
+
if landmarks is None:
|
74 |
+
print('No face detected!')
|
75 |
+
return res
|
76 |
+
|
77 |
+
print('%d faces detected!'%len(landmarks))
|
78 |
+
for landmark in landmarks:
|
79 |
+
# get facial 5 points
|
80 |
+
f5p = utils.get_f5p(landmark, img_brg)
|
81 |
+
|
82 |
+
# face alignment
|
83 |
+
head_img, trans_inv = warp_and_crop_face(
|
84 |
+
img,
|
85 |
+
f5p,
|
86 |
+
ratio=0.75,
|
87 |
+
reference_pts=get_reference_facial_points(default_square=True),
|
88 |
+
crop_size=(self.box_width, self.box_width),
|
89 |
+
return_trans_inv=True)
|
90 |
+
|
91 |
+
# head process
|
92 |
+
head_res = self.sess_head.run(
|
93 |
+
self.sess_head.graph.get_tensor_by_name(
|
94 |
+
'model_head/output_image:0'),
|
95 |
+
feed_dict={
|
96 |
+
'model_head/input_image:0': head_img[:, :, ::-1]
|
97 |
+
})
|
98 |
+
|
99 |
+
# merge head and background
|
100 |
+
head_trans_inv = cv2.warpAffine(
|
101 |
+
head_res,
|
102 |
+
trans_inv, (np.size(img, 1), np.size(img, 0)),
|
103 |
+
borderValue=(0, 0, 0))
|
104 |
+
|
105 |
+
mask = self.global_mask
|
106 |
+
mask_trans_inv = cv2.warpAffine(
|
107 |
+
mask,
|
108 |
+
trans_inv, (np.size(img, 1), np.size(img, 0)),
|
109 |
+
borderValue=(0, 0, 0))
|
110 |
+
mask_trans_inv = np.expand_dims(mask_trans_inv, 2)
|
111 |
+
|
112 |
+
res = mask_trans_inv * head_trans_inv + (1 - mask_trans_inv) * res
|
113 |
+
|
114 |
+
res = cv2.resize(res, (ori_w, ori_h), interpolation=cv2.INTER_AREA)
|
115 |
+
|
116 |
+
return res
|
117 |
+
|
118 |
+
|
119 |
+
|
120 |
+
|
source/facelib/LICENSE
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
Copyright (c) Peppa_Pig_Face_Engine
|
3 |
+
|
4 |
+
https://github.com/610265158/Peppa_Pig_Face_Engine
|
source/facelib/LK/__init__.py
ADDED
File without changes
|
source/facelib/LK/lk.py
ADDED
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
|
3 |
+
from modelscope.models.cv.cartoon.facelib.config import config as cfg
|
4 |
+
|
5 |
+
|
6 |
+
class GroupTrack():
|
7 |
+
|
8 |
+
def __init__(self):
|
9 |
+
self.old_frame = None
|
10 |
+
self.previous_landmarks_set = None
|
11 |
+
self.with_landmark = True
|
12 |
+
self.thres = cfg.TRACE.pixel_thres
|
13 |
+
self.alpha = cfg.TRACE.smooth_landmark
|
14 |
+
self.iou_thres = cfg.TRACE.iou_thres
|
15 |
+
|
16 |
+
def calculate(self, img, current_landmarks_set):
|
17 |
+
if self.previous_landmarks_set is None:
|
18 |
+
self.previous_landmarks_set = current_landmarks_set
|
19 |
+
result = current_landmarks_set
|
20 |
+
else:
|
21 |
+
previous_lm_num = self.previous_landmarks_set.shape[0]
|
22 |
+
if previous_lm_num == 0:
|
23 |
+
self.previous_landmarks_set = current_landmarks_set
|
24 |
+
result = current_landmarks_set
|
25 |
+
return result
|
26 |
+
else:
|
27 |
+
result = []
|
28 |
+
for i in range(current_landmarks_set.shape[0]):
|
29 |
+
not_in_flag = True
|
30 |
+
for j in range(previous_lm_num):
|
31 |
+
if self.iou(current_landmarks_set[i],
|
32 |
+
self.previous_landmarks_set[j]
|
33 |
+
) > self.iou_thres:
|
34 |
+
result.append(
|
35 |
+
self.smooth(current_landmarks_set[i],
|
36 |
+
self.previous_landmarks_set[j]))
|
37 |
+
not_in_flag = False
|
38 |
+
break
|
39 |
+
if not_in_flag:
|
40 |
+
result.append(current_landmarks_set[i])
|
41 |
+
|
42 |
+
result = np.array(result)
|
43 |
+
self.previous_landmarks_set = result
|
44 |
+
|
45 |
+
return result
|
46 |
+
|
47 |
+
def iou(self, p_set0, p_set1):
|
48 |
+
rec1 = [
|
49 |
+
np.min(p_set0[:, 0]),
|
50 |
+
np.min(p_set0[:, 1]),
|
51 |
+
np.max(p_set0[:, 0]),
|
52 |
+
np.max(p_set0[:, 1])
|
53 |
+
]
|
54 |
+
rec2 = [
|
55 |
+
np.min(p_set1[:, 0]),
|
56 |
+
np.min(p_set1[:, 1]),
|
57 |
+
np.max(p_set1[:, 0]),
|
58 |
+
np.max(p_set1[:, 1])
|
59 |
+
]
|
60 |
+
|
61 |
+
# computing area of each rectangles
|
62 |
+
S_rec1 = (rec1[2] - rec1[0]) * (rec1[3] - rec1[1])
|
63 |
+
S_rec2 = (rec2[2] - rec2[0]) * (rec2[3] - rec2[1])
|
64 |
+
|
65 |
+
# computing the sum_area
|
66 |
+
sum_area = S_rec1 + S_rec2
|
67 |
+
|
68 |
+
# find the each edge of intersect rectangle
|
69 |
+
x1 = max(rec1[0], rec2[0])
|
70 |
+
y1 = max(rec1[1], rec2[1])
|
71 |
+
x2 = min(rec1[2], rec2[2])
|
72 |
+
y2 = min(rec1[3], rec2[3])
|
73 |
+
|
74 |
+
# judge if there is an intersect
|
75 |
+
intersect = max(0, x2 - x1) * max(0, y2 - y1)
|
76 |
+
|
77 |
+
iou = intersect / (sum_area - intersect)
|
78 |
+
return iou
|
79 |
+
|
80 |
+
def smooth(self, now_landmarks, previous_landmarks):
|
81 |
+
result = []
|
82 |
+
for i in range(now_landmarks.shape[0]):
|
83 |
+
x = now_landmarks[i][0] - previous_landmarks[i][0]
|
84 |
+
y = now_landmarks[i][1] - previous_landmarks[i][1]
|
85 |
+
dis = np.sqrt(np.square(x) + np.square(y))
|
86 |
+
if dis < self.thres:
|
87 |
+
result.append(previous_landmarks[i])
|
88 |
+
else:
|
89 |
+
result.append(
|
90 |
+
self.do_moving_average(now_landmarks[i],
|
91 |
+
previous_landmarks[i]))
|
92 |
+
|
93 |
+
return np.array(result)
|
94 |
+
|
95 |
+
def do_moving_average(self, p_now, p_previous):
|
96 |
+
p = self.alpha * p_now + (1 - self.alpha) * p_previous
|
97 |
+
return p
|
source/facelib/__init__.py
ADDED
File without changes
|
source/facelib/config.py
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
|
3 |
+
import numpy as np
|
4 |
+
from easydict import EasyDict as edict
|
5 |
+
|
6 |
+
config = edict()
|
7 |
+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
|
8 |
+
|
9 |
+
config.DETECT = edict()
|
10 |
+
config.DETECT.topk = 10
|
11 |
+
config.DETECT.thres = 0.8
|
12 |
+
config.DETECT.input_shape = (512, 512, 3)
|
13 |
+
config.KEYPOINTS = edict()
|
14 |
+
config.KEYPOINTS.p_num = 68
|
15 |
+
config.KEYPOINTS.base_extend_range = [0.2, 0.3]
|
16 |
+
config.KEYPOINTS.input_shape = (160, 160, 3)
|
17 |
+
config.TRACE = edict()
|
18 |
+
config.TRACE.pixel_thres = 1
|
19 |
+
config.TRACE.smooth_box = 0.3
|
20 |
+
config.TRACE.smooth_landmark = 0.95
|
21 |
+
config.TRACE.iou_thres = 0.5
|
22 |
+
config.DATA = edict()
|
23 |
+
config.DATA.pixel_means = np.array([123., 116., 103.]) # RGB
|
source/facelib/face_detector.py
ADDED
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import time
|
2 |
+
|
3 |
+
import cv2
|
4 |
+
import numpy as np
|
5 |
+
import tensorflow as tf
|
6 |
+
|
7 |
+
from .config import config as cfg
|
8 |
+
|
9 |
+
if tf.__version__ >= '2.0':
|
10 |
+
tf = tf.compat.v1
|
11 |
+
|
12 |
+
|
13 |
+
class FaceDetector:
|
14 |
+
|
15 |
+
def __init__(self, dir):
|
16 |
+
|
17 |
+
self.model_path = dir + '/detector.pb'
|
18 |
+
self.thres = cfg.DETECT.thres
|
19 |
+
self.input_shape = cfg.DETECT.input_shape
|
20 |
+
|
21 |
+
self._graph = tf.Graph()
|
22 |
+
|
23 |
+
with self._graph.as_default():
|
24 |
+
self._graph, self._sess = self.init_model(self.model_path)
|
25 |
+
|
26 |
+
self.input_image = tf.get_default_graph().get_tensor_by_name(
|
27 |
+
'tower_0/images:0')
|
28 |
+
self.training = tf.get_default_graph().get_tensor_by_name(
|
29 |
+
'training_flag:0')
|
30 |
+
self.output_ops = [
|
31 |
+
tf.get_default_graph().get_tensor_by_name('tower_0/boxes:0'),
|
32 |
+
tf.get_default_graph().get_tensor_by_name('tower_0/scores:0'),
|
33 |
+
tf.get_default_graph().get_tensor_by_name(
|
34 |
+
'tower_0/num_detections:0'),
|
35 |
+
]
|
36 |
+
|
37 |
+
def __call__(self, image):
|
38 |
+
|
39 |
+
image, scale_x, scale_y = self.preprocess(
|
40 |
+
image,
|
41 |
+
target_width=self.input_shape[1],
|
42 |
+
target_height=self.input_shape[0])
|
43 |
+
|
44 |
+
image = np.expand_dims(image, 0)
|
45 |
+
|
46 |
+
boxes, scores, num_boxes = self._sess.run(
|
47 |
+
self.output_ops,
|
48 |
+
feed_dict={
|
49 |
+
self.input_image: image,
|
50 |
+
self.training: False
|
51 |
+
})
|
52 |
+
|
53 |
+
num_boxes = num_boxes[0]
|
54 |
+
boxes = boxes[0][:num_boxes]
|
55 |
+
|
56 |
+
scores = scores[0][:num_boxes]
|
57 |
+
|
58 |
+
to_keep = scores > self.thres
|
59 |
+
boxes = boxes[to_keep]
|
60 |
+
scores = scores[to_keep]
|
61 |
+
|
62 |
+
y1 = self.input_shape[0] / scale_y
|
63 |
+
x1 = self.input_shape[1] / scale_x
|
64 |
+
y2 = self.input_shape[0] / scale_y
|
65 |
+
x2 = self.input_shape[1] / scale_x
|
66 |
+
scaler = np.array([y1, x1, y2, x2], dtype='float32')
|
67 |
+
boxes = boxes * scaler
|
68 |
+
|
69 |
+
scores = np.expand_dims(scores, 0).reshape([-1, 1])
|
70 |
+
|
71 |
+
for i in range(boxes.shape[0]):
|
72 |
+
boxes[i] = np.array(
|
73 |
+
[boxes[i][1], boxes[i][0], boxes[i][3], boxes[i][2]])
|
74 |
+
return np.concatenate([boxes, scores], axis=1)
|
75 |
+
|
76 |
+
def preprocess(self, image, target_height, target_width, label=None):
|
77 |
+
|
78 |
+
h, w, c = image.shape
|
79 |
+
|
80 |
+
bimage = np.zeros(
|
81 |
+
shape=[target_height, target_width, c],
|
82 |
+
dtype=image.dtype) + np.array(
|
83 |
+
cfg.DATA.pixel_means, dtype=image.dtype)
|
84 |
+
long_side = max(h, w)
|
85 |
+
|
86 |
+
scale_x = scale_y = target_height / long_side
|
87 |
+
|
88 |
+
image = cv2.resize(image, None, fx=scale_x, fy=scale_y)
|
89 |
+
|
90 |
+
h_, w_, _ = image.shape
|
91 |
+
bimage[:h_, :w_, :] = image
|
92 |
+
|
93 |
+
return bimage, scale_x, scale_y
|
94 |
+
|
95 |
+
def init_model(self, *args):
|
96 |
+
pb_path = args[0]
|
97 |
+
|
98 |
+
def init_pb(model_path):
|
99 |
+
config = tf.ConfigProto()
|
100 |
+
config.gpu_options.per_process_gpu_memory_fraction = 0.2
|
101 |
+
compute_graph = tf.Graph()
|
102 |
+
compute_graph.as_default()
|
103 |
+
sess = tf.Session(config=config)
|
104 |
+
with tf.gfile.GFile(model_path, 'rb') as fid:
|
105 |
+
graph_def = tf.GraphDef()
|
106 |
+
graph_def.ParseFromString(fid.read())
|
107 |
+
tf.import_graph_def(graph_def, name='')
|
108 |
+
|
109 |
+
return (compute_graph, sess)
|
110 |
+
|
111 |
+
model = init_pb(pb_path)
|
112 |
+
|
113 |
+
graph = model[0]
|
114 |
+
sess = model[1]
|
115 |
+
|
116 |
+
return graph, sess
|
source/facelib/face_landmark.py
ADDED
@@ -0,0 +1,154 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import cv2
|
2 |
+
import numpy as np
|
3 |
+
import tensorflow as tf
|
4 |
+
|
5 |
+
from .config import config as cfg
|
6 |
+
|
7 |
+
if tf.__version__ >= '2.0':
|
8 |
+
tf = tf.compat.v1
|
9 |
+
|
10 |
+
|
11 |
+
class FaceLandmark:
|
12 |
+
|
13 |
+
def __init__(self, dir):
|
14 |
+
self.model_path = dir + '/keypoints.pb'
|
15 |
+
self.min_face = 60
|
16 |
+
self.keypoint_num = cfg.KEYPOINTS.p_num * 2
|
17 |
+
|
18 |
+
self._graph = tf.Graph()
|
19 |
+
|
20 |
+
with self._graph.as_default():
|
21 |
+
|
22 |
+
self._graph, self._sess = self.init_model(self.model_path)
|
23 |
+
self.img_input = tf.get_default_graph().get_tensor_by_name(
|
24 |
+
'tower_0/images:0')
|
25 |
+
self.embeddings = tf.get_default_graph().get_tensor_by_name(
|
26 |
+
'tower_0/prediction:0')
|
27 |
+
self.training = tf.get_default_graph().get_tensor_by_name(
|
28 |
+
'training_flag:0')
|
29 |
+
|
30 |
+
self.landmark = self.embeddings[:, :self.keypoint_num]
|
31 |
+
self.headpose = self.embeddings[:, -7:-4] * 90.
|
32 |
+
self.state = tf.nn.sigmoid(self.embeddings[:, -4:])
|
33 |
+
|
34 |
+
def __call__(self, img, bboxes):
|
35 |
+
landmark_result = []
|
36 |
+
state_result = []
|
37 |
+
for i, bbox in enumerate(bboxes):
|
38 |
+
landmark, state = self._one_shot_run(img, bbox, i)
|
39 |
+
if landmark is not None:
|
40 |
+
landmark_result.append(landmark)
|
41 |
+
state_result.append(state)
|
42 |
+
return np.array(landmark_result), np.array(state_result)
|
43 |
+
|
44 |
+
def simple_run(self, cropped_img):
|
45 |
+
with self._graph.as_default():
|
46 |
+
|
47 |
+
cropped_img = np.expand_dims(cropped_img, axis=0)
|
48 |
+
landmark, p, states = self._sess.run(
|
49 |
+
[self.landmark, self.headpose, self.state],
|
50 |
+
feed_dict={
|
51 |
+
self.img_input: cropped_img,
|
52 |
+
self.training: False
|
53 |
+
})
|
54 |
+
|
55 |
+
return landmark, states
|
56 |
+
|
57 |
+
def _one_shot_run(self, image, bbox, i):
|
58 |
+
|
59 |
+
bbox_width = bbox[2] - bbox[0]
|
60 |
+
bbox_height = bbox[3] - bbox[1]
|
61 |
+
if (bbox_width <= self.min_face and bbox_height <= self.min_face):
|
62 |
+
return None, None
|
63 |
+
add = int(max(bbox_width, bbox_height))
|
64 |
+
bimg = cv2.copyMakeBorder(
|
65 |
+
image,
|
66 |
+
add,
|
67 |
+
add,
|
68 |
+
add,
|
69 |
+
add,
|
70 |
+
borderType=cv2.BORDER_CONSTANT,
|
71 |
+
value=cfg.DATA.pixel_means)
|
72 |
+
bbox += add
|
73 |
+
|
74 |
+
one_edge = (1 + 2 * cfg.KEYPOINTS.base_extend_range[0]) * bbox_width
|
75 |
+
center = [(bbox[0] + bbox[2]) // 2, (bbox[1] + bbox[3]) // 2]
|
76 |
+
|
77 |
+
bbox[0] = center[0] - one_edge // 2
|
78 |
+
bbox[1] = center[1] - one_edge // 2
|
79 |
+
bbox[2] = center[0] + one_edge // 2
|
80 |
+
bbox[3] = center[1] + one_edge // 2
|
81 |
+
|
82 |
+
bbox = bbox.astype(np.int)
|
83 |
+
crop_image = bimg[bbox[1]:bbox[3], bbox[0]:bbox[2], :]
|
84 |
+
h, w, _ = crop_image.shape
|
85 |
+
crop_image = cv2.resize(
|
86 |
+
crop_image,
|
87 |
+
(cfg.KEYPOINTS.input_shape[1], cfg.KEYPOINTS.input_shape[0]))
|
88 |
+
crop_image = crop_image.astype(np.float32)
|
89 |
+
|
90 |
+
keypoints, state = self.simple_run(crop_image)
|
91 |
+
|
92 |
+
res = keypoints[0][:self.keypoint_num].reshape((-1, 2))
|
93 |
+
res[:, 0] = res[:, 0] * w / cfg.KEYPOINTS.input_shape[1]
|
94 |
+
res[:, 1] = res[:, 1] * h / cfg.KEYPOINTS.input_shape[0]
|
95 |
+
|
96 |
+
landmark = []
|
97 |
+
for _index in range(res.shape[0]):
|
98 |
+
x_y = res[_index]
|
99 |
+
landmark.append([
|
100 |
+
int(x_y[0] * cfg.KEYPOINTS.input_shape[0] + bbox[0] - add),
|
101 |
+
int(x_y[1] * cfg.KEYPOINTS.input_shape[1] + bbox[1] - add)
|
102 |
+
])
|
103 |
+
|
104 |
+
landmark = np.array(landmark, np.float32)
|
105 |
+
|
106 |
+
return landmark, state
|
107 |
+
|
108 |
+
def init_model(self, *args):
|
109 |
+
|
110 |
+
if len(args) == 1:
|
111 |
+
use_pb = True
|
112 |
+
pb_path = args[0]
|
113 |
+
else:
|
114 |
+
use_pb = False
|
115 |
+
meta_path = args[0]
|
116 |
+
restore_model_path = args[1]
|
117 |
+
|
118 |
+
def ini_ckpt():
|
119 |
+
graph = tf.Graph()
|
120 |
+
graph.as_default()
|
121 |
+
configProto = tf.ConfigProto()
|
122 |
+
configProto.gpu_options.allow_growth = True
|
123 |
+
sess = tf.Session(config=configProto)
|
124 |
+
# load_model(model_path, sess)
|
125 |
+
saver = tf.train.import_meta_graph(meta_path)
|
126 |
+
saver.restore(sess, restore_model_path)
|
127 |
+
|
128 |
+
print('Model restred!')
|
129 |
+
return (graph, sess)
|
130 |
+
|
131 |
+
def init_pb(model_path):
|
132 |
+
config = tf.ConfigProto()
|
133 |
+
config.gpu_options.per_process_gpu_memory_fraction = 0.2
|
134 |
+
compute_graph = tf.Graph()
|
135 |
+
compute_graph.as_default()
|
136 |
+
sess = tf.Session(config=config)
|
137 |
+
with tf.gfile.GFile(model_path, 'rb') as fid:
|
138 |
+
graph_def = tf.GraphDef()
|
139 |
+
graph_def.ParseFromString(fid.read())
|
140 |
+
tf.import_graph_def(graph_def, name='')
|
141 |
+
|
142 |
+
# saver = tf.train.Saver(tf.global_variables())
|
143 |
+
# saver.save(sess, save_path='./tmp.ckpt')
|
144 |
+
return (compute_graph, sess)
|
145 |
+
|
146 |
+
if use_pb:
|
147 |
+
model = init_pb(pb_path)
|
148 |
+
else:
|
149 |
+
model = ini_ckpt()
|
150 |
+
|
151 |
+
graph = model[0]
|
152 |
+
sess = model[1]
|
153 |
+
|
154 |
+
return graph, sess
|
source/facelib/facer.py
ADDED
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import time
|
2 |
+
|
3 |
+
import cv2
|
4 |
+
import numpy as np
|
5 |
+
|
6 |
+
from .config import config as cfg
|
7 |
+
from .face_detector import FaceDetector
|
8 |
+
from .face_landmark import FaceLandmark
|
9 |
+
from .LK.lk import GroupTrack
|
10 |
+
|
11 |
+
|
12 |
+
class FaceAna():
|
13 |
+
'''
|
14 |
+
by default the top3 facea sorted by area will be calculated for time reason
|
15 |
+
'''
|
16 |
+
|
17 |
+
def __init__(self, model_dir):
|
18 |
+
self.face_detector = FaceDetector(model_dir)
|
19 |
+
self.face_landmark = FaceLandmark(model_dir)
|
20 |
+
self.trace = GroupTrack()
|
21 |
+
|
22 |
+
self.track_box = None
|
23 |
+
self.previous_image = None
|
24 |
+
self.previous_box = None
|
25 |
+
|
26 |
+
self.diff_thres = 5
|
27 |
+
self.top_k = cfg.DETECT.topk
|
28 |
+
self.iou_thres = cfg.TRACE.iou_thres
|
29 |
+
self.alpha = cfg.TRACE.smooth_box
|
30 |
+
|
31 |
+
def run(self, image):
|
32 |
+
|
33 |
+
boxes = self.face_detector(image)
|
34 |
+
|
35 |
+
if boxes.shape[0] > self.top_k:
|
36 |
+
boxes = self.sort(boxes)
|
37 |
+
|
38 |
+
boxes_return = np.array(boxes)
|
39 |
+
landmarks, states = self.face_landmark(image, boxes)
|
40 |
+
|
41 |
+
if 1:
|
42 |
+
track = []
|
43 |
+
for i in range(landmarks.shape[0]):
|
44 |
+
track.append([
|
45 |
+
np.min(landmarks[i][:, 0]),
|
46 |
+
np.min(landmarks[i][:, 1]),
|
47 |
+
np.max(landmarks[i][:, 0]),
|
48 |
+
np.max(landmarks[i][:, 1])
|
49 |
+
])
|
50 |
+
tmp_box = np.array(track)
|
51 |
+
|
52 |
+
self.track_box = self.judge_boxs(boxes_return, tmp_box)
|
53 |
+
|
54 |
+
self.track_box, landmarks = self.sort_res(self.track_box, landmarks)
|
55 |
+
return self.track_box, landmarks, states
|
56 |
+
|
57 |
+
def sort_res(self, bboxes, points):
|
58 |
+
area = []
|
59 |
+
for bbox in bboxes:
|
60 |
+
bbox_width = bbox[2] - bbox[0]
|
61 |
+
bbox_height = bbox[3] - bbox[1]
|
62 |
+
area.append(bbox_height * bbox_width)
|
63 |
+
|
64 |
+
area = np.array(area)
|
65 |
+
picked = area.argsort()[::-1]
|
66 |
+
sorted_bboxes = [bboxes[x] for x in picked]
|
67 |
+
sorted_points = [points[x] for x in picked]
|
68 |
+
return np.array(sorted_bboxes), np.array(sorted_points)
|
69 |
+
|
70 |
+
def diff_frames(self, previous_frame, image):
|
71 |
+
if previous_frame is None:
|
72 |
+
return True
|
73 |
+
else:
|
74 |
+
_diff = cv2.absdiff(previous_frame, image)
|
75 |
+
diff = np.sum(
|
76 |
+
_diff) / previous_frame.shape[0] / previous_frame.shape[1] / 3.
|
77 |
+
return diff > self.diff_thres
|
78 |
+
|
79 |
+
def sort(self, bboxes):
|
80 |
+
if self.top_k > 100:
|
81 |
+
return bboxes
|
82 |
+
area = []
|
83 |
+
for bbox in bboxes:
|
84 |
+
|
85 |
+
bbox_width = bbox[2] - bbox[0]
|
86 |
+
bbox_height = bbox[3] - bbox[1]
|
87 |
+
area.append(bbox_height * bbox_width)
|
88 |
+
|
89 |
+
area = np.array(area)
|
90 |
+
|
91 |
+
picked = area.argsort()[-self.top_k:][::-1]
|
92 |
+
sorted_bboxes = [bboxes[x] for x in picked]
|
93 |
+
return np.array(sorted_bboxes)
|
94 |
+
|
95 |
+
def judge_boxs(self, previuous_bboxs, now_bboxs):
|
96 |
+
|
97 |
+
def iou(rec1, rec2):
|
98 |
+
|
99 |
+
# computing area of each rectangles
|
100 |
+
S_rec1 = (rec1[2] - rec1[0]) * (rec1[3] - rec1[1])
|
101 |
+
S_rec2 = (rec2[2] - rec2[0]) * (rec2[3] - rec2[1])
|
102 |
+
|
103 |
+
# computing the sum_area
|
104 |
+
sum_area = S_rec1 + S_rec2
|
105 |
+
|
106 |
+
# find the each edge of intersect rectangle
|
107 |
+
x1 = max(rec1[0], rec2[0])
|
108 |
+
y1 = max(rec1[1], rec2[1])
|
109 |
+
x2 = min(rec1[2], rec2[2])
|
110 |
+
y2 = min(rec1[3], rec2[3])
|
111 |
+
|
112 |
+
# judge if there is an intersect
|
113 |
+
intersect = max(0, x2 - x1) * max(0, y2 - y1)
|
114 |
+
|
115 |
+
return intersect / (sum_area - intersect)
|
116 |
+
|
117 |
+
if previuous_bboxs is None:
|
118 |
+
return now_bboxs
|
119 |
+
|
120 |
+
result = []
|
121 |
+
|
122 |
+
for i in range(now_bboxs.shape[0]):
|
123 |
+
contain = False
|
124 |
+
for j in range(previuous_bboxs.shape[0]):
|
125 |
+
if iou(now_bboxs[i], previuous_bboxs[j]) > self.iou_thres:
|
126 |
+
result.append(
|
127 |
+
self.smooth(now_bboxs[i], previuous_bboxs[j]))
|
128 |
+
contain = True
|
129 |
+
break
|
130 |
+
if not contain:
|
131 |
+
result.append(now_bboxs[i])
|
132 |
+
|
133 |
+
return np.array(result)
|
134 |
+
|
135 |
+
def smooth(self, now_box, previous_box):
|
136 |
+
|
137 |
+
return self.do_moving_average(now_box[:4], previous_box[:4])
|
138 |
+
|
139 |
+
def do_moving_average(self, p_now, p_previous):
|
140 |
+
p = self.alpha * p_now + (1 - self.alpha) * p_previous
|
141 |
+
return p
|
142 |
+
|
143 |
+
def reset(self):
|
144 |
+
'''
|
145 |
+
reset the previous info used foe tracking,
|
146 |
+
:return:
|
147 |
+
'''
|
148 |
+
self.track_box = None
|
149 |
+
self.previous_image = None
|
150 |
+
self.previous_box = None
|
source/mtcnn_pytorch/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
MIT License
|
2 |
+
|
3 |
+
Copyright (c) 2017 Dan Antoshchenko
|
4 |
+
|
5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6 |
+
of this software and associated documentation files (the "Software"), to deal
|
7 |
+
in the Software without restriction, including without limitation the rights
|
8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9 |
+
copies of the Software, and to permit persons to whom the Software is
|
10 |
+
furnished to do so, subject to the following conditions:
|
11 |
+
|
12 |
+
The above copyright notice and this permission notice shall be included in all
|
13 |
+
copies or substantial portions of the Software.
|
14 |
+
|
15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21 |
+
SOFTWARE.
|
source/mtcnn_pytorch/README.md
ADDED
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# MTCNN
|
2 |
+
|
3 |
+
`pytorch` implementation of **inference stage** of face detection algorithm described in
|
4 |
+
[Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Networks](https://arxiv.org/abs/1604.02878).
|
5 |
+
|
6 |
+
## Example
|
7 |
+
![example of a face detection](images/example.png)
|
8 |
+
|
9 |
+
## How to use it
|
10 |
+
Just download the repository and then do this
|
11 |
+
```python
|
12 |
+
from src import detect_faces
|
13 |
+
from PIL import Image
|
14 |
+
|
15 |
+
image = Image.open('image.jpg')
|
16 |
+
bounding_boxes, landmarks = detect_faces(image)
|
17 |
+
```
|
18 |
+
For examples see `test_on_images.ipynb`.
|
19 |
+
|
20 |
+
## Requirements
|
21 |
+
* pytorch 0.2
|
22 |
+
* Pillow, numpy
|
23 |
+
|
24 |
+
## Credit
|
25 |
+
This implementation is heavily inspired by:
|
26 |
+
* [pangyupo/mxnet_mtcnn_face_detection](https://github.com/pangyupo/mxnet_mtcnn_face_detection)
|
source/mtcnn_pytorch/__init__.py
ADDED
File without changes
|
source/mtcnn_pytorch/src/__init__.py
ADDED
File without changes
|
source/mtcnn_pytorch/src/align_trans.py
ADDED
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
Created on Mon Apr 24 15:43:29 2017
|
3 |
+
@author: zhaoy
|
4 |
+
"""
|
5 |
+
import cv2
|
6 |
+
import numpy as np
|
7 |
+
|
8 |
+
from .matlab_cp2tform import get_similarity_transform_for_cv2
|
9 |
+
|
10 |
+
# reference facial points, a list of coordinates (x,y)
|
11 |
+
dx = 1
|
12 |
+
dy = 1
|
13 |
+
REFERENCE_FACIAL_POINTS = [
|
14 |
+
[30.29459953 + dx, 51.69630051 + dy], # left eye
|
15 |
+
[65.53179932 + dx, 51.50139999 + dy], # right eye
|
16 |
+
[48.02519989 + dx, 71.73660278 + dy], # nose
|
17 |
+
[33.54930115 + dx, 92.3655014 + dy], # left mouth
|
18 |
+
[62.72990036 + dx, 92.20410156 + dy] # right mouth
|
19 |
+
]
|
20 |
+
|
21 |
+
DEFAULT_CROP_SIZE = (96, 112)
|
22 |
+
|
23 |
+
global FACIAL_POINTS
|
24 |
+
|
25 |
+
|
26 |
+
class FaceWarpException(Exception):
|
27 |
+
|
28 |
+
def __str__(self):
|
29 |
+
return 'In File {}:{}'.format(__file__, super.__str__(self))
|
30 |
+
|
31 |
+
|
32 |
+
def get_reference_facial_points(output_size=None,
|
33 |
+
inner_padding_factor=0.0,
|
34 |
+
outer_padding=(0, 0),
|
35 |
+
default_square=False):
|
36 |
+
|
37 |
+
tmp_5pts = np.array(REFERENCE_FACIAL_POINTS)
|
38 |
+
tmp_crop_size = np.array(DEFAULT_CROP_SIZE)
|
39 |
+
|
40 |
+
# 0) make the inner region a square
|
41 |
+
if default_square:
|
42 |
+
size_diff = max(tmp_crop_size) - tmp_crop_size
|
43 |
+
tmp_5pts += size_diff / 2
|
44 |
+
tmp_crop_size += size_diff
|
45 |
+
|
46 |
+
h_crop = tmp_crop_size[0]
|
47 |
+
w_crop = tmp_crop_size[1]
|
48 |
+
if (output_size):
|
49 |
+
if (output_size[0] == h_crop and output_size[1] == w_crop):
|
50 |
+
return tmp_5pts
|
51 |
+
|
52 |
+
if (inner_padding_factor == 0 and outer_padding == (0, 0)):
|
53 |
+
if output_size is None:
|
54 |
+
return tmp_5pts
|
55 |
+
else:
|
56 |
+
raise FaceWarpException(
|
57 |
+
'No paddings to do, output_size must be None or {}'.format(
|
58 |
+
tmp_crop_size))
|
59 |
+
|
60 |
+
# check output size
|
61 |
+
if not (0 <= inner_padding_factor <= 1.0):
|
62 |
+
raise FaceWarpException('Not (0 <= inner_padding_factor <= 1.0)')
|
63 |
+
|
64 |
+
factor = inner_padding_factor > 0 or outer_padding[0] > 0
|
65 |
+
factor = factor or outer_padding[1] > 0
|
66 |
+
if (factor and output_size is None):
|
67 |
+
output_size = tmp_crop_size * \
|
68 |
+
(1 + inner_padding_factor * 2).astype(np.int32)
|
69 |
+
output_size += np.array(outer_padding)
|
70 |
+
|
71 |
+
cond1 = outer_padding[0] < output_size[0]
|
72 |
+
cond2 = outer_padding[1] < output_size[1]
|
73 |
+
if not (cond1 and cond2):
|
74 |
+
raise FaceWarpException('Not (outer_padding[0] < output_size[0]'
|
75 |
+
'and outer_padding[1] < output_size[1])')
|
76 |
+
|
77 |
+
# 1) pad the inner region according inner_padding_factor
|
78 |
+
if inner_padding_factor > 0:
|
79 |
+
size_diff = tmp_crop_size * inner_padding_factor * 2
|
80 |
+
tmp_5pts += size_diff / 2
|
81 |
+
tmp_crop_size += np.round(size_diff).astype(np.int32)
|
82 |
+
|
83 |
+
# 2) resize the padded inner region
|
84 |
+
size_bf_outer_pad = np.array(output_size) - np.array(outer_padding) * 2
|
85 |
+
|
86 |
+
if size_bf_outer_pad[0] * tmp_crop_size[1] != size_bf_outer_pad[
|
87 |
+
1] * tmp_crop_size[0]:
|
88 |
+
raise FaceWarpException(
|
89 |
+
'Must have (output_size - outer_padding)'
|
90 |
+
'= some_scale * (crop_size * (1.0 + inner_padding_factor)')
|
91 |
+
|
92 |
+
scale_factor = size_bf_outer_pad[0].astype(np.float32) / tmp_crop_size[0]
|
93 |
+
tmp_5pts = tmp_5pts * scale_factor
|
94 |
+
|
95 |
+
# 3) add outer_padding to make output_size
|
96 |
+
reference_5point = tmp_5pts + np.array(outer_padding)
|
97 |
+
|
98 |
+
return reference_5point
|
99 |
+
|
100 |
+
|
101 |
+
def get_affine_transform_matrix(src_pts, dst_pts):
|
102 |
+
|
103 |
+
tfm = np.float32([[1, 0, 0], [0, 1, 0]])
|
104 |
+
n_pts = src_pts.shape[0]
|
105 |
+
ones = np.ones((n_pts, 1), src_pts.dtype)
|
106 |
+
src_pts_ = np.hstack([src_pts, ones])
|
107 |
+
dst_pts_ = np.hstack([dst_pts, ones])
|
108 |
+
|
109 |
+
A, res, rank, s = np.linalg.lstsq(src_pts_, dst_pts_)
|
110 |
+
|
111 |
+
if rank == 3:
|
112 |
+
tfm = np.float32([[A[0, 0], A[1, 0], A[2, 0]],
|
113 |
+
[A[0, 1], A[1, 1], A[2, 1]]])
|
114 |
+
elif rank == 2:
|
115 |
+
tfm = np.float32([[A[0, 0], A[1, 0], 0], [A[0, 1], A[1, 1], 0]])
|
116 |
+
|
117 |
+
return tfm
|
118 |
+
|
119 |
+
|
120 |
+
def warp_and_crop_face(src_img,
|
121 |
+
facial_pts,
|
122 |
+
ratio=0.84,
|
123 |
+
reference_pts=None,
|
124 |
+
crop_size=(96, 112),
|
125 |
+
align_type='similarity'
|
126 |
+
'',
|
127 |
+
return_trans_inv=False):
|
128 |
+
|
129 |
+
if reference_pts is None:
|
130 |
+
if crop_size[0] == 96 and crop_size[1] == 112:
|
131 |
+
reference_pts = REFERENCE_FACIAL_POINTS
|
132 |
+
else:
|
133 |
+
default_square = False
|
134 |
+
inner_padding_factor = 0
|
135 |
+
outer_padding = (0, 0)
|
136 |
+
output_size = crop_size
|
137 |
+
|
138 |
+
reference_pts = get_reference_facial_points(
|
139 |
+
output_size, inner_padding_factor, outer_padding,
|
140 |
+
default_square)
|
141 |
+
|
142 |
+
ref_pts = np.float32(reference_pts)
|
143 |
+
|
144 |
+
factor = ratio
|
145 |
+
ref_pts = (ref_pts - 112 / 2) * factor + 112 / 2
|
146 |
+
ref_pts *= crop_size[0] / 112.
|
147 |
+
|
148 |
+
ref_pts_shp = ref_pts.shape
|
149 |
+
if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2:
|
150 |
+
raise FaceWarpException(
|
151 |
+
'reference_pts.shape must be (K,2) or (2,K) and K>2')
|
152 |
+
|
153 |
+
if ref_pts_shp[0] == 2:
|
154 |
+
ref_pts = ref_pts.T
|
155 |
+
|
156 |
+
src_pts = np.float32(facial_pts)
|
157 |
+
src_pts_shp = src_pts.shape
|
158 |
+
if max(src_pts_shp) < 3 or min(src_pts_shp) != 2:
|
159 |
+
raise FaceWarpException(
|
160 |
+
'facial_pts.shape must be (K,2) or (2,K) and K>2')
|
161 |
+
|
162 |
+
if src_pts_shp[0] == 2:
|
163 |
+
src_pts = src_pts.T
|
164 |
+
|
165 |
+
if src_pts.shape != ref_pts.shape:
|
166 |
+
raise FaceWarpException(
|
167 |
+
'facial_pts and reference_pts must have the same shape')
|
168 |
+
|
169 |
+
if align_type == 'cv2_affine':
|
170 |
+
tfm = cv2.getAffineTransform(src_pts, ref_pts)
|
171 |
+
tfm_inv = cv2.getAffineTransform(ref_pts, src_pts)
|
172 |
+
|
173 |
+
elif align_type == 'affine':
|
174 |
+
tfm = get_affine_transform_matrix(src_pts, ref_pts)
|
175 |
+
tfm_inv = get_affine_transform_matrix(ref_pts, src_pts)
|
176 |
+
else:
|
177 |
+
tfm, tfm_inv = get_similarity_transform_for_cv2(src_pts, ref_pts)
|
178 |
+
|
179 |
+
face_img = cv2.warpAffine(
|
180 |
+
src_img,
|
181 |
+
tfm, (crop_size[0], crop_size[1]),
|
182 |
+
borderValue=(255, 255, 255))
|
183 |
+
|
184 |
+
if return_trans_inv:
|
185 |
+
return face_img, tfm_inv
|
186 |
+
else:
|
187 |
+
return face_img
|
source/mtcnn_pytorch/src/matlab_cp2tform.py
ADDED
@@ -0,0 +1,339 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
Created on Tue Jul 11 06:54:28 2017
|
3 |
+
|
4 |
+
@author: zhaoyafei
|
5 |
+
"""
|
6 |
+
|
7 |
+
import numpy as np
|
8 |
+
from numpy.linalg import inv, lstsq
|
9 |
+
from numpy.linalg import matrix_rank as rank
|
10 |
+
from numpy.linalg import norm
|
11 |
+
|
12 |
+
|
13 |
+
class MatlabCp2tormException(Exception):
|
14 |
+
|
15 |
+
def __str__(self):
|
16 |
+
return 'In File {}:{}'.format(__file__, super.__str__(self))
|
17 |
+
|
18 |
+
|
19 |
+
def tformfwd(trans, uv):
|
20 |
+
"""
|
21 |
+
Function:
|
22 |
+
----------
|
23 |
+
apply affine transform 'trans' to uv
|
24 |
+
|
25 |
+
Parameters:
|
26 |
+
----------
|
27 |
+
@trans: 3x3 np.array
|
28 |
+
transform matrix
|
29 |
+
@uv: Kx2 np.array
|
30 |
+
each row is a pair of coordinates (x, y)
|
31 |
+
|
32 |
+
Returns:
|
33 |
+
----------
|
34 |
+
@xy: Kx2 np.array
|
35 |
+
each row is a pair of transformed coordinates (x, y)
|
36 |
+
"""
|
37 |
+
uv = np.hstack((uv, np.ones((uv.shape[0], 1))))
|
38 |
+
xy = np.dot(uv, trans)
|
39 |
+
xy = xy[:, 0:-1]
|
40 |
+
return xy
|
41 |
+
|
42 |
+
|
43 |
+
def tforminv(trans, uv):
|
44 |
+
"""
|
45 |
+
Function:
|
46 |
+
----------
|
47 |
+
apply the inverse of affine transform 'trans' to uv
|
48 |
+
|
49 |
+
Parameters:
|
50 |
+
----------
|
51 |
+
@trans: 3x3 np.array
|
52 |
+
transform matrix
|
53 |
+
@uv: Kx2 np.array
|
54 |
+
each row is a pair of coordinates (x, y)
|
55 |
+
|
56 |
+
Returns:
|
57 |
+
----------
|
58 |
+
@xy: Kx2 np.array
|
59 |
+
each row is a pair of inverse-transformed coordinates (x, y)
|
60 |
+
"""
|
61 |
+
Tinv = inv(trans)
|
62 |
+
xy = tformfwd(Tinv, uv)
|
63 |
+
return xy
|
64 |
+
|
65 |
+
|
66 |
+
def findNonreflectiveSimilarity(uv, xy, options=None):
|
67 |
+
|
68 |
+
options = {'K': 2}
|
69 |
+
|
70 |
+
K = options['K']
|
71 |
+
M = xy.shape[0]
|
72 |
+
x = xy[:, 0].reshape((-1, 1)) # use reshape to keep a column vector
|
73 |
+
y = xy[:, 1].reshape((-1, 1)) # use reshape to keep a column vector
|
74 |
+
# print('--->x, y:\n', x, y
|
75 |
+
|
76 |
+
tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1))))
|
77 |
+
tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1))))
|
78 |
+
X = np.vstack((tmp1, tmp2))
|
79 |
+
# print('--->X.shape: ', X.shape
|
80 |
+
# print('X:\n', X
|
81 |
+
|
82 |
+
u = uv[:, 0].reshape((-1, 1)) # use reshape to keep a column vector
|
83 |
+
v = uv[:, 1].reshape((-1, 1)) # use reshape to keep a column vector
|
84 |
+
U = np.vstack((u, v))
|
85 |
+
# print('--->U.shape: ', U.shape
|
86 |
+
# print('U:\n', U
|
87 |
+
|
88 |
+
# We know that X * r = U
|
89 |
+
if rank(X) >= 2 * K:
|
90 |
+
r, _, _, _ = lstsq(X, U)
|
91 |
+
r = np.squeeze(r)
|
92 |
+
else:
|
93 |
+
raise Exception('cp2tform:twoUniquePointsReq')
|
94 |
+
|
95 |
+
# print('--->r:\n', r
|
96 |
+
|
97 |
+
sc = r[0]
|
98 |
+
ss = r[1]
|
99 |
+
tx = r[2]
|
100 |
+
ty = r[3]
|
101 |
+
|
102 |
+
Tinv = np.array([[sc, -ss, 0], [ss, sc, 0], [tx, ty, 1]])
|
103 |
+
|
104 |
+
# print('--->Tinv:\n', Tinv
|
105 |
+
|
106 |
+
T = inv(Tinv)
|
107 |
+
# print('--->T:\n', T
|
108 |
+
|
109 |
+
T[:, 2] = np.array([0, 0, 1])
|
110 |
+
|
111 |
+
return T, Tinv
|
112 |
+
|
113 |
+
|
114 |
+
def findSimilarity(uv, xy, options=None):
|
115 |
+
|
116 |
+
options = {'K': 2}
|
117 |
+
|
118 |
+
# uv = np.array(uv)
|
119 |
+
# xy = np.array(xy)
|
120 |
+
|
121 |
+
# Solve for trans1
|
122 |
+
trans1, trans1_inv = findNonreflectiveSimilarity(uv, xy, options)
|
123 |
+
|
124 |
+
# Solve for trans2
|
125 |
+
|
126 |
+
# manually reflect the xy data across the Y-axis
|
127 |
+
xyR = xy
|
128 |
+
xyR[:, 0] = -1 * xyR[:, 0]
|
129 |
+
|
130 |
+
trans2r, trans2r_inv = findNonreflectiveSimilarity(uv, xyR, options)
|
131 |
+
|
132 |
+
# manually reflect the tform to undo the reflection done on xyR
|
133 |
+
TreflectY = np.array([[-1, 0, 0], [0, 1, 0], [0, 0, 1]])
|
134 |
+
|
135 |
+
trans2 = np.dot(trans2r, TreflectY)
|
136 |
+
|
137 |
+
# Figure out if trans1 or trans2 is better
|
138 |
+
xy1 = tformfwd(trans1, uv)
|
139 |
+
norm1 = norm(xy1 - xy)
|
140 |
+
|
141 |
+
xy2 = tformfwd(trans2, uv)
|
142 |
+
norm2 = norm(xy2 - xy)
|
143 |
+
|
144 |
+
if norm1 <= norm2:
|
145 |
+
return trans1, trans1_inv
|
146 |
+
else:
|
147 |
+
trans2_inv = inv(trans2)
|
148 |
+
return trans2, trans2_inv
|
149 |
+
|
150 |
+
|
151 |
+
def get_similarity_transform(src_pts, dst_pts, reflective=True):
|
152 |
+
"""
|
153 |
+
Function:
|
154 |
+
----------
|
155 |
+
Find Similarity Transform Matrix 'trans':
|
156 |
+
u = src_pts[:, 0]
|
157 |
+
v = src_pts[:, 1]
|
158 |
+
x = dst_pts[:, 0]
|
159 |
+
y = dst_pts[:, 1]
|
160 |
+
[x, y, 1] = [u, v, 1] * trans
|
161 |
+
|
162 |
+
Parameters:
|
163 |
+
----------
|
164 |
+
@src_pts: Kx2 np.array
|
165 |
+
source points, each row is a pair of coordinates (x, y)
|
166 |
+
@dst_pts: Kx2 np.array
|
167 |
+
destination points, each row is a pair of transformed
|
168 |
+
coordinates (x, y)
|
169 |
+
@reflective: True or False
|
170 |
+
if True:
|
171 |
+
use reflective similarity transform
|
172 |
+
else:
|
173 |
+
use non-reflective similarity transform
|
174 |
+
|
175 |
+
Returns:
|
176 |
+
----------
|
177 |
+
@trans: 3x3 np.array
|
178 |
+
transform matrix from uv to xy
|
179 |
+
trans_inv: 3x3 np.array
|
180 |
+
inverse of trans, transform matrix from xy to uv
|
181 |
+
"""
|
182 |
+
|
183 |
+
if reflective:
|
184 |
+
trans, trans_inv = findSimilarity(src_pts, dst_pts)
|
185 |
+
else:
|
186 |
+
trans, trans_inv = findNonreflectiveSimilarity(src_pts, dst_pts)
|
187 |
+
|
188 |
+
return trans, trans_inv
|
189 |
+
|
190 |
+
|
191 |
+
def cvt_tform_mat_for_cv2(trans):
|
192 |
+
"""
|
193 |
+
Function:
|
194 |
+
----------
|
195 |
+
Convert Transform Matrix 'trans' into 'cv2_trans' which could be
|
196 |
+
directly used by cv2.warpAffine():
|
197 |
+
u = src_pts[:, 0]
|
198 |
+
v = src_pts[:, 1]
|
199 |
+
x = dst_pts[:, 0]
|
200 |
+
y = dst_pts[:, 1]
|
201 |
+
[x, y].T = cv_trans * [u, v, 1].T
|
202 |
+
|
203 |
+
Parameters:
|
204 |
+
----------
|
205 |
+
@trans: 3x3 np.array
|
206 |
+
transform matrix from uv to xy
|
207 |
+
|
208 |
+
Returns:
|
209 |
+
----------
|
210 |
+
@cv2_trans: 2x3 np.array
|
211 |
+
transform matrix from src_pts to dst_pts, could be directly used
|
212 |
+
for cv2.warpAffine()
|
213 |
+
"""
|
214 |
+
cv2_trans = trans[:, 0:2].T
|
215 |
+
|
216 |
+
return cv2_trans
|
217 |
+
|
218 |
+
|
219 |
+
def get_similarity_transform_for_cv2(src_pts, dst_pts, reflective=True):
|
220 |
+
"""
|
221 |
+
Function:
|
222 |
+
----------
|
223 |
+
Find Similarity Transform Matrix 'cv2_trans' which could be
|
224 |
+
directly used by cv2.warpAffine():
|
225 |
+
u = src_pts[:, 0]
|
226 |
+
v = src_pts[:, 1]
|
227 |
+
x = dst_pts[:, 0]
|
228 |
+
y = dst_pts[:, 1]
|
229 |
+
[x, y].T = cv_trans * [u, v, 1].T
|
230 |
+
|
231 |
+
Parameters:
|
232 |
+
----------
|
233 |
+
@src_pts: Kx2 np.array
|
234 |
+
source points, each row is a pair of coordinates (x, y)
|
235 |
+
@dst_pts: Kx2 np.array
|
236 |
+
destination points, each row is a pair of transformed
|
237 |
+
coordinates (x, y)
|
238 |
+
reflective: True or False
|
239 |
+
if True:
|
240 |
+
use reflective similarity transform
|
241 |
+
else:
|
242 |
+
use non-reflective similarity transform
|
243 |
+
|
244 |
+
Returns:
|
245 |
+
----------
|
246 |
+
@cv2_trans: 2x3 np.array
|
247 |
+
transform matrix from src_pts to dst_pts, could be directly used
|
248 |
+
for cv2.warpAffine()
|
249 |
+
"""
|
250 |
+
trans, trans_inv = get_similarity_transform(src_pts, dst_pts, reflective)
|
251 |
+
cv2_trans = cvt_tform_mat_for_cv2(trans)
|
252 |
+
cv2_trans_inv = cvt_tform_mat_for_cv2(trans_inv)
|
253 |
+
|
254 |
+
return cv2_trans, cv2_trans_inv
|
255 |
+
|
256 |
+
|
257 |
+
if __name__ == '__main__':
|
258 |
+
"""
|
259 |
+
u = [0, 6, -2]
|
260 |
+
v = [0, 3, 5]
|
261 |
+
x = [-1, 0, 4]
|
262 |
+
y = [-1, -10, 4]
|
263 |
+
|
264 |
+
# In Matlab, run:
|
265 |
+
#
|
266 |
+
# uv = [u'; v'];
|
267 |
+
# xy = [x'; y'];
|
268 |
+
# tform_sim=cp2tform(uv,xy,'similarity');
|
269 |
+
#
|
270 |
+
# trans = tform_sim.tdata.T
|
271 |
+
# ans =
|
272 |
+
# -0.0764 -1.6190 0
|
273 |
+
# 1.6190 -0.0764 0
|
274 |
+
# -3.2156 0.0290 1.0000
|
275 |
+
# trans_inv = tform_sim.tdata.Tinv
|
276 |
+
# ans =
|
277 |
+
#
|
278 |
+
# -0.0291 0.6163 0
|
279 |
+
# -0.6163 -0.0291 0
|
280 |
+
# -0.0756 1.9826 1.0000
|
281 |
+
# xy_m=tformfwd(tform_sim, u,v)
|
282 |
+
#
|
283 |
+
# xy_m =
|
284 |
+
#
|
285 |
+
# -3.2156 0.0290
|
286 |
+
# 1.1833 -9.9143
|
287 |
+
# 5.0323 2.8853
|
288 |
+
# uv_m=tforminv(tform_sim, x,y)
|
289 |
+
#
|
290 |
+
# uv_m =
|
291 |
+
#
|
292 |
+
# 0.5698 1.3953
|
293 |
+
# 6.0872 2.2733
|
294 |
+
# -2.6570 4.3314
|
295 |
+
"""
|
296 |
+
u = [0, 6, -2]
|
297 |
+
v = [0, 3, 5]
|
298 |
+
x = [-1, 0, 4]
|
299 |
+
y = [-1, -10, 4]
|
300 |
+
|
301 |
+
uv = np.array((u, v)).T
|
302 |
+
xy = np.array((x, y)).T
|
303 |
+
|
304 |
+
print('\n--->uv:')
|
305 |
+
print(uv)
|
306 |
+
print('\n--->xy:')
|
307 |
+
print(xy)
|
308 |
+
|
309 |
+
trans, trans_inv = get_similarity_transform(uv, xy)
|
310 |
+
|
311 |
+
print('\n--->trans matrix:')
|
312 |
+
print(trans)
|
313 |
+
|
314 |
+
print('\n--->trans_inv matrix:')
|
315 |
+
print(trans_inv)
|
316 |
+
|
317 |
+
print('\n---> apply transform to uv')
|
318 |
+
print('\nxy_m = uv_augmented * trans')
|
319 |
+
uv_aug = np.hstack((uv, np.ones((uv.shape[0], 1))))
|
320 |
+
xy_m = np.dot(uv_aug, trans)
|
321 |
+
print(xy_m)
|
322 |
+
|
323 |
+
print('\nxy_m = tformfwd(trans, uv)')
|
324 |
+
xy_m = tformfwd(trans, uv)
|
325 |
+
print(xy_m)
|
326 |
+
|
327 |
+
print('\n---> apply inverse transform to xy')
|
328 |
+
print('\nuv_m = xy_augmented * trans_inv')
|
329 |
+
xy_aug = np.hstack((xy, np.ones((xy.shape[0], 1))))
|
330 |
+
uv_m = np.dot(xy_aug, trans_inv)
|
331 |
+
print(uv_m)
|
332 |
+
|
333 |
+
print('\nuv_m = tformfwd(trans_inv, xy)')
|
334 |
+
uv_m = tformfwd(trans_inv, xy)
|
335 |
+
print(uv_m)
|
336 |
+
|
337 |
+
uv_m = tforminv(trans, xy)
|
338 |
+
print('\nuv_m = tforminv(trans, xy)')
|
339 |
+
print(uv_m)
|
source/utils.py
ADDED
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
|
3 |
+
import cv2
|
4 |
+
import numpy as np
|
5 |
+
|
6 |
+
|
7 |
+
def resize_size(image, size=720):
|
8 |
+
h, w, c = np.shape(image)
|
9 |
+
if min(h, w) > size:
|
10 |
+
if h > w:
|
11 |
+
h, w = int(size * h / w), size
|
12 |
+
else:
|
13 |
+
h, w = size, int(size * w / h)
|
14 |
+
image = cv2.resize(image, (w, h), interpolation=cv2.INTER_AREA)
|
15 |
+
return image
|
16 |
+
|
17 |
+
|
18 |
+
def padTo16x(image):
|
19 |
+
h, w, c = np.shape(image)
|
20 |
+
if h % 16 == 0 and w % 16 == 0:
|
21 |
+
return image, h, w
|
22 |
+
nh, nw = (h // 16 + 1) * 16, (w // 16 + 1) * 16
|
23 |
+
img_new = np.ones((nh, nw, 3), np.uint8) * 255
|
24 |
+
img_new[:h, :w, :] = image
|
25 |
+
|
26 |
+
return img_new, h, w
|
27 |
+
|
28 |
+
|
29 |
+
def get_f5p(landmarks, np_img):
|
30 |
+
eye_left = find_pupil(landmarks[36:41], np_img)
|
31 |
+
eye_right = find_pupil(landmarks[42:47], np_img)
|
32 |
+
if eye_left is None or eye_right is None:
|
33 |
+
print('cannot find 5 points with find_puil, used mean instead.!')
|
34 |
+
eye_left = landmarks[36:41].mean(axis=0)
|
35 |
+
eye_right = landmarks[42:47].mean(axis=0)
|
36 |
+
nose = landmarks[30]
|
37 |
+
mouth_left = landmarks[48]
|
38 |
+
mouth_right = landmarks[54]
|
39 |
+
f5p = [[eye_left[0], eye_left[1]], [eye_right[0], eye_right[1]],
|
40 |
+
[nose[0], nose[1]], [mouth_left[0], mouth_left[1]],
|
41 |
+
[mouth_right[0], mouth_right[1]]]
|
42 |
+
return f5p
|
43 |
+
|
44 |
+
|
45 |
+
def find_pupil(landmarks, np_img):
|
46 |
+
h, w, _ = np_img.shape
|
47 |
+
xmax = int(landmarks[:, 0].max())
|
48 |
+
xmin = int(landmarks[:, 0].min())
|
49 |
+
ymax = int(landmarks[:, 1].max())
|
50 |
+
ymin = int(landmarks[:, 1].min())
|
51 |
+
|
52 |
+
if ymin >= ymax or xmin >= xmax or ymin < 0 or xmin < 0 or ymax > h or xmax > w:
|
53 |
+
return None
|
54 |
+
eye_img_bgr = np_img[ymin:ymax, xmin:xmax, :]
|
55 |
+
eye_img = cv2.cvtColor(eye_img_bgr, cv2.COLOR_BGR2GRAY)
|
56 |
+
eye_img = cv2.equalizeHist(eye_img)
|
57 |
+
n_marks = landmarks - np.array([xmin, ymin]).reshape([1, 2])
|
58 |
+
eye_mask = cv2.fillConvexPoly(
|
59 |
+
np.zeros_like(eye_img), n_marks.astype(np.int32), 1)
|
60 |
+
ret, thresh = cv2.threshold(eye_img, 100, 255,
|
61 |
+
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
|
62 |
+
thresh = (1 - thresh / 255.) * eye_mask
|
63 |
+
cnt = 0
|
64 |
+
xm = []
|
65 |
+
ym = []
|
66 |
+
for i in range(thresh.shape[0]):
|
67 |
+
for j in range(thresh.shape[1]):
|
68 |
+
if thresh[i, j] > 0.5:
|
69 |
+
xm.append(j)
|
70 |
+
ym.append(i)
|
71 |
+
cnt += 1
|
72 |
+
if cnt != 0:
|
73 |
+
xm.sort()
|
74 |
+
ym.sort()
|
75 |
+
xm = xm[cnt // 2]
|
76 |
+
ym = ym[cnt // 2]
|
77 |
+
else:
|
78 |
+
xm = thresh.shape[1] / 2
|
79 |
+
ym = thresh.shape[0] / 2
|
80 |
+
|
81 |
+
return xm + xmin, ym + ymin
|
82 |
+
|
83 |
+
|
84 |
+
def all_file(file_dir):
|
85 |
+
L = []
|
86 |
+
for root, dirs, files in os.walk(file_dir):
|
87 |
+
for file in files:
|
88 |
+
extend = os.path.splitext(file)[1]
|
89 |
+
if extend == '.png' or extend == '.jpg' or extend == '.jpeg':
|
90 |
+
L.append(os.path.join(root, file))
|
91 |
+
return L
|
92 |
+
|
93 |
+
def initialize_mask(box_width):
|
94 |
+
h, w = [box_width, box_width]
|
95 |
+
mask = np.zeros((h, w), np.uint8)
|
96 |
+
|
97 |
+
center = (int(w / 2), int(h / 2))
|
98 |
+
axes = (int(w * 0.4), int(h * 0.49))
|
99 |
+
mask = cv2.ellipse(img=mask, center=center, axes=axes, angle=0, startAngle=0, endAngle=360, color=(1),
|
100 |
+
thickness=-1)
|
101 |
+
mask = cv2.distanceTransform(mask, cv2.DIST_L2, 3)
|
102 |
+
|
103 |
+
maxn = max(w, h) * 0.15
|
104 |
+
mask[(mask < 255) & (mask > 0)] = mask[(mask < 255) & (mask > 0)] / maxn
|
105 |
+
mask = np.clip(mask, 0, 1)
|
106 |
+
|
107 |
+
return mask.astype(float)
|