gazelib/label_transform.py

import cv2

import numpy as np


def get_eye_nose_landmarks(landmarks):
    assert landmarks.shape[0]==50 or landmarks.shape[0]==68
    if landmarks.shape[0] == 50:
        lm_6 = landmarks[[20, 23, 26, 29, 15, 19], :]  # the eye and nose landmarks
    elif landmarks.shape[0] == 68:
        lm_6 = landmarks[[36, 39, 42, 45, 31, 35], :]  # the eye and nose landmarks
    return lm_6
def get_eye_mouth_landmarks(landmarks):
    assert landmarks.shape[0]==50 or landmarks.shape[0]==68
    if landmarks.shape[0] == 50:
        lm_6 = landmarks[[20, 23, 26, 29, 32, 38], :]  # the eye and nose landmarks
    elif landmarks.shape[0] == 68:
        lm_6 = landmarks[[36,39,42,45,48,54], :]  # the eye and nose landmarks
    return lm_6

def mean_eye_nose(landmarks):
    assert landmarks.shape[0]==6
    # get the face center
    two_eye_center = np.mean(landmarks[0:4, :], axis=0).reshape(1,-1)
    nose_center = np.mean(landmarks[4:6, :], axis=0).reshape(1,-1)
    face_center = np.mean(np.concatenate((two_eye_center, nose_center), axis=0), axis=0).reshape(1,-1)
    return face_center
def mean_eye_mouth(landmarks):
    assert landmarks.shape[0]==6
    face_center = np.mean(landmarks, axis=0).reshape(1,-1) 
    return face_center

def get_face_center_by_nose(hR, ht, face_model_load):
    face_model = get_eye_nose_landmarks(face_model_load)  # the eye and nose landmarks
    Fc = np.dot(hR, face_model.T) + ht # 3D positions of facial landmarks
    face_center = mean_eye_nose(Fc.T).reshape((3, 1))  # get the face center
    return face_center, Fc

def get_face_center_by_mouth(hR, ht, face_model_load):
    face_model = get_eye_mouth_landmarks(face_model_load)  # the eye and nose landmarks
    Fc = np.dot(hR, face_model.T) + ht # 3D positions of facial landmarks
    face_center = mean_eye_mouth(Fc.T).reshape((3, 1))  # get the face center
    return face_center, Fc

def lm68_to_50(lm_68):
	'''
	lm_68: (68,2)
	'''
	lm_50 = np.zeros((50,2))
	lm_50[0] = lm_68[8]
	lm_50[1:44] = lm_68[17:60]
	lm_50[44:47] = lm_68[61:64]
	lm_50[47:50] = lm_68[65:68]
	return lm_50


def lm68_subset(lm_68, NUM_KPTS_TO_USE):
	'''
	lm_68: (68,2)
	'''
	if NUM_KPTS_TO_USE == 6:
		lm_68 = np.array(lm_68, dtype=np.float32)
		return lm_68[[36, 39, 42, 45, 31, 35], :]
	elif NUM_KPTS_TO_USE ==50:
		return lm68_to_50(lm_68)
	else:
		print('not supported yet')
		exit(0)

def lm50_subset(lm_50, NUM_KPTS_TO_USE):
	'''
	lm_50: (50,2)
	'''
	lm_50 = lm_50.copy()
	if NUM_KPTS_TO_USE == 6:
		lm_50 = lm_50[[20, 23, 26, 29, 15, 19], :]
		return lm_50
	elif NUM_KPTS_TO_USE ==50:
		return lm_50
	else:
		print('not supported yet')
		exit(0)

def get_face_center(landmarks_3d):
	'''
	landmarks_3d: (3, 6)
	--> 
	face_center: (3,1)
	'''
	two_eye_center = np.mean(landmarks_3d[:, 0:4], axis=1).reshape((3, 1))
	nose_center = np.mean(landmarks_3d[:, 4:6], axis=1).reshape((3, 1))
	face_center = np.mean(np.concatenate((two_eye_center, nose_center), axis=1), axis=1).reshape((3, 1))
	return face_center



def compute_R(lm6, dataname):
	'''
	6 landmarks in opencv coordinate
	dataname: mpii or xgaze
	the face center are computed differently 
		for mpii: the 6 landmarks are 4 eye + 2 mouth
		for xgaze: the 6 landmarks are 4 eye + 2 nose
	'''
	if dataname=='mpii':
		left_center = np.mean(lm6[2:4,:],axis=0)
		right_center = np.mean(lm6[:2,:],axis=0)
		face_center = np.mean(lm6,axis=0)
	elif dataname=='xgaze':
		left_center = np.mean(lm6[2:4,:],axis=0)
		right_center = np.mean(lm6[:2,:],axis=0)
		nose_center = np.mean(lm6[[4,5],:],axis=0)
		face_center = ( (left_center + right_center)/2 + nose_center ) /2

	distance = np.linalg.norm(face_center)

	hRx = left_center - right_center
	hRx /= np.linalg.norm(hRx)
	forward = (face_center/distance).reshape(3)
	down = np.cross(forward, hRx)
	down /= np.linalg.norm(down)
	right = np.cross(down, forward)
	right /= np.linalg.norm(right)
	R = np.c_[right, down, forward].T
	return R
	
def rotation_matrix(x, y, z):
	'''
	x, y, z: roll, pitch, yaw, (radians)
	'''
	Rx = np.array([[1,0,0],
				[0, np.cos(x), -np.sin(x)],
				[0, np.sin(x), np.cos(x)]])

	Ry = np.array([[ np.cos(y), 0, np.sin(y)],
				[ 0,         1,         0],
				[-np.sin(y), 0, np.cos(y)]])

	Rz = np.array([[np.cos(z), -np.sin(z), 0],
				[np.sin(z),  np.cos(z), 0],
				[0,0,1]])
	return Rz@Ry@Rx
def get_rotation(from_pose, target_pose):
	
	rotation1 = rotation_matrix( -from_pose[0], from_pose[1], 0)
	rotation2 = rotation_matrix(-target_pose[0], target_pose[1], 0)
	rotation = rotation2@np.linalg.inv(rotation1)
	return rotation

def hR_2_hr(hR):
	hr = np.array([np.arcsin(hR[1, 2]),
				np.arctan2(hR[0, 2], hR[2, 2])])
	return hr

def hr_2_hR(hr):
	hR = rotation_matrix( -hr[0], hr[1], 0)
	return hR



if __name__ == '__main__':
	# hr_norm = np.array([0.15, 0.2])
	# pose = np.array([-0.1, 0.3])
	# rotation1 = rotation_matrix( -hr_norm[0], hr_norm[1], 0)
	# rot = cv2.Rodrigues( np.array([hr_norm[0], hr_norm[1], 0])  )[0] 
	def to_hR(hr_norm):
		hR_norm = rotation_matrix( -hr_norm[0], hr_norm[1], 0)
		return hR_norm

	hr1 = np.array([0.15, 0.2])
	hr2 = np.array([0.10, 0.2])

	hr_t =  np.array([-0.1, 0.3])

	hR1 = to_hR(hr1)
	hR2 = to_hR(hr2)
	print('hR1: ', hR1)
	print('hR2: ', hR2)
	
	R1t = get_rotation(hr1, hr_t)

	hR1_ = np.dot(R1t, hR1)

	print('rotated hR_: ', hR1_)
	hr1_ = np.array([np.arcsin(hR1_[1, 2]),
				np.arctan2(hR1_[0, 2], hR1_[2, 2])])
	print('rotated hr1_: ', hr1_)
	print('hR t: ', to_hR(hr_t))
	hR2_ = np.dot(R1t, hR2)
	print('rotated hR2_: ', hR2_)
	# rotation2 = rotation_matrix( -pose[0], pose[1], 0)