File size: 12,424 Bytes
95f8bbc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
import numpy as np
import matplotlib.pyplot as plt
import pickle
import torch
import torch.nn.functional as F
from scipy.spatial.transform import Rotation   
from scipy.ndimage import binary_erosion, binary_dilation

import os
import json



def euler_angles_smooth(XYZ_euler_angles):

    if XYZ_euler_angles.ndim == 1:
        XYZ_euler_angles = XYZ_euler_angles[:, np.newaxis]

    for i in range(XYZ_euler_angles.shape[0]-1):
        for j in range(XYZ_euler_angles.shape[1]):
            # smooth
            if XYZ_euler_angles[i+1, j] - XYZ_euler_angles[i, j] > 180:
                XYZ_euler_angles[i+1:, j] = XYZ_euler_angles[i+1:, j] - 360
            elif XYZ_euler_angles[i+1, j] - XYZ_euler_angles[i, j] < -180:
                XYZ_euler_angles[i+1:, j] = XYZ_euler_angles[i+1:, j] + 360

    return np.squeeze(XYZ_euler_angles)



def xyz2euler_body(xyz, xyz_body_frame, X_dir=1.0, Y_dir=1.0):
    '''
        xyz: Coordinates from 3D human pose estimation. Each dimension: (frame, 3, xyz)
        xyz_body_frame: Coordinates of body frame. Used to calculate the Y direction rotation of body.
        X_dir: -1.0 for arm and body.
        Y_dir: -1.0 for body and head.
    '''

    # swap y and z to align the coordinate in the mine-imator
    xyz[:, :, [1, 2]] = xyz[:, :, [2, 1]]
    xyz[:, :, 0] = -xyz[:, :, 0]
    xyz_body_frame[:, :, [1, 2]] = xyz_body_frame[:, :, [2, 1]]
    xyz_body_frame[:, :, 0] = -xyz_body_frame[:, :, 0]

    p0, p1, p2 = torch.unbind(xyz, dim=1)
    p1_, p4_, p14_, p11_ = torch.unbind(xyz_body_frame, dim=1)

    # solve the cosine pose matrix
    Y = (p0 - p1) * Y_dir
    arm = p2 - p1

    Y = F.normalize(Y, dim=1)
    X = F.normalize(p11_ + p4_ - p1_ - p14_, dim=1)
    # X = F.normalize(torch.cross(X_dir*arm, Y), dim=1)  # TODO smooth
    Z = F.normalize(torch.cross(X, Y), dim=1)

    cos_pose_matrix = torch.stack([X, Y, Z], dim=2)
    r =  Rotation.from_matrix(cos_pose_matrix)
    YXZ_euler_angles = r.as_euler("YXZ", degrees=True)

    # bend
    bend = -(Y * F.normalize(arm, dim=1)).sum(dim=1) * Y_dir
    bend = torch.rad2deg(torch.acos(bend))

    # swap xyz
    YXZ_euler_angles[:, [0, 1, 2]] = YXZ_euler_angles[:, [1, 0, 2]]
    XYZ_euler_angles = YXZ_euler_angles

    # arm cos_pose_matrix
    Y_arm = F.normalize(arm, dim=1)
    X_arm = X
    Z_arm = F.normalize(torch.cross(X_arm, Y_arm), dim=1)
    cos_pose_matrix_arm = torch.stack([X_arm, Y_arm, Z_arm], dim=2)

    # avoid abrupt changes in angle
    XYZ_euler_angles = euler_angles_smooth(XYZ_euler_angles)
    bend = euler_angles_smooth(bend.numpy())

    return XYZ_euler_angles, bend, cos_pose_matrix_arm


def xyz2euler_relative(xyz, cos_body, X_dir=1.0, Y_dir=1.0, head=False, leg=False, euler_body=None):
    '''
        xyz: Coordinates from 3D human pose estimation. Each dimension: (frame, 3, xyz)
        X_dir: -1.0 for arm and body.
        Y_dir: -1.0 for body and head.
    '''

    # swap y and z to align the coordinate in the mine-imator
    xyz[:, :, [1, 2]] = xyz[:, :, [2, 1]]
    xyz[:, :, 0] = -xyz[:, :, 0]
    p0, p1, p2 = torch.unbind(xyz, dim=1)

    # solve the cosine pose matrix
    Y = (p0 - p1) * Y_dir
    arm = p2 - p1

    Y = F.normalize(Y, dim=1)
    X = F.normalize(torch.cross(X_dir*arm, Y), dim=1)  # TODO smooth
    Z = F.normalize(torch.cross(X, Y), dim=1)

    cos_pose_matrix = torch.stack([X, Y, Z], dim=2)

    if head == True:
        Y_arm = F.normalize(arm, dim=1)
        X_arm = X
        Z_arm = F.normalize(torch.cross(X_arm, Y_arm), dim=1)
        cos_pose_matrix = torch.stack([X_arm, Y_arm, Z_arm], dim=2)

    # relative to the body rotation Y
    if leg == True:
        euler_body_Y = euler_body * 0
        euler_body_Y[:, 0:1] = euler_body[:, 1:2]
        r_body_Y = Rotation.from_euler("YXZ", euler_body_Y, degrees=True)
        cos_body_Y = torch.from_numpy(r_body_Y.as_matrix())

    # relative to the body
    cos_relative = cos_body if leg == False else cos_body_Y.float()
    cos_pose_matrix = cos_relative.permute(0, 2, 1) @ cos_pose_matrix
    r =  Rotation.from_matrix(cos_pose_matrix)
    YXZ_euler_angles = r.as_euler("YXZ", degrees=True)

    # bend
    bend = -(Y * F.normalize(arm, dim=1)).sum(dim=1) * Y_dir
    bend = torch.rad2deg(torch.acos(bend))
    # if head == True:
    #     bend = bend * 0.5

    # swap xyz
    YXZ_euler_angles[:, [0, 1, 2]] = YXZ_euler_angles[:, [1, 0, 2]]
    XYZ_euler_angles = YXZ_euler_angles

    # avoid abrupt changes in angle
    XYZ_euler_angles = euler_angles_smooth(XYZ_euler_angles)
    bend = euler_angles_smooth(bend.numpy())

    return XYZ_euler_angles, bend


def calculate_body_offset(euler_body, euler_right_leg, bend_right_leg, euler_left_leg, bend_left_leg, length_leg=[6, 6], prior=False):
    '''
        Calculate the offset of the body to make the movement more realistic. 
        First, determine the foot positions of both legs based on the actual 
        effect of Euler angle rotation in Mine-imator. Then, determine which 
        leg is currently touching the ground and fix the grounded leg. This 
        allows the calculation of the body offset.

    '''

    def calculate_leg_coordinates(r_body_Y, euler_leg, bend_leg, length_leg, right=True):
        YXZ_euler_leg = euler_leg[:, [1, 0, 2]]
        r1 = Rotation.from_euler("YXZ", YXZ_euler_leg, degrees=True)
        m1 = r1.as_matrix()
        X1 = m1[:, :, 0]  # direction
        Y1 = m1[:, :, 1]  # vector to be rotated
        r2 = Rotation.from_rotvec(X1*bend_leg[:, np.newaxis], degrees=True)
        Y2 = r2.apply(Y1)  # reconstruct the arm vector
        coordinates = -(Y1 * length_leg[0] + Y2 * length_leg[1])
        coordinates[:, 0] = coordinates[:, 0] - 2 
        coordinates = r_body_Y.apply(coordinates)
        return coordinates

    # calculate the endpoint coordinates of two legs
    euler_body_Y = euler_body * 0
    euler_body_Y[:, 0:1] = euler_body[:, 1:2]
    r_body_Y = Rotation.from_euler("YXZ", euler_body_Y, degrees=True)
    right_coordinates = calculate_leg_coordinates(r_body_Y, euler_right_leg, bend_right_leg, length_leg)
    left_coordinates = calculate_leg_coordinates(r_body_Y, euler_left_leg, bend_left_leg, length_leg)
    # stack, 0: right, 1: left
    coordinates = np.stack([right_coordinates, left_coordinates], axis=1)
    
    # determine which leg grounded, 0: right, 1: left
    grounded_flag = (right_coordinates[:, 1] > left_coordinates[:, 1])*1
    # prior knowledge: The more bended legs are not grounded
    if prior == True:
        grounded_flag_left = (bend_right_leg - bend_left_leg) > 30
        grounded_flag_right = (bend_left_leg - bend_right_leg) > 30
        grounded_flag += grounded_flag_left*1
        grounded_flag *= (1 - grounded_flag_right)*1
    # smoothing
    grounded_flag = binary_erosion(grounded_flag, structure=np.ones(7))*1
    grounded_flag = binary_dilation(grounded_flag, structure=np.ones(7))*1

    body_POS = np.zeros_like(right_coordinates)

    # POS_Y
    ind = np.array(range(right_coordinates.shape[0]))
    body_POS[:, 1] = -coordinates[ind, grounded_flag, 1]

    # extract the X, Z coordinates of grounded leg in time t_1 
    X_t1 = coordinates[ind[:-1], grounded_flag[:-1], 0]
    Z_t1 = coordinates[ind[:-1], grounded_flag[:-1], 2]
    # extract the X, Z coordinates of grounded leg in time t_2
    # note that the split of grounded_flag not changed
    X_t2 = coordinates[ind[1:], grounded_flag[:-1], 0]
    Z_t2 = coordinates[ind[1:], grounded_flag[:-1], 2]

    # calculate the relative displacement between two frames
    X_relative = X_t2 - X_t1
    Z_relative = Z_t2 - Z_t1

    # calculate the absolute displacement
    X_abs = np.cumsum(X_relative)
    Z_abs = np.cumsum(Z_relative)

    body_POS[1:, 0] = -X_abs
    body_POS[1:, 2] = -Z_abs

    return body_POS


def add_keyframes(data, length, part_name, euler, bend, not_body=True, not_head=True, body_steve=False, body_POS=None):
    for i in range(length):
        if not_head:
            keyframes_dict = {
                "position": i,
                "part_name": part_name,
                "values": {
                    "ROT_X": float(euler[i][0]),
                    "ROT_Y": float(euler[i][2]),  # Y, Z args in mine-imator miframes is exchanged. Maybe a bug. 
                    "ROT_Z": float(euler[i][1]*not_body),
                    "BEND_ANGLE_X": float(bend[i])
                }
            }
        else:  # no bend
            keyframes_dict = {
                "position": i,
                "part_name": part_name,
                "values": {
                    "ROT_X": float(euler[i][0]),
                    "ROT_Y": float(euler[i][2]),
                    "ROT_Z": float(euler[i][1]),
                }
            }
        if body_steve == True:
            keyframes_dict = {
                "position": i,
                "values": {
                    "POS_X": float(body_POS[i][0]),
                    "POS_Y": float(body_POS[i][2]),
                    "POS_Z": float(body_POS[i][1]),
                    "ROT_Z": float(euler[i][1])
			    }
            }
        data["keyframes"].append(keyframes_dict)

    print(f"add_key_frames: {part_name}")


def hpe2keyframes(HPE_filename, FPS_mine_imator, keyframes_filename, prior=True):

    # read data
    with open(HPE_filename, 'rb') as file:
        data = np.load(file)
    print(f"open file: {HPE_filename}")
    xyz = data.copy()
    length = xyz.shape[0]

    # extract data from each body part
    xyz_right_leg = torch.from_numpy(xyz[:, 1:4, :])
    xyz_right_arm = torch.from_numpy(xyz[:, 14:17, :])
    xyz_left_leg = torch.from_numpy(xyz[:, 4:7, :])
    xyz_left_arm = torch.from_numpy(xyz[:, 11:14, :])
    xyz_body = torch.from_numpy(xyz[:, [0, 7, 8], :])
    xyz_body_frame = torch.from_numpy(xyz[:, [1, 4, 14, 11], :])
    xyz_head = torch.from_numpy(xyz[:, [8, 9, 10], :])

    # calculate the absolute euler angles of body
    euler_body, bend_body, cos_pos_matrix = xyz2euler_body(xyz_body, xyz_body_frame, X_dir=-1, Y_dir=-1)

    # calculate the relative euler angles of arm and head with respect to the body ROT_Y
    euler_right_leg, bend_right_leg = xyz2euler_relative(xyz_right_leg, cos_pos_matrix, leg=True, euler_body=euler_body)
    euler_left_leg, bend_left_leg = xyz2euler_relative(xyz_left_leg, cos_pos_matrix, leg=True, euler_body=euler_body)
    
    # calculate the relative euler angles of arm and head with respect to the upper body
    euler_right_arm, bend_right_arm = xyz2euler_relative(xyz_right_arm, cos_pos_matrix, X_dir=-1)
    euler_left_arm, bend_left_arm = xyz2euler_relative(xyz_left_arm, cos_pos_matrix, X_dir=-1)
    euler_head, bend_head = xyz2euler_relative(xyz_head, cos_pos_matrix, Y_dir=-1, head=True)
    
    # create json format data
    data = {
        "format": 34,
        "created_in": "2.0.0",  # mine-imator version
        "is_model": True,
        "tempo": FPS_mine_imator,  # FPS
        "length": length,  # keyframes length
        "keyframes": [
        ],
        "templates": [],
        "timelines": [],
        "resources": []
    }
    
    # relative offset makes the model more realistic
    # caculate the relative offset based on Euler angle and bending angle
    body_POS = calculate_body_offset(euler_body, euler_right_leg, bend_right_leg, euler_left_leg, bend_left_leg, prior=prior)


    add_keyframes(data, length, "left_leg", euler_left_leg, bend_left_leg)
    add_keyframes(data, length, "right_leg", euler_right_leg, bend_right_leg)
    add_keyframes(data, length, "left_arm", euler_left_arm, bend_left_arm)
    add_keyframes(data, length, "right_arm", euler_right_arm, bend_right_arm)
    add_keyframes(data, length, "body", euler_body, bend_body, not_body=False)
    add_keyframes(data, length, "head", euler_head, bend_head, not_head=False)
    add_keyframes(data, length, "abc", euler_body, bend_body, body_steve=True, body_POS=body_POS)  # TODO

    # save json
    with open(keyframes_filename, "w") as file:
        json.dump(data, file, indent=4)

    print(f"keyframes file saves successfully, file path: {os.path.abspath(keyframes_filename)}")
    


if __name__ == '__main__':
    # config
    HPE_filename = "outputs/test_3d_output_malaoshi_2-00_2-30_postprocess.npy"
    FPS_mine_imator = 30
    keyframes_filename = "steve_malaoshi2.miframes"
    prior = True
    hpe2keyframes(HPE_filename, FPS_mine_imator, keyframes_filename, prior=prior)

print("Done!")