import pickle import matplotlib.pyplot as plt import numpy as np import torch #%% #### looking at the data # from pymo.rotation_tools import unroll_1, unroll_2 # # thing = pickle.load(open("data/motions/gBR_sBM_cAll_d06_mBR4_ch10.pkl", "rb")) # thing = pickle.load(open("data/motions/gWA_sFM_cAll_d26_mWA1_ch09.pkl", "rb")) # thing = pickle.load(open("/home/guillefix/code/mocap/PyMO/demos/data/gBR_sBM_cAll_d04_mBR0_ch01.pkl", "rb")) # thing = pickle.load(open("/home/guillefix/Downloads/PyMO/demos/data/gBR_sBM_cAll_d04_mBR0_ch01.pkl", "rb")) # thing = pickle.load(open("data/motions/gBR_sBM_cAll_d04_mBR0_ch01.pkl", "rb")) # thing = pickle.load(open("/home/guillefix/Downloads/motions/gBR_sBM_cAll_d04_mBR0_ch01.pkl", "rb")) # thing = pickle.load(open("data/motions/gJB_sFM_cAll_d09_mJB1_ch21.pkl", "rb")) # thing = pickle.load(open("/home/guillefix/Downloads/aist_plusplus_final_motions_gBR_sBM_cAll_d04_mBR0_ch01.pkl", "rb")) # # thing['smpl_poses'].shape # thing.keys() # poses = thing['smpl_poses'] # poses = poses.reshape(-1,24*3) # # unrolled_poses = unroll_1(poses.reshape(-1,3)) # rots = poses[:,:3] # for i in range(24): # poses[:,i*3:(i+1)*3] = unroll_2(poses[:,i*3:(i+1)*3]) # unrolled_poses.shape # poses = unrolled_poses.reshape(-1,24*3) # poses[593:693,:3] # poses[593:693,:3][38:] # # poses[:300,:3] # import numpy as np # np.diff(poses[90:110,:3], axis=0) # np.diff(poses[90:110,:3], axis=0) # # plt.matshow(poses[:300,:3]) # # plt.matshow(poses[1000:1500,:3]) # plt.matshow(poses[:300,:3]) # # thing # # thing['smpl_poses'].shape # thing['smpl_trans'].shape # # # angles = [[euler_angles for euler_angles in np.array(joint_angles).reshape(-1,3)] for joint_angles in thing['smpl_poses']] # features = get_rot_matrices(thing['smpl_poses']) # # from scipy.spatial.transform import Rotation as R # # def get_rot_matrices(joint_traj): # return np.stack([np.concatenate([R.from_euler('xyz',euler_angles).as_matrix().flatten() for euler_angles in np.array(joint_angles).reshape(-1,3)]) for joint_angles in joint_traj]) # # def get_features(motion_data): # joint_angle_feats = get_rot_matrices((motion_data['smpl_poses'])) # return np.concatenate([joint_angle_feats,motion_data['smpl_trans']],1) # # # audio_feats = np.load("data/features/gWA_sBM_c03_d26_mWA1_ch08.mp3_mel_100.npy") # audio_feats = np.load("data/features/gWA_sBM_c01_d26_mWA1_ch08.mp3_mel_100.npy") # audio_feats = np.load("data/dev_audio/gBR_sBM_c01_d06_mBR4_ch10.mp3_mel_100.npy") # audio_feats = np.load("data/dev_audio/gBR_sBM_c01_d06_mBR2_ch02.mp3_mel_100.npy") # # audio_feats = np.load("data/dev_audio/gHO_sBM_c01_d19_mHO3_ch08.mp3_multi_mel_80.npy_ddc_hidden.npy") # audio_feats = np.load("data/features/gHO_sBM_cAll_d19_mHO3_ch08.mp3_multi_mel_80.npy_ddc_hidden.npy") # audio_feats = np.load("data/features/gWA_sFM_cAll_d26_mWA1_ch09.mp3_mel_ddcpca.npy") # motion_feats = np.load("data/features/gWA_sFM_cAll_d26_mWA1_ch09.pkl_joint_angles_mats.npy") # mf = np.load("test_data/gWA_sFM_cAll_d27_mWA2_ch17.joint_angles_scaled.npy") # mf = np.load("test_data/gWA_sFM_cAll_d27_mWA2_ch17.pkl_joint_angles_mats.npy") # sf = np.load("test_data/gWA_sFM_cAll_d27_mWA2_ch17.mp3_mel_ddcpca.npy") # sf = np.load("test_data/gWA_sFM_cAll_d27_mWA2_ch17.mel_ddcpca_scaled.npy") # # mf_mean=np.mean(mf,0,keepdims=True) # mf_std = np.std(mf,0,keepdims=True)+1e-5 # sf = (sf-np.mean(sf,0,keepdims=True))/(np.std(sf,0,keepdims=True)+1e-5) # mf = (mf-mf_mean)/(mf_std) # # # sf.mean(0) # # mf # # plt.matshow(mf) # plt.matshow(sf) # sf[0] # # # Audio_feats.shape # # audio_feats.shape # plt.matshow(motion_feats[100:200,:]) # # plt.matshow(audio_feats[100:200,:]) # # from sklearn import decomposition # # pca = decomposition.PCA(n_components=512) # # x = pca.fit_transform(audio_feats) # x.shape # # pca.transform(x[:5]) # # plt.matshow(x[100:200,:2]) # # audio_feats.shape # max(1,2) # # ########################### # #playing with masks # # import torch # # sz=20 # mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1) # mask[:3,:3] # mask[:,:10] = 1 # plt.imshow(mask) # mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0)) # # ########################### # #plotting masks hmm # #mega hacky lel # import numpy as np import matplotlib.pyplot as plt mask = np.load("analysis/weight_mats/"+"892818c5-f166-45a0-a228-648127ed87e3.np.npy") mask[0][120:140] mask[0][:140] import torch # maskt = torch.from_numpy(mask) # ps = torch.nn.functional.softmax(maskt,dim=-1) sm = lambda x: torch.nn.functional.softmax(torch.from_numpy(x),dim=-1) ps[0] mask[1].max() mask[1].max() mask.shape mask[0].shape import scipy.linalg mask[0].shape scipy.linalg.norm(mask[0],axis=1) mask[5] # plt.matshow(mask[0][:20]) plt.matshow(mask[0]) plt.matshow(sm(mask[0])) plt.matshow(sm(mask[0])) plt.matshow(sm(mask[0])[60:120,:120]) plt.matshow(sm(mask[0])[-60:,:120]) plt.matshow(sm(mask[9])[60:120,:120]) # plt.matshow(sum([sm(mask[i])[60:120,:120] for i in range(10)])) plt.matshow(sm(mask[0])[-60:,:120]) plt.matshow(acc[:,142:142+120]**2) plt.matshow(sm(mask[0][:,40:50])) plt.matshow(sm(mask[0][:,30:80])) plt.matshow(sm(mask[0][:3])) # plt.matshow(sm(mask[0][:20])) mask[9] plt.matshow(mask[3]) plt.matshow(sm(mask[2])) # mdata = np.load("data/dance_combined/aistpp_gMH_sFM_cAll_d22_mMH3_ch04.bvh_expmap_cr.npy") mdata = np.load("data/dance_combined/groovenet_2.bvh_expmap_cr.npy") acc = np.abs(np.diff(mdata[:,:-6],2,axis=0)).T mdata.shape acc.shape plt.matshow(acc[:,:120]) plt.matshow(sm(mask[9][:120])) plt.matshow(sm(mask[4][120:])) plt.matshow(sm(mask[2][:100])) plt.matshow(sm(mask[2][100:])) plt.matshow(np.log(sm(mask[9]))) plt.matshow(sm(mask[1][150:170])) plt.matshow(sm(mask[5])) plt.matshow(mask) plt.matshow(mask[:500]) plt.matshow(np.matmul(mask[:500],mask[:500].T)) mask mask.std(axis=1) plt.matshow(mask[9][0:1]) mask # # # ########################### # #audio # import librosa # # y_wav, sr = librosa.load("data/dev_audio/gBR_sBM_c01_d06_mBR5_ch10.mp3", sr=48000) # y_wav # envelope = librosa.onset.onset_strength(y=y_wav,hop_length=480) # # (y_wav.shape[0]/48000)/0.01 # # envelope.shape # # # ################### # ## # # seq_id="gWA_sFM_cAll_d27_mWA2_ch17" # # seq_id="mambo" # # # sf = np.load("data/features/"+seq_id+".mel_ddcpca_scaled.npy") # sf = np.load("test_data/"+seq_id+".mel_ddcpca_scaled.npy") # # mf = np.load("data/features/"+seq_id+".joint_angles_scaled.npy") # mf = np.load("test_data/"+seq_id+".joint_angles_scaled.npy") # # sf.shape # sf # plt.matshow(sf) # # #%% # # # rot_mats = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_moglow_test_filter/predicted_mods/gBR_sBM_cAll_d04_mBR2_ch01.joint_angles_scaled.generated.npy") # rot_mats = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_2/predicted_mods/gBR_sBM_cAll_d04_mBR2_ch01.joint_angles_scaled.generated.npy") # # rot_mats[:,0,:] # # rot_mats.shape # # rot_mats[:,0,:-3].max() # # plt.matshow(rot_mats[:,0,:]) # plt.matshow(rot_mats[:,0,:-3]) # plt.matshow(rot_mats[:,0,-3:]) # # ident = np.eye(3, dtype=np.float32) # rot_mats = rot_mats[:216].reshape(-1,9) # for j,mat in enumerate(rot_mats): # mat = mat.reshape(3,3) + ident # print(np.linalg.det(mat)) #%% from analysis.pymo.parsers import BVHParser from analysis.pymo.data import Joint, MocapData from analysis.pymo.preprocessing import * from analysis.pymo.viz_tools import * from analysis.pymo.writers import * from sklearn.pipeline import Pipeline # data = np.load("data/scaled_features/gWA_sFM_cAll_d26_mWA4_ch12.expmap_scaled.npy") # data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_flower_expmap/predicted_mods/gLO_sBM_cAll_d13_mLO3_ch06.expmap_scaled.generated.npy") # data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_moglow_expmap/predicted_mods/gLO_sBM_cAll_d13_mLO3_ch06.expmap_scaled.generated.npy") # data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_residual/predicted_mods/gLO_sBM_cAll_d13_mLO3_ch06.expmap_scaled.generated.npy") # data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_residual/predicted_mods/gWA_sBM_cAll_d26_mWA1_ch05.expmap_scaled.generated.npy") # data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_residual/predicted_mods/gBR_sBM_cAll_d04_mBR2_ch01.expmap_scaled.generated.npy") # data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_residual/predicted_mods/gPO_sBM_cAll_d10_mPO0_ch10.expmap_scaled.generated.npy") # data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_residual/predicted_mods/gJS_sBM_cAll_d03_mJS3_ch02.expmap_scaled.generated.npy") data = np.load("/home/guillefix/code/mt-lightning/inference/generated/aistpp_residual/predicted_mods/gMH_sBM_cAll_d24_mMH2_ch08.expmap_scaled.generated.npy") # data = np.load("data/scaled_features/gLO_sBM_cAll_d13_mLO3_ch06.expmap_scaled.npy") # data = np.load(""analysis/tmp/gWA_sFM_cAll_d26_mWA4_ch12.bvh_expmap.npz")["clips"] # transform = pickle.load(open("data/scaled_features/bvh_expmap_scaler.pkl", "rb")) # transform = pickle.load(open("data/scaled_features/bvh_expmap_scaler.pkl", "rb")) # # data = transform.inverse_transform(data) import joblib as jl #%% # pipeline = jl.load("data/scaled_features/motion_expmap_data_pipe.sav") pipeline = jl.load("data/scaled_features/motion_data_pipe.sav") fps=60 # pipeline = Pipeline([ # ('dwnsampl', DownSampler(tgt_fps=fps, keep_all=False)), # ('root', RootTransformer('pos_rot_deltas')), # # ('mir', Mirror(axis='X', append=True)), # ('jtsel', JointSelector(['Spine', 'Spine1', 'Spine2', 'Neck', 'Head', 'RightShoulder', 'RightArm', 'RightForeArm', 'RightHand', 'LeftShoulder', 'LeftArm', 'LeftForeArm', 'LeftHand', 'RightUpLeg', 'RightLeg', 'RightFoot', 'RightToeBase', 'LeftUpLeg', 'LeftLeg', 'LeftFoot', 'LeftToeBase'], include_root=True)), # ('exp', MocapParameterizer('expmap')), # # ('cnst', ConstantsRemover()), # ('np', Numpyfier()) # ]) data.shape # transform the data back to it's original shape # note: in a real scenario this is usually done with predicted data # note: some transformations (such as transforming to joint positions) are not inversible # bvh_data=pipeline.inverse_transform([data[:,0,:]]) # bvh_data=pipeline.inverse_transform([np.concatenate([data[:,0,:],np.zeros((data.shape[0],3))], 1)]) bvh_data=pipeline.inverse_transform([data[:,0,:]]) # bvh_data=pipeline.inverse_transform([data]) writer = BVHWriter() with open('analysis/tmp/converted.bvh','w') as f: writer.write(bvh_data[0], f) bvh2pos = MocapParameterizer('position') pos_data = bvh2pos.fit_transform(bvh_data) dest_dir = "analysis/tmp" filename="test" render_mp4(pos_data[0], f'{dest_dir}/{filename}.mp4', axis_scale=3, elev=45, azim=45) # import pandas as pd # # pd.__version__ # # fps=60 p = BVHParser() data_pipe = Pipeline([ ('dwnsampl', DownSampler(tgt_fps=fps, keep_all=False)), ('root', RootTransformer('pos_rot_deltas')), # ('mir', Mirror(axis='X', append=True)), ('jtsel', JointSelector(['Spine', 'Spine1', 'Spine2', 'Neck', 'Head', 'RightShoulder', 'RightArm', 'RightForeArm', 'RightHand', 'LeftShoulder', 'LeftArm', 'LeftForeArm', 'LeftHand', 'RightUpLeg', 'RightLeg', 'RightFoot', 'RightToeBase', 'LeftUpLeg', 'LeftLeg', 'LeftFoot', 'LeftToeBase'], include_root=True)), ('exp', MocapParameterizer('expmap')), # ('cnst', ConstantsRemover()), ('np', Numpyfier()) ]) f="analysis/tmp/gWA_sFM_cAll_d26_mWA4_ch12.bvh" out_data = data_pipe.fit_transform([p.parse(f)]) # out_data[0].values out_data.shape out_data[0,:10,-1] bvh_data=data_pipe.inverse_transform(out_data) writer = BVHWriter() with open('analysis/tmp/test.bvh','w') as f: writer.write(bvh_data[0], f) ##################