Spaces:
No application file
No application file
| import os | |
| # import clip | |
| from CLIP.clip import clip | |
| import numpy as np | |
| import torch | |
| from scipy import linalg | |
| from tqdm import tqdm | |
| import visualization.plot_3d_global as plot_3d | |
| from utils.motion_process import recover_from_ric | |
| from tqdm import trange | |
| def tensorborad_add_video_xyz(writer, xyz, nb_iter, tag, nb_vis=4, title_batch=None, outname=None): | |
| xyz = xyz[:1] | |
| bs, seq = xyz.shape[:2] | |
| xyz = xyz.reshape(bs, seq, -1, 3) | |
| plot_xyz = plot_3d.draw_to_batch(xyz.cpu().numpy(),title_batch, outname) | |
| plot_xyz =np.transpose(plot_xyz, (0, 1, 4, 2, 3)) | |
| writer.add_video(tag, plot_xyz, nb_iter, fps = 20) | |
| def evaluation_vqvae(out_dir, val_loader, net, logger, writer, nb_iter, best_fid, best_iter, best_div, best_top1, best_top2, best_top3, best_matching, eval_wrapper, draw = True, save = True, savegif=False, savenpy=False) : | |
| net.eval() | |
| nb_sample = 0 | |
| draw_org = [] | |
| draw_pred = [] | |
| draw_text = [] | |
| motion_annotation_list = [] | |
| motion_pred_list = [] | |
| R_precision_real = 0 | |
| R_precision = 0 | |
| nb_sample = 0 | |
| matching_score_real = 0 | |
| matching_score_pred = 0 | |
| for batch in val_loader: | |
| word_embeddings, pos_one_hots, caption, sent_len, motion, m_length, token, name = batch | |
| motion = motion.cuda() | |
| et, em = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, motion, m_length) | |
| bs, seq = motion.shape[0], motion.shape[1] | |
| num_joints = 21 if motion.shape[-1] == 251 else 22 | |
| pred_pose_eval = torch.zeros((bs, seq, motion.shape[-1])).cuda() | |
| for i in range(bs): | |
| pose = val_loader.dataset.inv_transform(motion[i:i+1, :m_length[i], :].detach().cpu().numpy()) | |
| pose_xyz = recover_from_ric(torch.from_numpy(pose).float().cuda(), num_joints) | |
| pred_pose, loss_commit, perplexity = net(motion[i:i+1, :m_length[i]]) | |
| pred_denorm = val_loader.dataset.inv_transform(pred_pose.detach().cpu().numpy()) | |
| pred_xyz = recover_from_ric(torch.from_numpy(pred_denorm).float().cuda(), num_joints) | |
| if savenpy: | |
| np.save(os.path.join(out_dir, name[i]+'_gt.npy'), pose_xyz[:, :m_length[i]].cpu().numpy()) | |
| np.save(os.path.join(out_dir, name[i]+'_pred.npy'), pred_xyz.detach().cpu().numpy()) | |
| pred_pose_eval[i:i+1,:m_length[i],:] = pred_pose | |
| if i < min(4, bs): | |
| draw_org.append(pose_xyz) | |
| draw_pred.append(pred_xyz) | |
| draw_text.append(caption[i]) | |
| et_pred, em_pred = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, pred_pose_eval, m_length) | |
| motion_pred_list.append(em_pred) | |
| motion_annotation_list.append(em) | |
| temp_R, temp_match = calculate_R_precision(et.cpu().numpy(), em.cpu().numpy(), top_k=3, sum_all=True) | |
| R_precision_real += temp_R | |
| matching_score_real += temp_match | |
| temp_R, temp_match = calculate_R_precision(et_pred.cpu().numpy(), em_pred.cpu().numpy(), top_k=3, sum_all=True) | |
| R_precision += temp_R | |
| matching_score_pred += temp_match | |
| nb_sample += bs | |
| motion_annotation_np = torch.cat(motion_annotation_list, dim=0).cpu().numpy() | |
| motion_pred_np = torch.cat(motion_pred_list, dim=0).cpu().numpy() | |
| gt_mu, gt_cov = calculate_activation_statistics(motion_annotation_np) | |
| mu, cov= calculate_activation_statistics(motion_pred_np) | |
| diversity_real = calculate_diversity(motion_annotation_np, 300 if nb_sample > 300 else 100) | |
| diversity = calculate_diversity(motion_pred_np, 300 if nb_sample > 300 else 100) | |
| R_precision_real = R_precision_real / nb_sample | |
| R_precision = R_precision / nb_sample | |
| matching_score_real = matching_score_real / nb_sample | |
| matching_score_pred = matching_score_pred / nb_sample | |
| fid = calculate_frechet_distance(gt_mu, gt_cov, mu, cov) | |
| msg = f"--> \t Eva. Iter {nb_iter} :, FID. {fid:.4f}, Diversity Real. {diversity_real:.4f}, Diversity. {diversity:.4f}, R_precision_real. {R_precision_real}, R_precision. {R_precision}, matching_score_real. {matching_score_real}, matching_score_pred. {matching_score_pred}" | |
| logger.info(msg) | |
| if draw: | |
| writer.add_scalar('./Test/FID', fid, nb_iter) | |
| writer.add_scalar('./Test/Diversity', diversity, nb_iter) | |
| writer.add_scalar('./Test/top1', R_precision[0], nb_iter) | |
| writer.add_scalar('./Test/top2', R_precision[1], nb_iter) | |
| writer.add_scalar('./Test/top3', R_precision[2], nb_iter) | |
| writer.add_scalar('./Test/matching_score', matching_score_pred, nb_iter) | |
| if nb_iter % 5000 == 0 : | |
| for ii in range(4): | |
| tensorborad_add_video_xyz(writer, draw_org[ii], nb_iter, tag='./Vis/org_eval'+str(ii), nb_vis=1, title_batch=[draw_text[ii]], outname=[os.path.join(out_dir, 'gt'+str(ii)+'.gif')] if savegif else None) | |
| if nb_iter % 5000 == 0 : | |
| for ii in range(4): | |
| tensorborad_add_video_xyz(writer, draw_pred[ii], nb_iter, tag='./Vis/pred_eval'+str(ii), nb_vis=1, title_batch=[draw_text[ii]], outname=[os.path.join(out_dir, 'pred'+str(ii)+'.gif')] if savegif else None) | |
| if fid < best_fid : | |
| print(fid,best_fid) | |
| msg = f"--> --> \t FID Improved from {best_fid:.5f} to {fid:.5f} !!!" | |
| logger.info(msg) | |
| best_fid, best_iter = fid, nb_iter | |
| if save: | |
| torch.save({'net' : net.state_dict()}, os.path.join(out_dir, 'net_best_fid.pth')) | |
| if abs(diversity_real - diversity) < abs(diversity_real - best_div) : | |
| msg = f"--> --> \t Diversity Improved from {best_div:.5f} to {diversity:.5f} !!!" | |
| logger.info(msg) | |
| best_div = diversity | |
| if save: | |
| torch.save({'net' : net.state_dict()}, os.path.join(out_dir, 'net_best_div.pth')) | |
| if R_precision[0] > best_top1 : | |
| msg = f"--> --> \t Top1 Improved from {best_top1:.4f} to {R_precision[0]:.4f} !!!" | |
| logger.info(msg) | |
| best_top1 = R_precision[0] | |
| if save: | |
| torch.save({'net' : net.state_dict()}, os.path.join(out_dir, 'net_best_top1.pth')) | |
| if R_precision[1] > best_top2 : | |
| msg = f"--> --> \t Top2 Improved from {best_top2:.4f} to {R_precision[1]:.4f} !!!" | |
| logger.info(msg) | |
| best_top2 = R_precision[1] | |
| if R_precision[2] > best_top3 : | |
| msg = f"--> --> \t Top3 Improved from {best_top3:.4f} to {R_precision[2]:.4f} !!!" | |
| logger.info(msg) | |
| best_top3 = R_precision[2] | |
| if matching_score_pred < best_matching : | |
| msg = f"--> --> \t matching_score Improved from {best_matching:.5f} to {matching_score_pred:.5f} !!!" | |
| logger.info(msg) | |
| best_matching = matching_score_pred | |
| if save: | |
| torch.save({'net' : net.state_dict()}, os.path.join(out_dir, 'net_best_matching.pth')) | |
| if save: | |
| torch.save({'net' : net.state_dict()}, os.path.join(out_dir, 'net_last.pth')) | |
| net.train() | |
| return best_fid, best_iter, best_div, best_top1, best_top2, best_top3, best_matching, writer, logger | |
| def evaluation_transformer(out_dir, val_loader, net, trans, logger, writer, nb_iter, best_fid, best_fid_syn,best_fid_perturbation,best_iter, best_div, best_top1, best_top2, best_top3, best_matching, clip_model, eval_wrapper, draw = True, save = True, savegif=False,PGD=None,crit=None) : | |
| trans.eval() | |
| #这里是不是应该clip也eval() | |
| nb_sample = 0 | |
| draw_org = [] | |
| draw_pred = [] | |
| draw_text = [] | |
| draw_text_pred = [] | |
| motion_annotation_list = [] | |
| motion_pred_list = [] | |
| motion_pred_per_list = [] | |
| R_precision_real = 0 | |
| R_precision = 0 | |
| matching_score_real = 0 | |
| matching_score_pred = 0 | |
| nb_sample = 0 | |
| for i in range(1): | |
| for batch in tqdm(val_loader): | |
| word_embeddings, pos_one_hots, clip_text, clip_text_perb, sent_len, pose, m_length, token, name = batch | |
| bs, seq = pose.shape[:2] | |
| num_joints = 21 if pose.shape[-1] == 251 else 22 | |
| text = clip.tokenize(clip_text, truncate=True).cuda() | |
| text_perb = clip.tokenize(clip_text_perb, truncate=True).cuda() | |
| feat_clip_text = clip_model.encode_text(text)[0].float() | |
| feat_clip_text_per = clip_model.encode_text(text_perb)[0].float() | |
| pred_pose_eval = torch.zeros((bs, seq, pose.shape[-1])).cuda() | |
| pred_pose_eval_per = torch.zeros((bs, seq, pose.shape[-1])).cuda() | |
| pred_len = torch.ones(bs).long() | |
| pred_len_per = torch.ones(bs).long() | |
| for k in range(bs): | |
| try: | |
| index_motion = trans.sample(feat_clip_text[k:k+1], False) | |
| index_motion_per = trans.sample(feat_clip_text_per[k:k+1], False) | |
| except: | |
| # print('---------------------') | |
| index_motion = torch.ones(1,1).cuda().long() | |
| index_motion_per = torch.ones(1,1).cuda().long() | |
| pred_pose = net.forward_decoder(index_motion) | |
| pred_pose_per = net.forward_decoder(index_motion_per) | |
| cur_len = pred_pose.shape[1] | |
| cur_len_per = pred_pose_per.shape[1] | |
| pred_len[k] = min(cur_len, seq) | |
| pred_len_per[k] = min(cur_len_per, seq) | |
| pred_pose_eval[k:k+1, :cur_len] = pred_pose[:, :seq] | |
| pred_pose_eval_per[k:k+1, :cur_len_per] = pred_pose_per[:, :seq] | |
| if draw: | |
| pred_denorm = val_loader.dataset.inv_transform(pred_pose.detach().cpu().numpy()) | |
| pred_xyz = recover_from_ric(torch.from_numpy(pred_denorm).float().cuda(), num_joints) | |
| if i == 0 and k < 4: | |
| draw_pred.append(pred_xyz) | |
| draw_text_pred.append(clip_text[k]) | |
| et_pred, em_pred = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, pred_pose_eval, pred_len) | |
| et_pred_per, em_pred_per = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, pred_pose_eval_per, pred_len_per) | |
| if i == 0: | |
| pose = pose.cuda().float() | |
| et, em = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, pose, m_length) | |
| motion_annotation_list.append(em) | |
| motion_pred_list.append(em_pred) | |
| motion_pred_per_list.append(em_pred_per) | |
| if draw: | |
| pose = val_loader.dataset.inv_transform(pose.detach().cpu().numpy()) | |
| pose_xyz = recover_from_ric(torch.from_numpy(pose).float().cuda(), num_joints) | |
| for j in range(min(4, bs)): | |
| draw_org.append(pose_xyz[j][:m_length[j]].unsqueeze(0)) | |
| draw_text.append(clip_text[j]) | |
| temp_R, temp_match = calculate_R_precision(et.cpu().numpy(), em.cpu().numpy(), top_k=3, sum_all=True) | |
| R_precision_real += temp_R | |
| matching_score_real += temp_match | |
| temp_R, temp_match = calculate_R_precision(et_pred.cpu().numpy(), em_pred.cpu().numpy(), top_k=3, sum_all=True) | |
| R_precision += temp_R | |
| matching_score_pred += temp_match | |
| nb_sample += bs | |
| motion_annotation_np = torch.cat(motion_annotation_list, dim=0).cpu().numpy() | |
| motion_pred_np = torch.cat(motion_pred_list, dim=0).cpu().numpy() | |
| motion_pred_per_np = torch.cat(motion_pred_per_list, dim=0).cpu().numpy() | |
| gt_mu, gt_cov = calculate_activation_statistics(motion_annotation_np) | |
| mu, cov= calculate_activation_statistics(motion_pred_np) | |
| mu_per, cov_per= calculate_activation_statistics(motion_pred_per_np) | |
| diversity_real = calculate_diversity(motion_annotation_np, 300 if nb_sample > 300 else 100) | |
| diversity = calculate_diversity(motion_pred_np, 300 if nb_sample > 300 else 100) | |
| R_precision_real = R_precision_real / nb_sample | |
| R_precision = R_precision / nb_sample | |
| matching_score_real = matching_score_real / nb_sample | |
| matching_score_pred = matching_score_pred / nb_sample | |
| fid = calculate_frechet_distance(gt_mu, gt_cov, mu, cov) | |
| fid_syn = calculate_frechet_distance(gt_mu,gt_cov,mu_per,cov_per) | |
| fid_perturbation = calculate_frechet_distance(mu_per, cov_per, mu, cov) | |
| msg = f"--> \t Eva. Iter {nb_iter} :, FID. {fid:.4f},FID_syn{fid_syn:.5f},FID_perturbation_and_origin.{fid_perturbation:.5f} Diversity Real. {diversity_real:.4f}, Diversity. {diversity:.4f}, R_precision_real. {R_precision_real}, R_precision. {R_precision}, matching_score_real. {matching_score_real}, matching_score_pred. {matching_score_pred}" | |
| logger.info(msg) | |
| if draw: | |
| writer.add_scalar('./Test/FID', fid, nb_iter) | |
| writer.add_scalar('./Test/FID_perturbation_and_origin', fid_perturbation, nb_iter) | |
| writer.add_scalar('./Test/FID_syn', fid_syn, nb_iter) | |
| writer.add_scalar('./Test/Diversity', diversity, nb_iter) | |
| writer.add_scalar('./Test/top1', R_precision[0], nb_iter) | |
| writer.add_scalar('./Test/top2', R_precision[1], nb_iter) | |
| writer.add_scalar('./Test/top3', R_precision[2], nb_iter) | |
| writer.add_scalar('./Test/matching_score', matching_score_pred, nb_iter) | |
| # if nb_iter % 10000 == 0 : | |
| # for ii in range(4): | |
| # tensorborad_add_video_xyz(writer, draw_org[ii], nb_iter, tag='./Vis/org_eval'+str(ii), nb_vis=1, title_batch=[draw_text[ii]], outname=[os.path.join(out_dir, 'gt'+str(ii)+'.gif')] if savegif else None) | |
| # if nb_iter % 10000 == 0 : | |
| # for ii in range(4): | |
| # tensorborad_add_video_xyz(writer, draw_pred[ii], nb_iter, tag='./Vis/pred_eval'+str(ii), nb_vis=1, title_batch=[draw_text_pred[ii]], outname=[os.path.join(out_dir, 'pred'+str(ii)+'.gif')] if savegif else None) | |
| if isinstance(best_fid, tuple): | |
| best_fid=best_fid[0] | |
| if isinstance(best_fid_perturbation, tuple): | |
| best_fid_perturbation=best_fid_perturbation[0] | |
| if fid < best_fid : | |
| msg = f"--> --> \t FID Improved from {best_fid:.5f} to {fid:.5f} !!!" | |
| logger.info(msg) | |
| best_fid, best_iter = fid, nb_iter | |
| if save: | |
| state_dict = clip_model.state_dict() | |
| torch.save(state_dict, os.path.join(out_dir, 'clip_best.pth')) | |
| torch.save({'trans' : trans.state_dict()}, os.path.join(out_dir, 'net_best_fid.pth')) | |
| msg = f"--> --> \t Current FID is {fid:.5f} !!!" | |
| logger.info(msg) | |
| if fid_syn < best_fid_syn: | |
| msg = f"--> --> \t FID_syn {best_fid_syn:.5f} to {fid_syn:.5f} !!!" | |
| logger.info(msg) | |
| best_fid_syn = fid_syn | |
| if fid_perturbation < best_fid_perturbation : | |
| msg = f"--> --> \t FID_perturbation_and_origin {best_fid_perturbation:.5f} to {fid_perturbation:.5f} !!!" | |
| logger.info(msg) | |
| best_fid_perturbation = fid_perturbation | |
| if matching_score_pred < best_matching : | |
| msg = f"--> --> \t matching_score Improved from {best_matching:.5f} to {matching_score_pred:.5f} !!!" | |
| logger.info(msg) | |
| best_matching = matching_score_pred | |
| if abs(diversity_real - diversity) < abs(diversity_real - best_div) : | |
| msg = f"--> --> \t Diversity Improved from {best_div:.5f} to {diversity:.5f} !!!" | |
| logger.info(msg) | |
| best_div = diversity | |
| if R_precision[0] > best_top1 : | |
| msg = f"--> --> \t Top1 Improved from {best_top1:.4f} to {R_precision[0]:.4f} !!!" | |
| logger.info(msg) | |
| best_top1 = R_precision[0] | |
| if R_precision[1] > best_top2 : | |
| msg = f"--> --> \t Top2 Improved from {best_top2:.4f} to {R_precision[1]:.4f} !!!" | |
| logger.info(msg) | |
| best_top2 = R_precision[1] | |
| if R_precision[2] > best_top3 : | |
| msg = f"--> --> \t Top3 Improved from {best_top3:.4f} to {R_precision[2]:.4f} !!!" | |
| logger.info(msg) | |
| best_top3 = R_precision[2] | |
| if save: | |
| state_dict = clip_model.state_dict() | |
| torch.save(state_dict, os.path.join(out_dir, 'clip_last.pth')) | |
| torch.save({'trans' : trans.state_dict()}, os.path.join(out_dir, 'net_last.pth')) | |
| trans.train() | |
| return best_fid, best_fid_syn, best_fid_perturbation, best_iter, best_div, best_top1, best_top2, best_top3, best_matching, writer, logger | |
| def evaluation_transformer_test(out_dir, val_loader, net, trans, logger, writer, nb_iter, best_fid,best_fid_word_perb,best_fid_perturbation, best_iter, best_div, best_top1, best_top2, best_top3, best_matching, best_multi, clip_model, eval_wrapper, draw = True, save = True, savegif=False, savenpy=False) : | |
| trans.eval() | |
| nb_sample = 0 | |
| draw_org = [] | |
| draw_pred = [] | |
| draw_text = [] | |
| draw_text_pred = [] | |
| draw_name = [] | |
| motion_annotation_list = [] | |
| motion_pred_list = [] | |
| motion_pred_per_list = [] | |
| motion_multimodality = [] | |
| R_precision_real = 0 | |
| R_precision = 0 | |
| matching_score_real = 0 | |
| matching_score_pred = 0 | |
| nb_sample = 0 | |
| for batch in tqdm(val_loader, desc="Validation Progress"): | |
| word_embeddings, pos_one_hots, clip_text, clip_text_perb, sent_len, pose, m_length, token, name = batch | |
| bs, seq = pose.shape[:2] | |
| num_joints = 21 if pose.shape[-1] == 251 else 22 | |
| text = clip.tokenize(clip_text, truncate=True).cuda() | |
| text_perb = clip.tokenize(clip_text_perb, truncate=True).cuda() | |
| feat_clip_text = clip_model.encode_text(text)[0].float() | |
| feat_clip_text_per = clip_model.encode_text(text_perb)[0].float() | |
| motion_multimodality_batch = [] | |
| for i in range(1): | |
| pred_pose_eval = torch.zeros((bs, seq, pose.shape[-1])).cuda() | |
| pred_pose_eval_per = torch.zeros((bs, seq, pose.shape[-1])).cuda() | |
| pred_len = torch.ones(bs).long() | |
| pred_len_per = torch.ones(bs).long() | |
| for k in range(bs): | |
| try: | |
| index_motion = trans.sample(feat_clip_text[k:k+1], True) | |
| index_motion_per = trans.sample(feat_clip_text_per[k:k+1], True) | |
| except: | |
| index_motion = torch.ones(1,1).cuda().long() | |
| index_motion_per = torch.ones(1,1).cuda().long() | |
| pred_pose = net.forward_decoder(index_motion) | |
| pred_pose_per = net.forward_decoder(index_motion_per) | |
| cur_len = pred_pose.shape[1] | |
| cur_len_per = pred_pose_per.shape[1] | |
| pred_len[k] = min(cur_len, seq) | |
| pred_len_per[k] = min(cur_len_per, seq) | |
| pred_pose_eval[k:k+1, :cur_len] = pred_pose[:, :seq] | |
| pred_pose_eval_per[k:k+1, :cur_len_per] = pred_pose_per[:, :seq] | |
| if i == 0 and (draw or savenpy): | |
| pred_denorm = val_loader.dataset.inv_transform(pred_pose.detach().cpu().numpy()) | |
| pred_xyz = recover_from_ric(torch.from_numpy(pred_denorm).float().cuda(), num_joints) | |
| if savenpy: | |
| np.save(os.path.join(out_dir, name[k]+'_pred.npy'), pred_xyz.detach().cpu().numpy()) | |
| if draw: | |
| if i == 0: | |
| draw_pred.append(pred_xyz) | |
| draw_text_pred.append(clip_text[k]) | |
| draw_name.append(name[k]) | |
| et_pred, em_pred = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, pred_pose_eval, pred_len) | |
| et_pred_per, em_pred_per = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, pred_pose_eval_per, pred_len_per) | |
| # motion_multimodality_batch.append(em_pred.reshape(bs, 1, -1)) | |
| if i == 0: | |
| pose = pose.cuda().float() | |
| et, em = eval_wrapper.get_co_embeddings(word_embeddings, pos_one_hots, sent_len, pose, m_length) | |
| motion_annotation_list.append(em) | |
| motion_pred_list.append(em_pred) | |
| motion_pred_per_list.append(em_pred_per) | |
| if draw or savenpy: | |
| pose = val_loader.dataset.inv_transform(pose.detach().cpu().numpy()) | |
| pose_xyz = recover_from_ric(torch.from_numpy(pose).float().cuda(), num_joints) | |
| if savenpy: | |
| for j in range(bs): | |
| np.save(os.path.join(out_dir, name[j]+'_gt.npy'), pose_xyz[j][:m_length[j]].unsqueeze(0).cpu().numpy()) | |
| if draw: | |
| for j in range(bs): | |
| draw_org.append(pose_xyz[j][:m_length[j]].unsqueeze(0)) | |
| draw_text.append(clip_text[j]) | |
| temp_R, temp_match = calculate_R_precision(et.cpu().numpy(), em.cpu().numpy(), top_k=3, sum_all=True) | |
| R_precision_real += temp_R | |
| matching_score_real += temp_match | |
| temp_R, temp_match = calculate_R_precision(et_pred.cpu().numpy(), em_pred.cpu().numpy(), top_k=3, sum_all=True) | |
| R_precision += temp_R | |
| matching_score_pred += temp_match | |
| nb_sample += bs | |
| # motion_multimodality.append(torch.cat(motion_multimodality_batch, dim=1)) | |
| motion_annotation_np = torch.cat(motion_annotation_list, dim=0).cpu().numpy() | |
| motion_pred_np = torch.cat(motion_pred_list, dim=0).cpu().numpy() | |
| motion_pred_per_np = torch.cat(motion_pred_per_list, dim=0).cpu().numpy() | |
| gt_mu, gt_cov = calculate_activation_statistics(motion_annotation_np) | |
| mu, cov= calculate_activation_statistics(motion_pred_np) # mu cov使用的是motion_perb_np | |
| mu_per, cov_per= calculate_activation_statistics(motion_pred_per_np) | |
| gt_mu[np.isnan(gt_mu) | np.isinf(gt_mu)] = 0.0 | |
| gt_cov[np.isnan(gt_cov) | np.isinf(gt_cov)] = 0.0 | |
| mu[np.isnan(mu) | np.isinf(mu)] = 0.0 | |
| cov[np.isnan(cov) | np.isinf(cov)] = 0.0 | |
| mu_per[np.isnan(mu_per) | np.isinf(mu_per)] = 0.0 | |
| cov_per[np.isnan(cov_per) | np.isinf(cov_per)] = 0.0 | |
| diversity_real = calculate_diversity(motion_annotation_np, 300 if nb_sample > 300 else 100) | |
| diversity = calculate_diversity(motion_pred_np, 300 if nb_sample > 300 else 100) | |
| R_precision_real = R_precision_real / nb_sample | |
| R_precision = R_precision / nb_sample | |
| matching_score_real = matching_score_real / nb_sample | |
| matching_score_pred = matching_score_pred / nb_sample | |
| multimodality = 0 | |
| # motion_multimodality = torch.cat(motion_multimodality, dim=0).cpu().numpy() | |
| # multimodality = calculate_multimodality(motion_multimodality, 10) | |
| try: | |
| fid = calculate_frechet_distance(gt_mu, gt_cov, mu, cov) | |
| fid_perturbation = calculate_frechet_distance(mu_per, cov_per, mu, cov) | |
| fid_word_perb = calculate_frechet_distance(gt_mu,gt_cov,mu_per,cov_per) | |
| except: | |
| print('数据有问题！！') | |
| msg = f"--> \t Eva. Iter {nb_iter} :, FID. {fid:.4f}, FID_syn. {fid_word_perb:.5f}, FID_Perturbation. {fid_perturbation:.4f}, Diversity Real. {diversity_real:.4f}, Diversity. {diversity:.4f}, R_precision_real. {R_precision_real}, R_precision. {R_precision}, matching_score_real. {matching_score_real}, matching_score_pred. {matching_score_pred}, multimodality. {multimodality:.4f}" | |
| logger.info(msg) | |
| if draw: | |
| for ii in range(len(draw_org)): | |
| tensorborad_add_video_xyz(writer, draw_org[ii], nb_iter, tag='./Vis/'+draw_name[ii]+'_org', nb_vis=1, title_batch=[draw_text[ii]], outname=[os.path.join(out_dir, draw_name[ii]+'_skel_gt.gif')] if savegif else None) | |
| tensorborad_add_video_xyz(writer, draw_pred[ii], nb_iter, tag='./Vis/'+draw_name[ii]+'_pred', nb_vis=1, title_batch=[draw_text_pred[ii]], outname=[os.path.join(out_dir, draw_name[ii]+'_skel_pred.gif')] if savegif else None) | |
| trans.train() | |
| return fid,fid_word_perb,fid_perturbation, best_iter, diversity, R_precision[0], R_precision[1], R_precision[2], matching_score_pred, multimodality, writer, logger | |
| # (X - X_train)*(X - X_train) = -2X*X_train + X*X + X_train*X_train | |
| def euclidean_distance_matrix(matrix1, matrix2): | |
| """ | |
| Params: | |
| -- matrix1: N1 x D | |
| -- matrix2: N2 x D | |
| Returns: | |
| -- dist: N1 x N2 | |
| dist[i, j] == distance(matrix1[i], matrix2[j]) | |
| """ | |
| assert matrix1.shape[1] == matrix2.shape[1] | |
| d1 = -2 * np.dot(matrix1, matrix2.T) # shape (num_test, num_train) | |
| d2 = np.sum(np.square(matrix1), axis=1, keepdims=True) # shape (num_test, 1) | |
| d3 = np.sum(np.square(matrix2), axis=1) # shape (num_train, ) | |
| dists = np.sqrt(d1 + d2 + d3) # broadcasting | |
| return dists | |
| def calculate_top_k(mat, top_k): | |
| size = mat.shape[0] | |
| gt_mat = np.expand_dims(np.arange(size), 1).repeat(size, 1) | |
| bool_mat = (mat == gt_mat) | |
| correct_vec = False | |
| top_k_list = [] | |
| for i in range(top_k): | |
| # print(correct_vec, bool_mat[:, i]) | |
| correct_vec = (correct_vec | bool_mat[:, i]) | |
| # print(correct_vec) | |
| top_k_list.append(correct_vec[:, None]) | |
| top_k_mat = np.concatenate(top_k_list, axis=1) | |
| return top_k_mat | |
| def calculate_R_precision(embedding1, embedding2, top_k, sum_all=False): | |
| dist_mat = euclidean_distance_matrix(embedding1, embedding2) | |
| matching_score = dist_mat.trace() | |
| argmax = np.argsort(dist_mat, axis=1) | |
| top_k_mat = calculate_top_k(argmax, top_k) | |
| if sum_all: | |
| return top_k_mat.sum(axis=0), matching_score | |
| else: | |
| return top_k_mat, matching_score | |
| def calculate_multimodality(activation, multimodality_times): | |
| assert len(activation.shape) == 3 | |
| assert activation.shape[1] > multimodality_times | |
| num_per_sent = activation.shape[1] | |
| first_dices = np.random.choice(num_per_sent, multimodality_times, replace=False) | |
| second_dices = np.random.choice(num_per_sent, multimodality_times, replace=False) | |
| dist = linalg.norm(activation[:, first_dices] - activation[:, second_dices], axis=2) | |
| return dist.mean() | |
| def calculate_diversity(activation, diversity_times): | |
| assert len(activation.shape) == 2 | |
| assert activation.shape[0] > diversity_times | |
| num_samples = activation.shape[0] | |
| first_indices = np.random.choice(num_samples, diversity_times, replace=False) | |
| second_indices = np.random.choice(num_samples, diversity_times, replace=False) | |
| dist = linalg.norm(activation[first_indices] - activation[second_indices], axis=1) | |
| return dist.mean() | |
| def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6): | |
| mu1 = np.atleast_1d(mu1) | |
| mu2 = np.atleast_1d(mu2) | |
| sigma1 = np.atleast_2d(sigma1) | |
| sigma2 = np.atleast_2d(sigma2) | |
| assert mu1.shape == mu2.shape, \ | |
| 'Training and test mean vectors have different lengths' | |
| assert sigma1.shape == sigma2.shape, \ | |
| 'Training and test covariances have different dimensions' | |
| diff = mu1 - mu2 | |
| # Product might be almost singular | |
| covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False) | |
| if not np.isfinite(covmean).all(): | |
| msg = ('fid calculation produces singular product; ' | |
| 'adding %s to diagonal of cov estimates') % eps | |
| print(msg) | |
| offset = np.eye(sigma1.shape[0]) * eps | |
| covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset)) | |
| # Numerical error might give slight imaginary component | |
| if np.iscomplexobj(covmean): | |
| if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3): | |
| m = np.max(np.abs(covmean.imag)) | |
| raise ValueError('Imaginary component {}'.format(m)) | |
| covmean = covmean.real | |
| tr_covmean = np.trace(covmean) | |
| return (diff.dot(diff) + np.trace(sigma1) | |
| + np.trace(sigma2) - 2 * tr_covmean) | |
| def calculate_activation_statistics(activations): | |
| mu = np.mean(activations, axis=0) | |
| cov = np.cov(activations, rowvar=False) | |
| return mu, cov | |
| def calculate_frechet_feature_distance(feature_list1, feature_list2): | |
| feature_list1 = np.stack(feature_list1) | |
| feature_list2 = np.stack(feature_list2) | |
| # normalize the scale | |
| mean = np.mean(feature_list1, axis=0) | |
| std = np.std(feature_list1, axis=0) + 1e-10 | |
| feature_list1 = (feature_list1 - mean) / std | |
| feature_list2 = (feature_list2 - mean) / std | |
| dist = calculate_frechet_distance( | |
| mu1=np.mean(feature_list1, axis=0), | |
| sigma1=np.cov(feature_list1, rowvar=False), | |
| mu2=np.mean(feature_list2, axis=0), | |
| sigma2=np.cov(feature_list2, rowvar=False), | |
| ) | |
| return dist |