a

.gitignore

@@ -0,0 +1,2 @@
+*.pkl
+*.pt

app.py

@@ -38,7 +38,10 @@ def predict(file_path: str):
     temp_bash_file = create_bash_file(temp_file_path)
     
     os.system(f"bash {temp_bash_file}")
-    return temp_file_path
+    
+    res_file_path = "/tmp/denoising/save/predicted_infos_seed_0_tag_20231104_017_jts_spatial_t_100__st_0.npy"
+    
+    return res_file_path
     
 
 demo = gr.Interface(

data_loaders/humanml/data/dataset_ours_single_seq.py

@@ -6184,6 +6184,14 @@ class GRAB_Dataset_V19_Arctic_from_Pred(torch.utils.data.Dataset): # GRAB datass
         )
         
         
+        self.reversed_list = torch.tensor(
+            [0, 5, 6, 7, 9, 10, 11, 17, 18, 19, 13, 14, 15, 1, 2, 3, 4, 8, 12, 16, 20], dtype=torch.long
+        )
+        
+        self.forward_list = torch.tensor(
+            [0, 13, 14, 15, 16, 1, 2, 3, 17, 4, 5, 6, 18, 10, 11, 12, 19, 7, 8, 9, 20], dtype=torch.long
+        )
+        
         ### Load field data from root folders ### ## obj root folder ##
         self.obj_root_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_objs"
         self.obj_params_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_params"
@@ -7113,16 +7121,22 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         self.start_idx = self.args.start_idx
         
         # load datas # grab path; grab sequences #
-        grab_path =  "/data1/xueyi/GRAB_extracted"
-        ## grab contactmesh ## id2objmeshname
-        obj_mesh_path = os.path.join(grab_path, 'tools/object_meshes/contact_meshes')
-        id2objmeshname = []
-        obj_meshes = sorted(os.listdir(obj_mesh_path))
-        # objectmesh name #
-        id2objmeshname = [obj_meshes[i].split(".")[0] for i in range(len(obj_meshes))]
-        self.id2objmeshname = id2objmeshname
-        
+        # grab_path =  "/data1/xueyi/GRAB_extracted"
+        # ## grab contactmesh ## id2objmeshname
+        # obj_mesh_path = os.path.join(grab_path, 'tools/object_meshes/contact_meshes')
+        # id2objmeshname = []
+        # obj_meshes = sorted(os.listdir(obj_mesh_path))
+        # # objectmesh name #
+        # id2objmeshname = [obj_meshes[i].split(".")[0] for i in range(len(obj_meshes))]
+        # self.id2objmeshname = id2objmeshname
+        
+        self.reversed_list = torch.tensor(
+            [0, 5, 6, 7, 9, 10, 11, 17, 18, 19, 13, 14, 15, 1, 2, 3, 4, 8, 12, 16, 20], dtype=torch.long
+        )
         
+        self.forward_list = torch.tensor(
+            [0, 13, 14, 15, 16, 1, 2, 3, 17, 4, 5, 6, 18, 10, 11, 12, 19, 7, 8, 9, 20], dtype=torch.long
+        )
         
         self.aug_trans_T = 0.05
         self.aug_rot_T = 0.3
@@ -7142,9 +7156,10 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         
         ## predicted infos fn ##
         self.data_folder = data_folder
-        self.subj_data_folder = '/data1/xueyi/GRAB_processed_wsubj'
+        # self.subj_data_folder = '/data1/xueyi/GRAB_processed_wsubj'
         # self.subj_corr_data_folder = args.subj_corr_data_folder
-        self.mano_path = "/data1/xueyi/mano_models/mano/models" ### mano_path
+        # self.mano_path = "/data1/xueyi/mano_models/mano/models" ### mano_path
+        self.mano_path = "./"
         ## mano paths ##
         self.aug = True
         self.use_anchors = False
@@ -7152,19 +7167,19 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         
         self.use_anchors = args.use_anchors
         
-        self.grab_path = "/data1/xueyi/GRAB_extracted"
-        obj_mesh_path = os.path.join(self.grab_path, 'tools/object_meshes/contact_meshes')
-        id2objmesh = []
-        obj_meshes = sorted(os.listdir(obj_mesh_path))
-        for i, fn in enumerate(obj_meshes):
-            id2objmesh.append(os.path.join(obj_mesh_path, fn))
-        self.id2objmesh = id2objmesh
-        self.id2meshdata = {}
+        # self.grab_path = "/data1/xueyi/GRAB_extracted"
+        # obj_mesh_path = os.path.join(self.grab_path, 'tools/object_meshes/contact_meshes')
+        # id2objmesh = []
+        # obj_meshes = sorted(os.listdir(obj_mesh_path))
+        # for i, fn in enumerate(obj_meshes):
+        #     id2objmesh.append(os.path.join(obj_mesh_path, fn))
+        # self.id2objmesh = id2objmesh
+        # self.id2meshdata = {}
         
         ## obj root folder; ##
-        ### Load field data from root folders ###
-        self.obj_root_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_objs"
-        self.obj_params_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_params" # # and base points 
+        # ### Load field data from root folders ###
+        # self.obj_root_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_objs"
+        # self.obj_params_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_params" # # and base points 
         
         self.load_meta = True
         
@@ -7204,36 +7219,6 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
             self.predicted_hand_joints = outputs # nf x nnjoints x 3 #
             self.predicted_hand_joints = torch.from_numpy(self.predicted_hand_joints).float()
             
-            # if 'rhand_trans' in data:
-            #     # outputs = data['outputs']
-            #     self.predicted_hand_trans = data['rhand_trans'] # nframes x 3 
-            #     self.predicted_hand_rot = data['rhand_rot'] # nframes x 3 
-            #     self.predicted_hand_theta = data['rhand_theta']
-            #     self.predicted_hand_beta = data['rhand_beta']
-            #     self.predicted_hand_trans = torch.from_numpy(self.predicted_hand_trans).float() # nframes x 3 
-            #     self.predicted_hand_rot = torch.from_numpy(self.predicted_hand_rot).float() # nframes x 3 
-            #     self.predicted_hand_theta = torch.from_numpy(self.predicted_hand_theta).float() # nframes x 24 
-            #     self.predicted_hand_beta = torch.from_numpy(self.predicted_hand_beta).float() # 10,
-                
-            #     self.predicted_hand_trans_opt = data_opt['rhand_trans'] # nframes x 3 
-            #     self.predicted_hand_rot_opt = data_opt['rhand_rot'] # nframes x 3 
-            #     self.predicted_hand_theta_opt = data_opt['rhand_theta']
-            #     self.predicted_hand_beta_opt = data_opt['rhand_beta']
-            #     self.predicted_hand_trans_opt = torch.from_numpy(self.predicted_hand_trans_opt).float() # nframes x 3 
-            #     self.predicted_hand_rot_opt = torch.from_numpy(self.predicted_hand_rot_opt).float() # nframes x 3 
-            #     self.predicted_hand_theta_opt = torch.from_numpy(self.predicted_hand_theta_opt).float() # nframes x 24 
-            #     self.predicted_hand_beta_opt = torch.from_numpy(self.predicted_hand_beta_opt).float() # 10,
-                
-            #     self.predicted_hand_trans[9:] = self.predicted_hand_trans_opt[9:]
-            #     self.predicted_hand_rot[9:] = self.predicted_hand_rot_opt[9:]
-            #     self.predicted_hand_theta[ 9:] = self.predicted_hand_theta_opt[ 9:]
-            #     # self.predicted_hand_trans[:, 9:] = self.predicted_hand_trans_opt[:, 9:]
-                
-            # else:
-            #     self.predicted_hand_trans = None
-            #     self.predicted_hand_rot = None
-            #     self.predicted_hand_theta = None
-            #     self.predicted_hand_beta = None
             
         else:
             self.predicted_hand_joints = None
@@ -7276,25 +7261,25 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         )
         
         
-        ### Load field data from root folders ### ## obj root folder ##
-        self.obj_root_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_objs"
-        self.obj_params_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_params"
+        # ### Load field data from root folders ### ## obj root folder ##
+        # self.obj_root_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_objs"
+        # self.obj_params_folder = "/data1/xueyi/GRAB_extracted/tools/object_meshes/contact_meshes_params"
         
         
-        # anchor_load_driver, masking_load_driver #
-        # use_anchors, self.hand_palm_vertex_mask #
-        if self.use_anchors: # use anchors # anchor_load_driver, masking_load_driver #
-            # anchor_load_driver, masking_load_driver # 
-            inpath = "/home/xueyi/sim/CPF/assets" # contact potential field; assets # ##
-            fvi, aw, _, _ = anchor_load_driver(inpath)
-            self.face_vertex_index = torch.from_numpy(fvi).long()
-            self.anchor_weight = torch.from_numpy(aw).float()
+        # # anchor_load_driver, masking_load_driver #
+        # # use_anchors, self.hand_palm_vertex_mask #
+        # if self.use_anchors: # use anchors # anchor_load_driver, masking_load_driver #
+        #     # anchor_load_driver, masking_load_driver # 
+        #     inpath = "/home/xueyi/sim/CPF/assets" # contact potential field; assets # ##
+        #     fvi, aw, _, _ = anchor_load_driver(inpath)
+        #     self.face_vertex_index = torch.from_numpy(fvi).long()
+        #     self.anchor_weight = torch.from_numpy(aw).float()
             
-            anchor_path = os.path.join("/home/xueyi/sim/CPF/assets", "anchor")
-            palm_path = os.path.join("/home/xueyi/sim/CPF/assets", "hand_palm_full.txt")
-            hand_region_assignment, hand_palm_vertex_mask = masking_load_driver(anchor_path, palm_path)
-            # self.hand_palm_vertex_mask for hand palm mask #
-            self.hand_palm_vertex_mask = torch.from_numpy(hand_palm_vertex_mask).bool() ## the mask for hand palm to get hand anchors #
+        #     anchor_path = os.path.join("/home/xueyi/sim/CPF/assets", "anchor")
+        #     palm_path = os.path.join("/home/xueyi/sim/CPF/assets", "hand_palm_full.txt")
+        #     hand_region_assignment, hand_palm_vertex_mask = masking_load_driver(anchor_path, palm_path)
+        #     # self.hand_palm_vertex_mask for hand palm mask #
+        #     self.hand_palm_vertex_mask = torch.from_numpy(hand_palm_vertex_mask).bool() ## the mask for hand palm to get hand anchors #
         
         files_clean = [self.seq_path]
         
@@ -7325,57 +7310,6 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         # (288, 4, 4)
         return clip_data
         
-        # if f is None:
-        #   cur_clip = self.clips[clip_idx]
-        #   if len(cur_clip) > 3:
-        #       return cur_clip
-        #   f = cur_clip[2]
-        # clip_clean = np.load(f)
-        # # pert_folder_nm = self.split + '_pert'
-        # pert_folder_nm = self.split
-        # # if not self.use_pert:
-        # #     pert_folder_nm = self.split
-        # # clip_pert = np.load(os.path.join(self.data_folder, pert_folder_nm, os.path.basename(f)))
-        
-        
-        # ##### load subj params #####
-        # pure_file_name = f.split("/")[-1].split(".")[0]
-        # pure_subj_params_fn = f"{pure_file_name}_subj.npy"  
-                
-        # subj_params_fn = os.path.join(self.subj_data_folder, self.split, pure_subj_params_fn)
-        # subj_params = np.load(subj_params_fn, allow_pickle=True).item()
-        # rhand_transl = subj_params["rhand_transl"]
-        # rhand_betas = subj_params["rhand_betas"]
-        # # rhand_pose = clip_clean['f2'] ## rhand pose ##
-        
-        # object_global_orient = clip_clean['f5'] ## clip_len x 3 --> orientation 
-        # object_trcansl = clip_clean['f6'] ## cliplen x 3 --> translation
-        
-        # object_idx = clip_clean['f7'][0].item()
-        
-        # pert_subj_params_fn = os.path.join(self.subj_data_folder, pert_folder_nm, pure_subj_params_fn)
-        # pert_subj_params = np.load(pert_subj_params_fn, allow_pickle=True).item()
-        # ##### load subj params #####
-        
-        # # meta data -> lenght of the current clip  -> construct meta data from those saved meta data -> load file on the fly # clip file name -> yes...
-        # # print(f"rhand_transl: {rhand_transl.shape},rhand_betas: {rhand_betas.shape}, rhand_pose: {rhand_pose.shape} ")
-        # ### pert and clean pair for encoding and decoding ###
-        
-        # # maxx_clip_len = 
-        # loaded_clip = (
-        #     0, rhand_transl.shape[0], clip_clean,
-        #     [clip_clean['f9'], clip_clean['f11'], clip_clean['f10'], clip_clean['f1'],  clip_clean['f2'], rhand_transl, rhand_betas, object_global_orient, object_trcansl, object_idx], pert_subj_params, 
-        # )
-        # # self.clips[clip_idx] = loaded_clip
-        
-        # return loaded_clip
-        
-        # self.clips.append((self.len, self.len+clip_len, clip_pert,
-        #     [clip_clean['f9'], clip_clean['f11'], clip_clean['f10'], clip_clean['f1'],  clip_clean['f2'], rhand_transl, rhand_betas], pert_subj_params, 
-        #     # subj_corr_data, pert_subj_corr_data
-        #     ))
-        
-    
         
     def get_idx_to_mesh_data(self, obj_id):
         if obj_id not in self.id2meshdata:
@@ -7520,9 +7454,7 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         ### aug_global_orient_var, aug_pose_var, aug_transl_var ###
         #### ==== get random augmented pose, rot, transl ==== ####
         # rnd_aug_global_orient_var, rnd_aug_pose_var, rnd_aug_transl_var #
-        aug_trans, aug_rot, aug_pose = 0.01, 0.05, 0.3
-        aug_trans, aug_rot, aug_pose = 0.001, 0.05, 0.3
-        aug_trans, aug_rot, aug_pose = 0.000, 0.05, 0.3
+
         aug_trans, aug_rot, aug_pose = 0.000, 0.00, 0.00
         # noise scale #
         # aug_trans, aug_rot, aug_pose = 0.01, 0.05, 0.3 # scale 1 for the standard scale
@@ -7570,17 +7502,19 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         rhand_verts = rhand_verts * 0.001
         rhand_joints = rhand_joints * 0.001
         
+        rhand_joints = rhand_joints[:, self.reversed_list]
+        
         offset_cur_to_raw = raw_hand_verts[0, 0] - rhand_verts[0, 0]
         rhand_verts = rhand_verts + offset_cur_to_raw.unsqueeze(0)
         rhand_joints = rhand_joints + offset_cur_to_raw.unsqueeze(0)
         
-        # rhand_anchors, pert_rhand_anchors #
-        # rhand_anchors, canon_rhand_anchors #
-        # use_anchors, self.hand_palm_vertex_mask #
-        if self.use_anchors: # # rhand_anchors: bsz x nn_hand_anchors x 3 #
-            # rhand_anchors = rhand_verts[:, self.hand_palm_vertex_mask] # nf x nn_anchors x 3 --> for the anchor points ##
-            rhand_anchors = recover_anchor_batch(rhand_verts, self.face_vertex_index, self.anchor_weight.unsqueeze(0).repeat(self.window_size, 1, 1))
-            # print(f"rhand_anchors: {rhand_anchors.size()}") ### recover rhand verts here ###
+        # # rhand_anchors, pert_rhand_anchors #
+        # # rhand_anchors, canon_rhand_anchors #
+        # # use_anchors, self.hand_palm_vertex_mask #
+        # if self.use_anchors: # # rhand_anchors: bsz x nn_hand_anchors x 3 #
+        #     # rhand_anchors = rhand_verts[:, self.hand_palm_vertex_mask] # nf x nn_anchors x 3 --> for the anchor points ##
+        #     rhand_anchors = recover_anchor_batch(rhand_verts, self.face_vertex_index, self.anchor_weight.unsqueeze(0).repeat(self.window_size, 1, 1))
+        #     # print(f"rhand_anchors: {rhand_anchors.size()}") ### recover rhand verts here ###
         
         
         
@@ -7600,13 +7534,16 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         )
         pert_rhand_verts = pert_rhand_verts * 0.001 # verts 
         pert_rhand_joints = pert_rhand_joints * 0.001 # joints
+        
+        pert_rhand_joints = pert_rhand_joints[:, self.reversed_list]
+        
         pert_rhand_verts = pert_rhand_verts + offset_cur_to_raw.unsqueeze(0)
         pert_rhand_joints = pert_rhand_joints + offset_cur_to_raw.unsqueeze(0)
         
-        if self.use_anchors:
-            # pert_rhand_anchors = pert_rhand_verts[:, self.hand_palm_vertex_mask]
-            pert_rhand_anchors = recover_anchor_batch(pert_rhand_verts, self.face_vertex_index, self.anchor_weight.unsqueeze(0).repeat(self.window_size, 1, 1))
-            # print(f"rhand_anchors: {rhand_anchors.size()}") ### recover rhand verts here ###
+        # if self.use_anchors:
+        #     # pert_rhand_anchors = pert_rhand_verts[:, self.hand_palm_vertex_mask]
+        #     pert_rhand_anchors = recover_anchor_batch(pert_rhand_verts, self.face_vertex_index, self.anchor_weight.unsqueeze(0).repeat(self.window_size, 1, 1))
+        #     # print(f"rhand_anchors: {rhand_anchors.size()}") ### recover rhand verts here ###
         
         # use_canon_joints
         
@@ -7617,6 +7554,8 @@ class GRAB_Dataset_V19_HHO(torch.utils.data.Dataset): # GRAB datasset #
         canon_pert_rhand_verts = canon_pert_rhand_verts * 0.001 # verts 
         canon_pert_rhand_joints = canon_pert_rhand_joints * 0.001 # joints
         
+        canon_pert_rhand_joints = canon_pert_rhand_joints[:, self.reversed_list]
+        
         # if self.use_anchors:
         #     # canon_pert_rhand_anchors = canon_pert_rhand_verts[:, self.hand_palm_vertex_mask]
         #     canon_pert_rhand_anchors = recover_anchor_batch(canon_pert_rhand_verts, self.face_vertex_index, self.anchor_weight.unsqueeze(0).repeat(self.window_size, 1, 1))
@@ -8155,15 +8094,15 @@ class GRAB_Dataset_V19_From_Evaluated_Info(torch.utils.data.Dataset):
         self.inst_normalization = args.inst_normalization
         
         
-        # load datas # grab path; grab sequences #
-        grab_path =  "/data1/xueyi/GRAB_extracted"
-        ## grab contactmesh ## id2objmeshname
-        obj_mesh_path = os.path.join(grab_path, 'tools/object_meshes/contact_meshes')
-        id2objmeshname = []
-        obj_meshes = sorted(os.listdir(obj_mesh_path))
-        # objectmesh name #
-        id2objmeshname = [obj_meshes[i].split(".")[0] for i in range(len(obj_meshes))]
-        self.id2objmeshname = id2objmeshname
+        # # load datas # grab path; grab sequences #
+        # grab_path =  "/data1/xueyi/GRAB_extracted"
+        # ## grab contactmesh ## id2objmeshname
+        # obj_mesh_path = os.path.join(grab_path, 'tools/object_meshes/contact_meshes')
+        # id2objmeshname = []
+        # obj_meshes = sorted(os.listdir(obj_mesh_path))
+        # # objectmesh name #
+        # id2objmeshname = [obj_meshes[i].split(".")[0] for i in range(len(obj_meshes))]
+        # self.id2objmeshname = id2objmeshname
         
         
         ## the predicted_info_fn
@@ -8234,14 +8173,14 @@ class GRAB_Dataset_V19_From_Evaluated_Info(torch.utils.data.Dataset):
         self.use_anchors = False
 
         
-        self.grab_path = "/data1/xueyi/GRAB_extracted"
-        obj_mesh_path = os.path.join(self.grab_path, 'tools/object_meshes/contact_meshes')
-        id2objmesh = []
-        obj_meshes = sorted(os.listdir(obj_mesh_path))
-        for i, fn in enumerate(obj_meshes):
-            id2objmesh.append(os.path.join(obj_mesh_path, fn))
-        self.id2objmesh = id2objmesh
-        self.id2meshdata = {}
+        # self.grab_path = "/data1/xueyi/GRAB_extracted"
+        # obj_mesh_path = os.path.join(self.grab_path, 'tools/object_meshes/contact_meshes')
+        # id2objmesh = []
+        # obj_meshes = sorted(os.listdir(obj_mesh_path))
+        # for i, fn in enumerate(obj_meshes):
+        #     id2objmesh.append(os.path.join(obj_mesh_path, fn))
+        # self.id2objmesh = id2objmesh
+        # self.id2meshdata = {}
         
         ## obj root folder; ##
         ### Load field data from root folders ###

gradio_inter/predict_from_file.py

@@ -144,7 +144,14 @@ import pickle as pkl
 
 
 def main():
-  
+    
+    forward_map = torch.tensor(
+        [0, 13, 14, 15, 16, 1, 2, 3, 17, 4, 5, 6, 18, 10, 11, 12, 19, 7, 8, 9, 20], dtype=torch.long
+    )
+    reversed_map = torch.tensor(
+        [0, 5, 6, 7, 9, 10, 11, 17, 18, 19, 13, 14, 15, 1, 2, 3, 4, 8, 12, 16, 20], dtype=torch.long
+    )
+    
     args = train_args()
     
     fixseed(args.seed)
@@ -192,6 +199,7 @@ def main():
     st_idxes = list(range(0, num_ending_clearning_frames, nn_st_skip))
     if st_idxes[-1] + num_cleaning_frames < nn_frames:
         st_idxes.append(nn_frames - num_cleaning_frames)
+    st_idxes = [st_idxes[0]]
     print(f"st_idxes: {st_idxes}")
     
     
@@ -303,6 +311,11 @@ def main():
                     cur_ins_targets = cur_targets[cur_ins_rel_idx] / data_scale_factor
 
                     cur_ins_outputs = cur_outputs[cur_ins_rel_idx] / data_scale_factor
+                    
+                    cur_ins_targets = cur_ins_targets[..., forward_map, :]
+                    cur_ins_outputs = cur_ins_outputs[..., forward_map, :]
+                    
+                    
                     cur_ins_pert_verts = cur_pert_verts[cur_ins_rel_idx, ...]
                     cur_ins_verts = cur_verts[cur_ins_rel_idx, ...]
                     

requirements.txt

@@ -7,9 +7,9 @@ torch==1.12.1
 blobfile==2.0.1
 manopth @ git+https://github.com/hassony2/manopth.git
 numpy==1.23.1
-psutil
+psutil==5.9.2
 scikit-learn
-scipy
+scipy==1.9.3
 tensorboard
 tensorboardx
 tqdm

sample/predict_taco.py

@@ -163,8 +163,8 @@ def main():
     with open(args_path, 'w') as fw:
         json.dump(vars(args), fw, indent=4, sort_keys=True)
         
-    tot_hho_seq_paths = [args.single_seq_path]
-    tot_hho_seq_tags = ["test"]
+    # tot_hho_seq_paths = [args.single_seq_path]
+    # tot_hho_seq_tags = ["test"]
     
     data_dict = pkl.load(open(args.single_seq_path, 'rb'))
     data_hand_verts = data_dict['hand_verts']
@@ -180,9 +180,14 @@ def main():
     st_idxes = list(range(0, num_ending_clearning_frames, nn_st_skip))
     if st_idxes[-1] + num_cleaning_frames < nn_frames:
         st_idxes.append(nn_frames - num_cleaning_frames)
+        
+    st_idxes = [st_idxes[0]]
     print(f"st_idxes: {st_idxes}")
     
-    for cur_seed in range(0, 122, 11):
+    tot_seeds = [0]
+    
+    # for cur_seed in range(0, 122, 11):
+    for cur_seed in tot_seeds:
         args.seed = cur_seed 
         
         

sample/reconstruct_data_taco.py

@@ -9,14 +9,14 @@ import numpy as np
 import os, argparse, copy, json
 import pickle as pkl
 from scipy.spatial.transform import Rotation as R
-from psbody.mesh import Mesh
+# from psbody.mesh import Mesh
 from manopth.manolayer import ManoLayer
 
 import trimesh
 from utils import *
-import utils
+# import utils
 import utils.model_util as model_util
-from utils.anchor_utils import masking_load_driver, anchor_load_driver, recover_anchor_batch
+# from utils.anchor_utils import masking_load_driver, anchor_load_driver, recover_anchor_batch
 
 from utils.parser_util import train_args
 
@@ -50,41 +50,42 @@ def get_penetration_masks(obj_verts, obj_faces, hand_verts):
 # interpolatio for unsmooth hand parameters? #
 def get_optimized_hand_fr_joints_v4_anchors(joints, base_pts, tot_base_pts_trans, tot_base_normals_trans, with_contact_opt=False, nn_hand_params=24, rt_vars=False, with_proj=False, obj_verts_trans=None, obj_faces=None, with_params_smoothing=False, dist_thres=0.005, with_ctx_mask=False):
   # obj_verts_trans, obj_faces
-  joints = torch.from_numpy(joints).float().cuda() # joints 
-  base_pts = torch.from_numpy(base_pts).float().cuda() # base_pts 
+  joints = torch.from_numpy(joints).float() #  # joints 
+  base_pts = torch.from_numpy(base_pts).float() #  # base_pts 
   
   if nn_hand_params < 45:
     use_pca = True
   else:
     use_pca = False
   
-  tot_base_pts_trans = torch.from_numpy(tot_base_pts_trans).float().cuda()
-  tot_base_normals_trans = torch.from_numpy(tot_base_normals_trans).float().cuda()
+  tot_base_pts_trans = torch.from_numpy(tot_base_pts_trans).float()
+  tot_base_normals_trans = torch.from_numpy(tot_base_normals_trans).float()
   ### start optimization ###
   # setup MANO layer
-  mano_path = "/data1/xueyi/mano_models/mano/models"
+  # mano_path = "/data1/xueyi/mano_models/mano/models"
+  mano_path = "./"
   nn_hand_params = 24
   use_pca = True
-  if not use_left:
-    mano_layer = ManoLayer(
-        flat_hand_mean=False,
-        side='right',
-        mano_root=mano_path, # mano_root #
-        ncomps=nn_hand_params, # hand params # 
-        use_pca=use_pca, # pca for pca #
-        root_rot_mode='axisang',
-        joint_rot_mode='axisang'
-    ).cuda()
-  else:
-    mano_layer = ManoLayer(
-        flat_hand_mean=True,
-        side='left',
-        mano_root=mano_path, # mano_root #
-        ncomps=nn_hand_params, # hand params # 
-        use_pca=use_pca, # pca for pca #
-        root_rot_mode='axisang',
-        joint_rot_mode='axisang'
-    ).cuda()
+  # if not use_left:
+  #   mano_layer = ManoLayer(
+  #       flat_hand_mean=False,
+  #       side='right',
+  #       mano_root=mano_path, # mano_root #
+  #       ncomps=nn_hand_params, # hand params # 
+  #       use_pca=use_pca, # pca for pca #
+  #       root_rot_mode='axisang',
+  #       joint_rot_mode='axisang'
+  #   )
+  # else:
+  mano_layer = ManoLayer(
+      flat_hand_mean=True,
+      side='left',
+      mano_root=mano_path, # mano_root #
+      ncomps=nn_hand_params, # hand params # 
+      use_pca=use_pca, # pca for pca #
+      root_rot_mode='axisang',
+      joint_rot_mode='axisang'
+  )
 
   # mano_layer = ManoLayer(
   #     flat_hand_mean=False,
@@ -94,31 +95,31 @@ def get_optimized_hand_fr_joints_v4_anchors(joints, base_pts, tot_base_pts_trans
   #     use_pca=use_pca, # pca for pca #
   #     root_rot_mode='axisang',
   #     joint_rot_mode='axisang'
-  # ).cuda()
+  # )
   nn_frames = joints.size(0)
   
   
   # anchor_load_driver, masking_load_driver #
-  inpath = "/home/xueyi/sim/CPF/assets" # contact potential field; assets # ##
-  fvi, aw, _, _ = anchor_load_driver(inpath)
-  face_vertex_index = torch.from_numpy(fvi).long().cuda()
-  anchor_weight = torch.from_numpy(aw).float().cuda()
+  # inpath = "/home/xueyi/sim/CPF/assets" # contact potential field; assets # ##
+  # fvi, aw, _, _ = anchor_load_driver(inpath)
+  # face_vertex_index = torch.from_numpy(fvi).long()
+  # anchor_weight = torch.from_numpy(aw).float()
   
-  anchor_path = os.path.join("/home/xueyi/sim/CPF/assets", "anchor")
-  palm_path = os.path.join("/home/xueyi/sim/CPF/assets", "hand_palm_full.txt")
-  hand_region_assignment, hand_palm_vertex_mask = masking_load_driver(anchor_path, palm_path)
-  # self.hand_palm_vertex_mask for hand palm mask #
-  hand_palm_vertex_mask = torch.from_numpy(hand_palm_vertex_mask).bool().cuda() ## the mask for hand palm to get hand anchors #
+  # anchor_path = os.path.join("/home/xueyi/sim/CPF/assets", "anchor")
+  # palm_path = os.path.join("/home/xueyi/sim/CPF/assets", "hand_palm_full.txt")
+  # hand_region_assignment, hand_palm_vertex_mask = masking_load_driver(anchor_path, palm_path)
+  # # self.hand_palm_vertex_mask for hand palm mask #
+  # hand_palm_vertex_mask = torch.from_numpy(hand_palm_vertex_mask).bool() ## the mask for hand palm to get hand anchors #
       
   
   
 
   # initialize variables
-  beta_var = torch.randn([1, 10]).cuda()
+  beta_var = torch.randn([1, 10])
   # first 3 global orientation
-  rot_var = torch.randn([nn_frames, 3]).cuda()
-  theta_var = torch.randn([nn_frames, nn_hand_params]).cuda()
-  transl_var = torch.randn([nn_frames, 3]).cuda()
+  rot_var = torch.randn([nn_frames, 3])
+  theta_var = torch.randn([nn_frames, nn_hand_params])
+  transl_var = torch.randn([nn_frames, 3])
   
   # 3 + 45 + 3 = 51 for computing #
   # transl_var = tot_rhand_transl.unsqueeze(0).repeat(args.num_init, 1, 1).contiguous().to(device).view(args.num_init * num_frames, 3).contiguous()
@@ -151,11 +152,11 @@ def get_optimized_hand_fr_joints_v4_anchors(joints, base_pts, tot_base_pts_trans
   nk_contact_pts = 2
   minn_dist[:, :-5] = 1e9
   minn_topk_dist, minn_topk_idx = torch.topk(minn_dist, k=nk_contact_pts, largest=False) # 
-  # joints_idx_rng_exp = torch.arange(nn_joints).unsqueeze(0).cuda() == 
+  # joints_idx_rng_exp = torch.arange(nn_joints).unsqueeze(0) == 
   minn_topk_mask = torch.zeros_like(minn_dist)
   # minn_topk_mask[minn_topk_idx] = 1. # nf x nnjoints #
   minn_topk_mask[:, -5: -3] = 1.
-  basepts_idx_range = torch.arange(nn_base_pts).unsqueeze(0).unsqueeze(0).cuda()
+  basepts_idx_range = torch.arange(nn_base_pts).unsqueeze(0).unsqueeze(0)
   minn_dist_mask = basepts_idx_range == minn_dist_idx.unsqueeze(-1) # nf x nnjoints x nnbasepts
   # for seq 101
   # minn_dist_mask[31:, -5, :] = minn_dist_mask[30: 31, -5, :]
@@ -349,7 +350,7 @@ def get_optimized_hand_fr_joints_v4_anchors(joints, base_pts, tot_base_pts_trans
       else:
         smoothed_theta_var.append(cur_theta_var.detach().clone())
     smoothed_theta_var = torch.stack(smoothed_theta_var, dim=0) # smoothed_theta_var: nf x nn_theta_dim
-    theta_var = torch.randn_like(smoothed_theta_var).cuda()
+    theta_var = torch.randn_like(smoothed_theta_var)
     theta_var.data = smoothed_theta_var.data
     # theta_var = smoothed_theta_var.clone()
     theta_var.requires_grad_() # for 
@@ -395,258 +396,260 @@ def get_optimized_hand_fr_joints_v4_anchors(joints, base_pts, tot_base_pts_trans
   
   
   window_size = hand_verts.size(0)
-  if with_contact_opt:
-    num_iters = 2000
-    num_iters = 1000 # seq 77 # if with contact opt #
-    # num_iters = 500 # seq 77
-    ori_theta_var = theta_var.detach().clone()
-    
-    # tot_base_pts_trans # nf x nn_base_pts x 3
-    disp_base_pts_trans = tot_base_pts_trans[1:] - tot_base_pts_trans[:-1] # (nf - 1) x nn_base_pts x 3
-    disp_base_pts_trans = torch.cat( # nf x nn_base_pts x 3 
-      [disp_base_pts_trans, disp_base_pts_trans[-1:]], dim=0
-    )
-    
-    rhand_anchors = recover_anchor_batch(hand_verts.detach(), face_vertex_index, anchor_weight.unsqueeze(0).repeat(window_size, 1, 1))
-
-    dist_joints_to_base_pts = torch.sum(
-      (rhand_anchors.unsqueeze(-2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1 # nf x nnjoints x nnbasepts #
-    )
-    
-    nn_base_pts = dist_joints_to_base_pts.size(-1)
-    nn_joints = dist_joints_to_base_pts.size(1)
-    
-    dist_joints_to_base_pts = torch.sqrt(dist_joints_to_base_pts) # nf x nnjoints x nnbasepts #
-    minn_dist, minn_dist_idx = torch.min(dist_joints_to_base_pts, dim=-1) # nf x nnjoints #
-    
-    nk_contact_pts = 2
-    # minn_dist[:, :-5] = 1e9
-    # minn_topk_dist, minn_topk_idx = torch.topk(minn_dist, k=nk_contact_pts, largest=False) # 
-    # joints_idx_rng_exp = torch.arange(nn_joints).unsqueeze(0).cuda() == 
-    # minn_topk_mask = torch.zeros_like(minn_dist)
-    # minn_topk_mask[minn_topk_idx] = 1. # nf x nnjoints #
-    # minn_topk_mask[:, -5: -3] = 1.
-    basepts_idx_range = torch.arange(nn_base_pts).unsqueeze(0).unsqueeze(0).cuda()
-    minn_dist_mask = basepts_idx_range == minn_dist_idx.unsqueeze(-1) # nf x nnjoints x nnbasepts
-    # for seq 101
-    # minn_dist_mask[31:, -5, :] = minn_dist_mask[30: 31, -5, :]
-    minn_dist_mask = minn_dist_mask.float()
-    
-    # 
-    # for seq 47
-    # if cat_nm in ["Scissors"]:
-    #   # minn_dist_mask[:] = minn_dist_mask[11:12, :, :]
-    #   # minn_dist_mask[:11] = False
-
-    #   # if i_test_seq == 24:
-    #   #   minn_dist_mask[20:] = minn_dist_mask[20:21, :, :]
-    #   # else:
-
-    #   # minn_dist_mask[:] = minn_dist_mask[11:12, :, :]
-    #   minn_dist_mask[:] = minn_dist_mask[20:21, :, :]
-    
-    attraction_mask_new = (tot_base_pts_trans_disp_mask.float().unsqueeze(-1).unsqueeze(-1) + minn_dist_mask.float()) > 1.5
-    
-    
-    
-    # joints: nf x nn_jts_pts x 3; nf x nn_base_pts x 3 
-    dist_joints_to_base_pts_trans = torch.sum(
-      (rhand_anchors.unsqueeze(2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1 # nf x nn_jts_pts x nn_base_pts
-    )
-    minn_dist_joints_to_base_pts, minn_dist_idxes = torch.min(dist_joints_to_base_pts_trans, dim=-1) # nf x nn_jts_pts # nf x nn_jts_pts # 
-    nearest_base_normals = model_util.batched_index_select_ours(tot_base_normals_trans, indices=minn_dist_idxes, dim=1) # nf x nn_base_pts x 3 --> nf x nn_jts_pts x 3 # # nf x nn_jts_pts x 3 #
-    nearest_base_pts_trans = model_util.batched_index_select_ours(disp_base_pts_trans, indices=minn_dist_idxes, dim=1) # nf x nn_jts_ts x 3 #
-    dot_nearest_base_normals_trans = torch.sum(
-      nearest_base_normals * nearest_base_pts_trans, dim=-1 # nf x nn_jts 
-    )
-    trans_normals_mask = dot_nearest_base_normals_trans < 0. # nf x nn_jts # nf x nn_jts #
-    nearest_dist = torch.sqrt(minn_dist_joints_to_base_pts)
-    
-    # dist_thres
-    nearest_dist_mask = nearest_dist < dist_thres # hoi seq
-    # nearest_dist_mask = nearest_dist < 0.005 # hoi seq
-    
-    # nearest_dist_mask = nearest_dist < 0.03 # hoi seq
-    # nearest_dist_mask = nearest_dist < 0.5 # hoi seq # seq 47
-    # nearest_dist_mask = nearest_dist < 0.1 # hoi seq # seq 47
-    
-    
-    
-    # nearest_dist_mask = nearest_dist < 0.1
-    k_attr = 100.
-    joint_attraction_k = torch.exp(-1. * k_attr * nearest_dist)
-    # attraction_mask_new_new = (attraction_mask_new.float() + trans_normals_mask.float().unsqueeze(-1) + nearest_dist_mask.float().unsqueeze(-1)) > 2.5
-    
-    attraction_mask_new_new = (attraction_mask_new.float() + nearest_dist_mask.float().unsqueeze(-1)) > 1.5
-    
-    # if cat_nm in ["ToyCar", "Pliers", "Bottle", "Mug", "Scissors"]:
-    anchor_masks = [2, 3, 4, 9, 10, 11, 15, 16, 17, 22, 23, 24, 29, 30, 31]
-    anchor_nmasks = [iid for iid in range(attraction_mask_new_new.size(1)) if iid not in anchor_masks]
-    anchor_nmasks = torch.tensor(anchor_nmasks, dtype=torch.long).cuda()
-    attraction_mask_new_new[:, anchor_nmasks, :] = False # scissors 
-    
-    # # for seq 47
-    # elif cat_nm in ["Scissors"]:
-    #   anchor_masks = [2, 3, 4, 15, 16, 17, 22, 23, 24]
-    #   anchor_nmasks = [iid for iid in range(attraction_mask_new_new.size(1)) if iid not in anchor_masks]
-    #   anchor_nmasks = torch.tensor(anchor_nmasks, dtype=torch.long).cuda()
-    #   attraction_mask_new_new[:, anchor_nmasks, :] = False
-    
-    # anchor_masks = torch.array([2, 3, 4, 15, 16, 17, 22, 23, 24], dtype=torch.long).cuda()
-    # anchor_masks = torch.arange(attraction_mask_new_new.size(1)).unsqueeze(0).unsqueeze(-1).cuda() != 
-    
-    # [2, 3, 4]
-    # [9, 10, 11]
-    # [15, 16, 17]
-    # [22, 23, 24]
-    # seq 47: [2, 3, 4, 15, 16, 17, 22, 23, 24]
-    # motion planning?  # 
-    
-    
-    transl_var_ori = transl_var.clone().detach()
-    # transl_var, theta_var, rot_var, beta_var # 
-    # opt = optim.Adam([rot_var, transl_var, theta_var], lr=learning_rate)
-    # opt = optim.Adam([transl_var, theta_var], lr=learning_rate)
-    opt = optim.Adam([transl_var, theta_var, rot_var], lr=learning_rate)
-    # opt = optim.Adam([theta_var, rot_var], lr=learning_rate)
-    scheduler = optim.lr_scheduler.StepLR(opt, step_size=num_iters, gamma=0.5)
-    for i in range(num_iters):
-        opt.zero_grad()
-        # mano_layer
-        hand_verts, hand_joints = mano_layer(torch.cat([rot_var, theta_var], dim=-1),
-            beta_var.unsqueeze(1).repeat(1, nn_frames, 1).view(-1, 10), transl_var)
-        hand_verts = hand_verts.view( nn_frames, 778, 3) * 0.001
-        hand_joints = hand_joints.view(nn_frames, -1, 3) * 0.001
+  
+  
+  # if with_contact_opt:
+  #   num_iters = 2000
+  #   num_iters = 1000 # seq 77 # if with contact opt #
+  #   # num_iters = 500 # seq 77
+  #   ori_theta_var = theta_var.detach().clone()
+    
+  #   # tot_base_pts_trans # nf x nn_base_pts x 3
+  #   disp_base_pts_trans = tot_base_pts_trans[1:] - tot_base_pts_trans[:-1] # (nf - 1) x nn_base_pts x 3
+  #   disp_base_pts_trans = torch.cat( # nf x nn_base_pts x 3 
+  #     [disp_base_pts_trans, disp_base_pts_trans[-1:]], dim=0
+  #   )
+    
+  #   rhand_anchors = recover_anchor_batch(hand_verts.detach(), face_vertex_index, anchor_weight.unsqueeze(0).repeat(window_size, 1, 1))
+
+  #   dist_joints_to_base_pts = torch.sum(
+  #     (rhand_anchors.unsqueeze(-2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1 # nf x nnjoints x nnbasepts #
+  #   )
+    
+  #   nn_base_pts = dist_joints_to_base_pts.size(-1)
+  #   nn_joints = dist_joints_to_base_pts.size(1)
+    
+  #   dist_joints_to_base_pts = torch.sqrt(dist_joints_to_base_pts) # nf x nnjoints x nnbasepts #
+  #   minn_dist, minn_dist_idx = torch.min(dist_joints_to_base_pts, dim=-1) # nf x nnjoints #
+    
+  #   nk_contact_pts = 2
+  #   # minn_dist[:, :-5] = 1e9
+  #   # minn_topk_dist, minn_topk_idx = torch.topk(minn_dist, k=nk_contact_pts, largest=False) # 
+  #   # joints_idx_rng_exp = torch.arange(nn_joints).unsqueeze(0) == 
+  #   # minn_topk_mask = torch.zeros_like(minn_dist)
+  #   # minn_topk_mask[minn_topk_idx] = 1. # nf x nnjoints #
+  #   # minn_topk_mask[:, -5: -3] = 1.
+  #   basepts_idx_range = torch.arange(nn_base_pts).unsqueeze(0).unsqueeze(0)
+  #   minn_dist_mask = basepts_idx_range == minn_dist_idx.unsqueeze(-1) # nf x nnjoints x nnbasepts
+  #   # for seq 101
+  #   # minn_dist_mask[31:, -5, :] = minn_dist_mask[30: 31, -5, :]
+  #   minn_dist_mask = minn_dist_mask.float()
+    
+  #   # 
+  #   # for seq 47
+  #   # if cat_nm in ["Scissors"]:
+  #   #   # minn_dist_mask[:] = minn_dist_mask[11:12, :, :]
+  #   #   # minn_dist_mask[:11] = False
+
+  #   #   # if i_test_seq == 24:
+  #   #   #   minn_dist_mask[20:] = minn_dist_mask[20:21, :, :]
+  #   #   # else:
+
+  #   #   # minn_dist_mask[:] = minn_dist_mask[11:12, :, :]
+  #   #   minn_dist_mask[:] = minn_dist_mask[20:21, :, :]
+    
+  #   attraction_mask_new = (tot_base_pts_trans_disp_mask.float().unsqueeze(-1).unsqueeze(-1) + minn_dist_mask.float()) > 1.5
+    
+    
+    
+  #   # joints: nf x nn_jts_pts x 3; nf x nn_base_pts x 3 
+  #   dist_joints_to_base_pts_trans = torch.sum(
+  #     (rhand_anchors.unsqueeze(2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1 # nf x nn_jts_pts x nn_base_pts
+  #   )
+  #   minn_dist_joints_to_base_pts, minn_dist_idxes = torch.min(dist_joints_to_base_pts_trans, dim=-1) # nf x nn_jts_pts # nf x nn_jts_pts # 
+  #   nearest_base_normals = model_util.batched_index_select_ours(tot_base_normals_trans, indices=minn_dist_idxes, dim=1) # nf x nn_base_pts x 3 --> nf x nn_jts_pts x 3 # # nf x nn_jts_pts x 3 #
+  #   nearest_base_pts_trans = model_util.batched_index_select_ours(disp_base_pts_trans, indices=minn_dist_idxes, dim=1) # nf x nn_jts_ts x 3 #
+  #   dot_nearest_base_normals_trans = torch.sum(
+  #     nearest_base_normals * nearest_base_pts_trans, dim=-1 # nf x nn_jts 
+  #   )
+  #   trans_normals_mask = dot_nearest_base_normals_trans < 0. # nf x nn_jts # nf x nn_jts #
+  #   nearest_dist = torch.sqrt(minn_dist_joints_to_base_pts)
+    
+  #   # dist_thres
+  #   nearest_dist_mask = nearest_dist < dist_thres # hoi seq
+  #   # nearest_dist_mask = nearest_dist < 0.005 # hoi seq
+    
+  #   # nearest_dist_mask = nearest_dist < 0.03 # hoi seq
+  #   # nearest_dist_mask = nearest_dist < 0.5 # hoi seq # seq 47
+  #   # nearest_dist_mask = nearest_dist < 0.1 # hoi seq # seq 47
+    
+    
+    
+  #   # nearest_dist_mask = nearest_dist < 0.1
+  #   k_attr = 100.
+  #   joint_attraction_k = torch.exp(-1. * k_attr * nearest_dist)
+  #   # attraction_mask_new_new = (attraction_mask_new.float() + trans_normals_mask.float().unsqueeze(-1) + nearest_dist_mask.float().unsqueeze(-1)) > 2.5
+    
+  #   attraction_mask_new_new = (attraction_mask_new.float() + nearest_dist_mask.float().unsqueeze(-1)) > 1.5
+    
+  #   # if cat_nm in ["ToyCar", "Pliers", "Bottle", "Mug", "Scissors"]:
+  #   anchor_masks = [2, 3, 4, 9, 10, 11, 15, 16, 17, 22, 23, 24, 29, 30, 31]
+  #   anchor_nmasks = [iid for iid in range(attraction_mask_new_new.size(1)) if iid not in anchor_masks]
+  #   anchor_nmasks = torch.tensor(anchor_nmasks, dtype=torch.long)
+  #   attraction_mask_new_new[:, anchor_nmasks, :] = False # scissors 
+    
+  #   # # for seq 47
+  #   # elif cat_nm in ["Scissors"]:
+  #   #   anchor_masks = [2, 3, 4, 15, 16, 17, 22, 23, 24]
+  #   #   anchor_nmasks = [iid for iid in range(attraction_mask_new_new.size(1)) if iid not in anchor_masks]
+  #   #   anchor_nmasks = torch.tensor(anchor_nmasks, dtype=torch.long)
+  #   #   attraction_mask_new_new[:, anchor_nmasks, :] = False
+    
+  #   # anchor_masks = torch.array([2, 3, 4, 15, 16, 17, 22, 23, 24], dtype=torch.long)
+  #   # anchor_masks = torch.arange(attraction_mask_new_new.size(1)).unsqueeze(0).unsqueeze(-1) != 
+    
+  #   # [2, 3, 4]
+  #   # [9, 10, 11]
+  #   # [15, 16, 17]
+  #   # [22, 23, 24]
+  #   # seq 47: [2, 3, 4, 15, 16, 17, 22, 23, 24]
+  #   # motion planning?  # 
+    
+    
+  #   transl_var_ori = transl_var.clone().detach()
+  #   # transl_var, theta_var, rot_var, beta_var # 
+  #   # opt = optim.Adam([rot_var, transl_var, theta_var], lr=learning_rate)
+  #   # opt = optim.Adam([transl_var, theta_var], lr=learning_rate)
+  #   opt = optim.Adam([transl_var, theta_var, rot_var], lr=learning_rate)
+  #   # opt = optim.Adam([theta_var, rot_var], lr=learning_rate)
+  #   scheduler = optim.lr_scheduler.StepLR(opt, step_size=num_iters, gamma=0.5)
+  #   for i in range(num_iters):
+  #       opt.zero_grad()
+  #       # mano_layer
+  #       hand_verts, hand_joints = mano_layer(torch.cat([rot_var, theta_var], dim=-1),
+  #           beta_var.unsqueeze(1).repeat(1, nn_frames, 1).view(-1, 10), transl_var)
+  #       hand_verts = hand_verts.view( nn_frames, 778, 3) * 0.001
+  #       hand_joints = hand_joints.view(nn_frames, -1, 3) * 0.001
         
-        joints_pred_loss = torch.sum(
-          (hand_joints - joints) ** 2, dim=-1
-        ).mean()
+  #       joints_pred_loss = torch.sum(
+  #         (hand_joints - joints) ** 2, dim=-1
+  #       ).mean()
         
-        # 
-        rhand_anchors = recover_anchor_batch(hand_verts, face_vertex_index, anchor_weight.unsqueeze(0).repeat(window_size, 1, 1))
+  #       # 
+  #       rhand_anchors = recover_anchor_batch(hand_verts, face_vertex_index, anchor_weight.unsqueeze(0).repeat(window_size, 1, 1))
 
         
-        # dist_joints_to_base_pts_sqr = torch.sum(
-        #     (hand_joints.unsqueeze(2) - base_pts.unsqueeze(0).unsqueeze(1)) ** 2, dim=-1
-        # ) # nf x nnb x 3 ---- nf x nnj x 1 x 3 
-        dist_joints_to_base_pts_sqr = torch.sum(
-            (rhand_anchors.unsqueeze(2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1
-        )
-        # attaction_loss = 0.5 * affinity_scores * dist_joints_to_base_pts_sqr
-        attaction_loss = 0.5 * dist_joints_to_base_pts_sqr
-        # attaction_loss = attaction_loss
-        # attaction_loss = torch.mean(attaction_loss[..., -5:, :] * minn_dist_mask[..., -5:, :])
+  #       # dist_joints_to_base_pts_sqr = torch.sum(
+  #       #     (hand_joints.unsqueeze(2) - base_pts.unsqueeze(0).unsqueeze(1)) ** 2, dim=-1
+  #       # ) # nf x nnb x 3 ---- nf x nnj x 1 x 3 
+  #       dist_joints_to_base_pts_sqr = torch.sum(
+  #           (rhand_anchors.unsqueeze(2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1
+  #       )
+  #       # attaction_loss = 0.5 * affinity_scores * dist_joints_to_base_pts_sqr
+  #       attaction_loss = 0.5 * dist_joints_to_base_pts_sqr
+  #       # attaction_loss = attaction_loss
+  #       # attaction_loss = torch.mean(attaction_loss[..., -5:, :] * minn_dist_mask[..., -5:, :])
         
-        # attaction_loss = torch.mean(attaction_loss * attraction_mask)
-        # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # attaction_loss = torch.mean(attaction_loss * attraction_mask)
+  #       # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
-        # seq 80
-        # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # seq 80
+  #       # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
-        # seq 70
-        # attaction_loss = torch.mean(attaction_loss[10:, -5:-3, :] * minn_dist_mask[10:, -5:-3, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # seq 70
+  #       # attaction_loss = torch.mean(attaction_loss[10:, -5:-3, :] * minn_dist_mask[10:, -5:-3, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
-        # new version relying on new mask #
-        # attaction_loss = torch.mean(attaction_loss[:, -5:-3, :] * attraction_mask_new[:, -5:-3, :])
-        ### original version ###
-        # attaction_loss = torch.mean(attaction_loss[20:, -3:, :] * attraction_mask_new[20:, -3:, :])
+  #       # new version relying on new mask #
+  #       # attaction_loss = torch.mean(attaction_loss[:, -5:-3, :] * attraction_mask_new[:, -5:-3, :])
+  #       ### original version ###
+  #       # attaction_loss = torch.mean(attaction_loss[20:, -3:, :] * attraction_mask_new[20:, -3:, :])
         
-        # attaction_loss = torch.mean(attaction_loss[:, -5:, :] * attraction_mask_new_new[:, -5:, :] * joint_attraction_k[:, -5:].unsqueeze(-1))
+  #       # attaction_loss = torch.mean(attaction_loss[:, -5:, :] * attraction_mask_new_new[:, -5:, :] * joint_attraction_k[:, -5:].unsqueeze(-1))
         
         
         
-        attaction_loss = torch.mean(attaction_loss[:, :, :] * attraction_mask_new_new[:, :, :] * joint_attraction_k[:, :].unsqueeze(-1))
+  #       attaction_loss = torch.mean(attaction_loss[:, :, :] * attraction_mask_new_new[:, :, :] * joint_attraction_k[:, :].unsqueeze(-1))
         
         
-        # seq mug
-        # attaction_loss = torch.mean(attaction_loss[4:, -5:-4, :] * minn_dist_mask[4:, -5:-4, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # seq mug
+  #       # attaction_loss = torch.mean(attaction_loss[4:, -5:-4, :] * minn_dist_mask[4:, -5:-4, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
         
-        # opt.zero_grad()
-        pose_smoothness_loss = F.mse_loss(theta_var.view(nn_frames, -1)[1:], theta_var.view(nn_frames, -1)[:-1])
-        # joints_smoothness_loss = joint_acc_loss(hand_verts, J_regressor.to(device))
-        shape_prior_loss = torch.mean(beta_var**2)
-        pose_prior_loss = torch.mean(theta_var**2)
-        joints_smoothness_loss = F.mse_loss(hand_joints.view(nn_frames, -1, 3)[1:], hand_joints.view(nn_frames, -1, 3)[:-1])
-        # =0.05
-        # # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001 + joints_smoothness_loss * 100.
-        # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.000001 + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001 + joints_smoothness_loss * 200.
+  #       # opt.zero_grad()
+  #       pose_smoothness_loss = F.mse_loss(theta_var.view(nn_frames, -1)[1:], theta_var.view(nn_frames, -1)[:-1])
+  #       # joints_smoothness_loss = joint_acc_loss(hand_verts, J_regressor.to(device))
+  #       shape_prior_loss = torch.mean(beta_var**2)
+  #       pose_prior_loss = torch.mean(theta_var**2)
+  #       joints_smoothness_loss = F.mse_loss(hand_joints.view(nn_frames, -1, 3)[1:], hand_joints.view(nn_frames, -1, 3)[:-1])
+  #       # =0.05
+  #       # # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001 + joints_smoothness_loss * 100.
+  #       # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.000001 + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001 + joints_smoothness_loss * 200.
         
-        # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + joints_smoothness_loss * 200.
+  #       # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + joints_smoothness_loss * 200.
         
-        # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.03 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + attaction_loss * 10000 # + joints_smoothness_loss * 200.
+  #       # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.03 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + attaction_loss * 10000 # + joints_smoothness_loss * 200.
         
-        theta_smoothness_loss = F.mse_loss(theta_var, ori_theta_var)
+  #       theta_smoothness_loss = F.mse_loss(theta_var, ori_theta_var)
 
-        transl_smoothness_loss = F.mse_loss(transl_var_ori, transl_var)
-        # loss = attaction_loss * 1000. + theta_smoothness_loss * 0.00001
+  #       transl_smoothness_loss = F.mse_loss(transl_var_ori, transl_var)
+  #       # loss = attaction_loss * 1000. + theta_smoothness_loss * 0.00001
         
-        # attraction loss, joint prediction loss, joints smoothness loss #
-        # loss = attaction_loss * 1000. + joints_pred_loss 
-        ### general ###
-        # loss = attaction_loss * 1000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
+  #       # attraction loss, joint prediction loss, joints smoothness loss #
+  #       # loss = attaction_loss * 1000. + joints_pred_loss 
+  #       ### general ###
+  #       # loss = attaction_loss * 1000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
         
-        # tune for seq 140
-        loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
-        # ToyCar # 
-        loss = attaction_loss * 10000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
+  #       # tune for seq 140
+  #       loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
+  #       # ToyCar # 
+  #       loss = attaction_loss * 10000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
         
-        # if cat_nm in ["Scissors"]:
-        #   # scissors and 
-        #   # if dist_thres < 0.05:
-        #   #   loss = transl_smoothness_loss * 0.5 + attaction_loss * 10000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5
-        #   # else:          
-        #   #   # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05
-        #   #   loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
-        #   #   # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
-
-        #   if dist_thres < 0.05:
-        #     # loss = transl_smoothness_loss * 0.5 + attaction_loss * 10000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5
-        #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.5 
-
-        #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + pose_smoothness_loss * 0.0005 + joints_smoothness_loss * 0.5
-
-        #     nearest_dist_shape, _ = torch.min(nearest_dist, dim=-1)
-        #     nearest_dist_shape_mask = nearest_dist_shape > 0.01
-        #     transl_smoothness_loss_v2 = torch.sum((transl_var_ori - transl_var) ** 2, dim=-1)
-        #     transl_smoothness_loss_v2 = torch.mean(transl_smoothness_loss_v2[nearest_dist_shape_mask])
-        #     loss = transl_smoothness_loss * 0.5 + attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.5
-
-        #   else:          
-        #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05
-        #     loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
+  #       # if cat_nm in ["Scissors"]:
+  #       #   # scissors and 
+  #       #   # if dist_thres < 0.05:
+  #       #   #   loss = transl_smoothness_loss * 0.5 + attaction_loss * 10000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5
+  #       #   # else:          
+  #       #   #   # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05
+  #       #   #   loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
+  #       #   #   # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
+
+  #       #   if dist_thres < 0.05:
+  #       #     # loss = transl_smoothness_loss * 0.5 + attaction_loss * 10000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5
+  #       #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.5 
+
+  #       #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + pose_smoothness_loss * 0.0005 + joints_smoothness_loss * 0.5
+
+  #       #     nearest_dist_shape, _ = torch.min(nearest_dist, dim=-1)
+  #       #     nearest_dist_shape_mask = nearest_dist_shape > 0.01
+  #       #     transl_smoothness_loss_v2 = torch.sum((transl_var_ori - transl_var) ** 2, dim=-1)
+  #       #     transl_smoothness_loss_v2 = torch.mean(transl_smoothness_loss_v2[nearest_dist_shape_mask])
+  #       #     loss = transl_smoothness_loss * 0.5 + attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.5
+
+  #       #   else:          
+  #       #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05
+  #       #     loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
             
-        #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
+  #       #     # loss = attaction_loss * 10000. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.005
            
-        # loss = joints_pred_loss * 20 + joints_smoothness_loss * 200. + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001
-        # loss = joints_pred_loss * 30  # + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001
+  #       # loss = joints_pred_loss * 20 + joints_smoothness_loss * 200. + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001
+  #       # loss = joints_pred_loss * 30  # + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001
         
-        # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5 + attaction_loss * 0.001 + joints_smoothness_loss * 1.0
+  #       # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5 + attaction_loss * 0.001 + joints_smoothness_loss * 1.0
         
-        # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5  + attaction_loss * 2000000.  # + joints_smoothness_loss * 10.0
+  #       # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5  + attaction_loss * 2000000.  # + joints_smoothness_loss * 10.0
         
-        # loss = joints_pred_loss * 20 #  + pose_smoothness_loss * 0.5 + attaction_loss * 100. + joints_smoothness_loss * 10.0
-        # loss = joints_pred_loss * 30 + attaction_loss * 0.001
+  #       # loss = joints_pred_loss * 20 #  + pose_smoothness_loss * 0.5 + attaction_loss * 100. + joints_smoothness_loss * 10.0
+  #       # loss = joints_pred_loss * 30 + attaction_loss * 0.001
         
-        opt.zero_grad()
-        loss.backward()
-        opt.step()
-        scheduler.step()
+  #       opt.zero_grad()
+  #       loss.backward()
+  #       opt.step()
+  #       scheduler.step()
         
-        print('Iter {}: {}'.format(i, loss.item()), flush=True)
-        print('\tShape Prior Loss: {}'.format(shape_prior_loss.item()))
-        print('\tPose Prior Loss: {}'.format(pose_prior_loss.item()))
-        print('\tPose Smoothness Loss: {}'.format(pose_smoothness_loss.item()))
-        print('\tJoints Prediction Loss: {}'.format(joints_pred_loss.item()))
-        print('\tAttraction Loss: {}'.format(attaction_loss.item()))
-        print('\tJoint Smoothness Loss: {}'.format(joints_smoothness_loss.item()))
-        # theta_smoothness_loss
-        print('\tTheta Smoothness Loss: {}'.format(theta_smoothness_loss.item()))
-        # transl_smoothness_loss
-        print('\tTransl Smoothness Loss: {}'.format(transl_smoothness_loss.item()))
+  #       print('Iter {}: {}'.format(i, loss.item()), flush=True)
+  #       print('\tShape Prior Loss: {}'.format(shape_prior_loss.item()))
+  #       print('\tPose Prior Loss: {}'.format(pose_prior_loss.item()))
+  #       print('\tPose Smoothness Loss: {}'.format(pose_smoothness_loss.item()))
+  #       print('\tJoints Prediction Loss: {}'.format(joints_pred_loss.item()))
+  #       print('\tAttraction Loss: {}'.format(attaction_loss.item()))
+  #       print('\tJoint Smoothness Loss: {}'.format(joints_smoothness_loss.item()))
+  #       # theta_smoothness_loss
+  #       print('\tTheta Smoothness Loss: {}'.format(theta_smoothness_loss.item()))
+  #       # transl_smoothness_loss
+  #       print('\tTransl Smoothness Loss: {}'.format(transl_smoothness_loss.item()))
   
-  rhand_anchors_np = rhand_anchors.detach().cpu().numpy()
-  np.save("out_anchors.npy", rhand_anchors_np)
+  # rhand_anchors_np = rhand_anchors.detach().cpu().numpy()
+  # np.save("out_anchors.npy", rhand_anchors_np)
   ### optimized dict before projection ###
   bf_proj_optimized_dict = {
     'bf_ctx_mask_verts': hand_verts.detach().cpu().numpy(),
@@ -659,48 +662,48 @@ def get_optimized_hand_fr_joints_v4_anchors(joints, base_pts, tot_base_pts_trans
   
   
   
-  ### 
-  if with_ctx_mask:
-    tot_penetration_masks_bf_contact_opt_frame_nmask_np = tot_penetration_masks_bf_contact_opt_frame_nmask.detach().cpu().numpy()
-    hand_verts = hand_verts.detach()
-    hand_joints = hand_joints.detach()
-    rot_var = rot_var.detach()
-    theta_var = theta_var.detach()
-    beta_var = beta_var.detach()
-    transl_var = transl_var.detach()
-    hand_verts = hand_verts.detach()
-    
-    # tot_base_pts_trans_disp_mask ### total penetration masks  
-    tot_base_pts_trans_disp_mask_n = (1. - tot_base_pts_trans_disp_mask.float()) > 0.5 ### that object do not move
-    tot_penetration_masks_bf_contact_opt_frame_nmask = (tot_penetration_masks_bf_contact_opt_frame_nmask.float() + tot_base_pts_trans_disp_mask_n.float()) > 1.5
-    hand_verts[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_verts_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
-    hand_joints[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_joints_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
-    rot_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_rot_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
-    theta_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_theta_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
-    # beta_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_beta_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
-    transl_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_transl_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
-    
-    rot_var_tmp = torch.randn_like(rot_var)
-    theta_var_tmp = torch.randn_like(theta_var)
-    transl_var_tmp = torch.randn_like(transl_var)
-    
-    rot_var_tmp.data = rot_var.data.clone()
-    theta_var_tmp.data = theta_var.data.clone()
-    transl_var_tmp.data = transl_var.data.clone()
-    
-    rot_var = torch.randn_like(rot_var)
-    theta_var = torch.randn_like(theta_var)
-    transl_var = torch.randn_like(transl_var)
-    
-    rot_var.data = rot_var_tmp.data.clone()
-    theta_var.data = theta_var_tmp.data.clone()
-    transl_var.data = transl_var_tmp.data.clone()
-    
-    
-    rot_var = rot_var.requires_grad_()
-    theta_var = theta_var.requires_grad_()
-    beta_var = beta_var.requires_grad_()
-    transl_var = transl_var.requires_grad_()
+  # ### 
+  # if with_ctx_mask:
+  #   tot_penetration_masks_bf_contact_opt_frame_nmask_np = tot_penetration_masks_bf_contact_opt_frame_nmask.detach().cpu().numpy()
+  #   hand_verts = hand_verts.detach()
+  #   hand_joints = hand_joints.detach()
+  #   rot_var = rot_var.detach()
+  #   theta_var = theta_var.detach()
+  #   beta_var = beta_var.detach()
+  #   transl_var = transl_var.detach()
+  #   hand_verts = hand_verts.detach()
+    
+  #   # tot_base_pts_trans_disp_mask ### total penetration masks  
+  #   tot_base_pts_trans_disp_mask_n = (1. - tot_base_pts_trans_disp_mask.float()) > 0.5 ### that object do not move
+  #   tot_penetration_masks_bf_contact_opt_frame_nmask = (tot_penetration_masks_bf_contact_opt_frame_nmask.float() + tot_base_pts_trans_disp_mask_n.float()) > 1.5
+  #   hand_verts[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_verts_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
+  #   hand_joints[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_joints_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
+  #   rot_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_rot_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
+  #   theta_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_theta_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
+  #   # beta_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_beta_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
+  #   transl_var[tot_penetration_masks_bf_contact_opt_frame_nmask] = bf_ct_transl_var_th[tot_penetration_masks_bf_contact_opt_frame_nmask]
+    
+  #   rot_var_tmp = torch.randn_like(rot_var)
+  #   theta_var_tmp = torch.randn_like(theta_var)
+  #   transl_var_tmp = torch.randn_like(transl_var)
+    
+  #   rot_var_tmp.data = rot_var.data.clone()
+  #   theta_var_tmp.data = theta_var.data.clone()
+  #   transl_var_tmp.data = transl_var.data.clone()
+    
+  #   rot_var = torch.randn_like(rot_var)
+  #   theta_var = torch.randn_like(theta_var)
+  #   transl_var = torch.randn_like(transl_var)
+    
+  #   rot_var.data = rot_var_tmp.data.clone()
+  #   theta_var.data = theta_var_tmp.data.clone()
+  #   transl_var.data = transl_var_tmp.data.clone()
+    
+    
+  #   rot_var = rot_var.requires_grad_()
+  #   theta_var = theta_var.requires_grad_()
+  #   beta_var = beta_var.requires_grad_()
+  #   transl_var = transl_var.requires_grad_()
     
     
   
@@ -722,142 +725,142 @@ def get_optimized_hand_fr_joints_v4_anchors(joints, base_pts, tot_base_pts_trans
     'bf_proj_transl_var': bf_proj_transl_var,
   } )
 
-  if with_proj:
-    num_iters = 2000
-    num_iters = 1000 # seq 77 # if with contact opt #
-    # num_iters = 500 # seq 77
-    ori_theta_var = theta_var.detach().clone()
-    
-    nearest_base_pts=None
-    nearest_base_normals=None
-    
-    # obj_verts_trans, obj_faces
-    if cat_nm in ["Mug"]:
-      # tot_penetration_masks = None
-      tot_penetration_masks = get_penetration_masks(obj_verts_trans, obj_faces, hand_verts)
-    else:
-      tot_penetration_masks = get_penetration_masks(obj_verts_trans, obj_faces, hand_verts)
-    # tot_penetration_masks = None
-    
-    # opt = optim.Adam([rot_var, transl_var, theta_var], lr=learning_rate)
-    # opt = optim.Adam([transl_var, theta_var], lr=learning_rate)
-    opt = optim.Adam([transl_var, theta_var, rot_var], lr=learning_rate)
-    scheduler = optim.lr_scheduler.StepLR(opt, step_size=num_iters, gamma=0.5)
-    for i in range(num_iters):
-        opt.zero_grad()
-        # mano_layer
-        hand_verts, hand_joints = mano_layer(torch.cat([rot_var, theta_var], dim=-1),
-            beta_var.unsqueeze(1).repeat(1, nn_frames, 1).view(-1, 10), transl_var)
-        hand_verts = hand_verts.view( nn_frames, 778, 3) * 0.001
-        hand_joints = hand_joints.view(nn_frames, -1, 3) * 0.001
+  # if with_proj:
+  #   num_iters = 2000
+  #   num_iters = 1000 # seq 77 # if with contact opt #
+  #   # num_iters = 500 # seq 77
+  #   ori_theta_var = theta_var.detach().clone()
+    
+  #   nearest_base_pts=None
+  #   nearest_base_normals=None
+    
+  #   # obj_verts_trans, obj_faces
+  #   if cat_nm in ["Mug"]:
+  #     # tot_penetration_masks = None
+  #     tot_penetration_masks = get_penetration_masks(obj_verts_trans, obj_faces, hand_verts)
+  #   else:
+  #     tot_penetration_masks = get_penetration_masks(obj_verts_trans, obj_faces, hand_verts)
+  #   # tot_penetration_masks = None
+    
+  #   # opt = optim.Adam([rot_var, transl_var, theta_var], lr=learning_rate)
+  #   # opt = optim.Adam([transl_var, theta_var], lr=learning_rate)
+  #   opt = optim.Adam([transl_var, theta_var, rot_var], lr=learning_rate)
+  #   scheduler = optim.lr_scheduler.StepLR(opt, step_size=num_iters, gamma=0.5)
+  #   for i in range(num_iters):
+  #       opt.zero_grad()
+  #       # mano_layer
+  #       hand_verts, hand_joints = mano_layer(torch.cat([rot_var, theta_var], dim=-1),
+  #           beta_var.unsqueeze(1).repeat(1, nn_frames, 1).view(-1, 10), transl_var)
+  #       hand_verts = hand_verts.view( nn_frames, 778, 3) * 0.001
+  #       hand_joints = hand_joints.view(nn_frames, -1, 3) * 0.001
         
-        # tot_base_pts_trans, tot_base_normals_trans #
-        # obj_verts_trans, ### nearest base pts ####
-        proj_loss, nearest_base_pts, nearest_base_normals  = get_proj_losses(hand_verts, tot_base_pts_trans, tot_base_normals_trans, tot_penetration_masks, nearest_base_pts=nearest_base_pts, nearest_base_normals=nearest_base_normals)
+  #       # tot_base_pts_trans, tot_base_normals_trans #
+  #       # obj_verts_trans, ### nearest base pts ####
+  #       proj_loss, nearest_base_pts, nearest_base_normals  = get_proj_losses(hand_verts, tot_base_pts_trans, tot_base_normals_trans, tot_penetration_masks, nearest_base_pts=nearest_base_pts, nearest_base_normals=nearest_base_normals)
         
-        # rhand_anchors = recover_anchor_batch(hand_verts, face_vertex_index, anchor_weight.unsqueeze(0).repeat(window_size, 1, 1))
+  #       # rhand_anchors = recover_anchor_batch(hand_verts, face_vertex_index, anchor_weight.unsqueeze(0).repeat(window_size, 1, 1))
 
 
-        # hand_joints = rhand_anchors
+  #       # hand_joints = rhand_anchors
         
-        joints_pred_loss = torch.sum(
-          (hand_joints - joints) ** 2, dim=-1
-        ).mean()
+  #       joints_pred_loss = torch.sum(
+  #         (hand_joints - joints) ** 2, dim=-1
+  #       ).mean()
         
         
         
         
-        # # dist_joints_to_base_pts_sqr = torch.sum(
-        # #     (hand_joints.unsqueeze(2) - base_pts.unsqueeze(0).unsqueeze(1)) ** 2, dim=-1
-        # # ) # nf x nnb x 3 ---- nf x nnj x 1 x 3 
-        # dist_joints_to_base_pts_sqr = torch.sum(
-        #     (rhand_anchors.unsqueeze(2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1
-        # )
-        # # attaction_loss = 0.5 * affinity_scores * dist_joints_to_base_pts_sqr
-        # attaction_loss = 0.5 * dist_joints_to_base_pts_sqr
-        # # attaction_loss = attaction_loss
-        # # attaction_loss = torch.mean(attaction_loss[..., -5:, :] * minn_dist_mask[..., -5:, :])
+  #       # # dist_joints_to_base_pts_sqr = torch.sum(
+  #       # #     (hand_joints.unsqueeze(2) - base_pts.unsqueeze(0).unsqueeze(1)) ** 2, dim=-1
+  #       # # ) # nf x nnb x 3 ---- nf x nnj x 1 x 3 
+  #       # dist_joints_to_base_pts_sqr = torch.sum(
+  #       #     (rhand_anchors.unsqueeze(2) - tot_base_pts_trans.unsqueeze(1)) ** 2, dim=-1
+  #       # )
+  #       # # attaction_loss = 0.5 * affinity_scores * dist_joints_to_base_pts_sqr
+  #       # attaction_loss = 0.5 * dist_joints_to_base_pts_sqr
+  #       # # attaction_loss = attaction_loss
+  #       # # attaction_loss = torch.mean(attaction_loss[..., -5:, :] * minn_dist_mask[..., -5:, :])
         
-        # # attaction_loss = torch.mean(attaction_loss * attraction_mask)
-        # # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # # attaction_loss = torch.mean(attaction_loss * attraction_mask)
+  #       # # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
-        # # seq 80
-        # # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # # seq 80
+  #       # # attaction_loss = torch.mean(attaction_loss[46:, -5:-3, :] * minn_dist_mask[46:, -5:-3, :]) + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
-        # # seq 70
-        # # attaction_loss = torch.mean(attaction_loss[10:, -5:-3, :] * minn_dist_mask[10:, -5:-3, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # # seq 70
+  #       # # attaction_loss = torch.mean(attaction_loss[10:, -5:-3, :] * minn_dist_mask[10:, -5:-3, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
-        # # new version relying on new mask #
-        # # attaction_loss = torch.mean(attaction_loss[:, -5:-3, :] * attraction_mask_new[:, -5:-3, :])
-        # ### original version ###
-        # # attaction_loss = torch.mean(attaction_loss[20:, -3:, :] * attraction_mask_new[20:, -3:, :])
+  #       # # new version relying on new mask #
+  #       # # attaction_loss = torch.mean(attaction_loss[:, -5:-3, :] * attraction_mask_new[:, -5:-3, :])
+  #       # ### original version ###
+  #       # # attaction_loss = torch.mean(attaction_loss[20:, -3:, :] * attraction_mask_new[20:, -3:, :])
         
-        # # attaction_loss = torch.mean(attaction_loss[:, -5:, :] * attraction_mask_new_new[:, -5:, :] * joint_attraction_k[:, -5:].unsqueeze(-1))
+  #       # # attaction_loss = torch.mean(attaction_loss[:, -5:, :] * attraction_mask_new_new[:, -5:, :] * joint_attraction_k[:, -5:].unsqueeze(-1))
         
         
         
-        # attaction_loss = torch.mean(attaction_loss[:, :, :] * attraction_mask_new_new[:, :, :] * joint_attraction_k[:, :].unsqueeze(-1))
+  #       # attaction_loss = torch.mean(attaction_loss[:, :, :] * attraction_mask_new_new[:, :, :] * joint_attraction_k[:, :].unsqueeze(-1))
         
         
-        # seq mug
-        # attaction_loss = torch.mean(attaction_loss[4:, -5:-4, :] * minn_dist_mask[4:, -5:-4, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
+  #       # seq mug
+  #       # attaction_loss = torch.mean(attaction_loss[4:, -5:-4, :] * minn_dist_mask[4:, -5:-4, :]) # + torch.mean(attaction_loss[:40, -5:-3, :] * minn_dist_mask[:40, -5:-3, :])
         
         
-        # opt.zero_grad()
-        pose_smoothness_loss = F.mse_loss(theta_var.view(nn_frames, -1)[1:], theta_var.view(nn_frames, -1)[:-1])
-        # joints_smoothness_loss = joint_acc_loss(hand_verts, J_regressor.to(device))
-        shape_prior_loss = torch.mean(beta_var**2)
-        pose_prior_loss = torch.mean(theta_var**2)
-        joints_smoothness_loss = F.mse_loss(hand_joints.view(nn_frames, -1, 3)[1:], hand_joints.view(nn_frames, -1, 3)[:-1])
-        # =0.05
-        # # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001 + joints_smoothness_loss * 100.
-        # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.000001 + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001 + joints_smoothness_loss * 200.
+  #       # opt.zero_grad()
+  #       pose_smoothness_loss = F.mse_loss(theta_var.view(nn_frames, -1)[1:], theta_var.view(nn_frames, -1)[:-1])
+  #       # joints_smoothness_loss = joint_acc_loss(hand_verts, J_regressor.to(device))
+  #       shape_prior_loss = torch.mean(beta_var**2)
+  #       pose_prior_loss = torch.mean(theta_var**2)
+  #       joints_smoothness_loss = F.mse_loss(hand_joints.view(nn_frames, -1, 3)[1:], hand_joints.view(nn_frames, -1, 3)[:-1])
+  #       # =0.05
+  #       # # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001 + joints_smoothness_loss * 100.
+  #       # loss = joints_pred_loss * 30 + pose_smoothness_loss * 0.000001 + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001 + joints_smoothness_loss * 200.
         
-        # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + joints_smoothness_loss * 200.
+  #       # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.05 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + joints_smoothness_loss * 200.
         
-        # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.03 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + attaction_loss * 10000 # + joints_smoothness_loss * 200.
+  #       # loss = joints_pred_loss * 5000 + pose_smoothness_loss * 0.03 + shape_prior_loss * 0.0002 + pose_prior_loss * 0.0005 # + attaction_loss * 10000 # + joints_smoothness_loss * 200.
         
-        theta_smoothness_loss = F.mse_loss(theta_var, ori_theta_var)
-        # loss = attaction_loss * 1000. + theta_smoothness_loss * 0.00001
+  #       theta_smoothness_loss = F.mse_loss(theta_var, ori_theta_var)
+  #       # loss = attaction_loss * 1000. + theta_smoothness_loss * 0.00001
         
-        # attraction loss, joint prediction loss, joints smoothness loss #
-        # loss = attaction_loss * 1000. + joints_pred_loss 
-        ### general ###
-        # loss = attaction_loss * 1000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
+  #       # attraction loss, joint prediction loss, joints smoothness loss #
+  #       # loss = attaction_loss * 1000. + joints_pred_loss 
+  #       ### general ###
+  #       # loss = attaction_loss * 1000. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
         
-        # tune for seq 140
-        # loss = proj_loss * 1. + joints_pred_loss * 0.05 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
-        loss = proj_loss * 1. + joints_pred_loss * 1.0 + joints_smoothness_loss * 0.5 
-        if cat_nm in ["Pliers"]:
-          #  loss = proj_loss * 1. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05 
-          loss = proj_loss * 1. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05 
-
-        elif cat_nm in ["Bottle"]:
-          loss = proj_loss * 1. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.05 
-        # loss = joints_pred_loss * 20 + joints_smoothness_loss * 200. + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001
-        # loss = joints_pred_loss * 30  # + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001
+  #       # tune for seq 140
+  #       # loss = proj_loss * 1. + joints_pred_loss * 0.05 + joints_smoothness_loss * 0.5 # + pose_prior_loss * 0.00005  # + shape_prior_loss * 0.001 # + pose_smoothness_loss * 0.5
+  #       loss = proj_loss * 1. + joints_pred_loss * 1.0 + joints_smoothness_loss * 0.5 
+  #       if cat_nm in ["Pliers"]:
+  #         #  loss = proj_loss * 1. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05 
+  #         loss = proj_loss * 1. + joints_pred_loss * 0.0001 + joints_smoothness_loss * 0.05 
+
+  #       elif cat_nm in ["Bottle"]:
+  #         loss = proj_loss * 1. + joints_pred_loss * 0.01 + joints_smoothness_loss * 0.05 
+  #       # loss = joints_pred_loss * 20 + joints_smoothness_loss * 200. + shape_prior_loss * 0.0001 + pose_prior_loss * 0.00001
+  #       # loss = joints_pred_loss * 30  # + shape_prior_loss * 0.001 + pose_prior_loss * 0.0001
         
-        # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5 + attaction_loss * 0.001 + joints_smoothness_loss * 1.0
+  #       # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5 + attaction_loss * 0.001 + joints_smoothness_loss * 1.0
         
-        # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5  + attaction_loss * 2000000.  # + joints_smoothness_loss * 10.0
+  #       # loss = joints_pred_loss * 20 + pose_smoothness_loss * 0.5  + attaction_loss * 2000000.  # + joints_smoothness_loss * 10.0
         
-        # loss = joints_pred_loss * 20 #  + pose_smoothness_loss * 0.5 + attaction_loss * 100. + joints_smoothness_loss * 10.0
-        # loss = joints_pred_loss * 30 + attaction_loss * 0.001
+  #       # loss = joints_pred_loss * 20 #  + pose_smoothness_loss * 0.5 + attaction_loss * 100. + joints_smoothness_loss * 10.0
+  #       # loss = joints_pred_loss * 30 + attaction_loss * 0.001
         
-        opt.zero_grad()
-        loss.backward()
-        opt.step()
-        scheduler.step()
+  #       opt.zero_grad()
+  #       loss.backward()
+  #       opt.step()
+  #       scheduler.step()
         
-        print('Iter {}: {}'.format(i, loss.item()), flush=True)
-        print('\tShape Prior Loss: {}'.format(shape_prior_loss.item()))
-        print('\tPose Prior Loss: {}'.format(pose_prior_loss.item()))
-        print('\tPose Smoothness Loss: {}'.format(pose_smoothness_loss.item()))
-        print('\tJoints Prediction Loss: {}'.format(joints_pred_loss.item()))
-        print('\tproj_loss Loss: {}'.format(proj_loss.item()))
-        print('\tJoint Smoothness Loss: {}'.format(joints_smoothness_loss.item()))
-        # theta_smoothness_loss
-        print('\tTheta Smoothness Loss: {}'.format(theta_smoothness_loss.item()))
+  #       print('Iter {}: {}'.format(i, loss.item()), flush=True)
+  #       print('\tShape Prior Loss: {}'.format(shape_prior_loss.item()))
+  #       print('\tPose Prior Loss: {}'.format(pose_prior_loss.item()))
+  #       print('\tPose Smoothness Loss: {}'.format(pose_smoothness_loss.item()))
+  #       print('\tJoints Prediction Loss: {}'.format(joints_pred_loss.item()))
+  #       print('\tproj_loss Loss: {}'.format(proj_loss.item()))
+  #       print('\tJoint Smoothness Loss: {}'.format(joints_smoothness_loss.item()))
+  #       # theta_smoothness_loss
+  #       print('\tTheta Smoothness Loss: {}'.format(theta_smoothness_loss.item()))
   
   ### ### verts and joints before contact opt ### ###
   # bf_ct_verts, bf_ct_joints #
@@ -1091,12 +1094,17 @@ def get_proj_losses(hand_verts, base_pts, base_normals, tot_penetration_masks, n
 
 
 ## optimization ##
-if __name__=='__main__':
+# if __name__=='__main__':
+def reconstruct_from_file(single_seq_path):
+  # args = train_args()
   
-  args = train_args()
   
-
-  single_seq_path = args.single_seq_path
+  # use_left = True
+  test_tag = "20231104_017_jts_spatial_t_100_"
+  save_dir = "/tmp/denoising/save"
+  # single_seq_path = 
+  
+  single_seq_path = single_seq_path
   
   print(f"Reconstructing meshes for sequence: {single_seq_path}")
   
@@ -1112,20 +1120,27 @@ if __name__=='__main__':
   st_idxes = list(range(0, num_ending_clearning_frames, nn_st_skip))
   if st_idxes[-1] + num_cleaning_frames < nn_frames:
       st_idxes.append(nn_frames - num_cleaning_frames)
+      
+  st_idxes = [st_idxes[0]]
   print(f"st_idxes: {st_idxes}")
   
+  
+  
+  
   clip_ending_idxes = [st_idxes[cur_ts + 1] - st_idxes[cur_ts] for cur_ts in range(len(st_idxes) - 1)]
 
 
-  test_tags = [f"{args.test_tag}_st_{cur_st}" for cur_st in st_idxes]
+  test_tags = [f"{test_tag}_st_{cur_st}" for cur_st in st_idxes]
 
 
-  pred_infos_sv_folder = args.save_dir
-  new_pred_infos_sv_folder = args.save_dir
+  pred_infos_sv_folder = save_dir
+  new_pred_infos_sv_folder = save_dir
 
 
 
-  tot_rnd_seeds = range(0, 121, 11)
+  # tot_rnd_seeds = range(0, 121, 11)
+  
+  tot_rnd_seeds = [0]
 
   # st_idxx = 0
 
@@ -1227,8 +1242,8 @@ if __name__=='__main__':
     
     #### obj_verts_trans, obj_faces ####
     obj_verts_trans = np.matmul(obj_verts, tot_obj_rot) + tot_obj_transl.reshape(tot_obj_transl.shape[0], 1, tot_obj_transl.shape[1]) 
-    obj_faces = data['template_obj_fs']
-    print(f"obj_verts_trans: {obj_verts_trans.shape}, obj_faces: {obj_faces.shape}")
+    obj_faces = None
+    # print(f"obj_verts_trans: {obj_verts_trans.shape}, obj_faces: {obj_faces.shape}")
     
 
     
@@ -1267,10 +1282,12 @@ if __name__=='__main__':
     )
     
     
+    optimized_sv_infos.update({'predicted_info': data})
+    
     optimized_sv_infos_sv_fn_nm = f"optimized_infos_sv_dict_seed_{seed}_tag_{test_tags[0]}_ntag_{len(test_tags)}.npy"
     
     optimized_sv_infos_sv_fn = os.path.join(new_pred_infos_sv_folder, optimized_sv_infos_sv_fn_nm)
     np.save(optimized_sv_infos_sv_fn, optimized_sv_infos)
     print(f"optimized infos saved to {optimized_sv_infos_sv_fn}")
   
-
+  return optimized_sv_infos_sv_fn

scripts/val_gradio/predict_taco_rndseed_spatial.sh

@@ -94,7 +94,7 @@ export diff_hand_params=""
 export diff_basejtsrel=""
 export diff_realbasejtsrel="--diff_realbasejtsrel"
 # export model_path=./ckpts/model000519000.pt 
-export model_path=/home/xueyi/sim/Generalizable-HOI-Denoising/ckpt/model001039000.pt
+export model_path=./model001039000.pt
 
 
 
@@ -120,7 +120,7 @@ export use_reverse=""
 export seed=0
 
 
-export save_dir=./data/taco/result
+export save_dir=/tmp/denoising
 
 
 export cuda_ids=2

scripts/val_gradio/predict_taco_rndseed_spatial_temp.sh

@@ -94,7 +94,7 @@ export diff_hand_params=""
 export diff_basejtsrel=""
 export diff_realbasejtsrel="--diff_realbasejtsrel"
 # export model_path=./ckpts/model000519000.pt 
-export model_path=/home/xueyi/sim/Generalizable-HOI-Denoising/ckpt/model001039000.pt
+export model_path=./model001039000.pt
 
 
 
@@ -120,7 +120,7 @@ export use_reverse=""
 export seed=0
 
 
-export save_dir=./data/taco/result
+export save_dir=/tmp/denoising
 
 
 export cuda_ids=2

test_predict_from_file.py

@@ -12,6 +12,8 @@ import shutil
 # from gradio_inter.predict_from_file import predict_from_file
 from gradio_inter.create_bash_file import create_bash_file
 
+from sample.reconstruct_data_taco import reconstruct_from_file
+
 def create_temp_file(path: str) -> str:
     temp_dir = tempfile.gettempdir()
     temp_folder = os.path.join(temp_dir, "denoising")
@@ -36,11 +38,15 @@ def predict(file_path: str):
     temp_file_path = create_temp_file(file_path)
     # predict_from_file
     print(f"temp_path: {temp_file_path}")
+    
     temp_bash_file = create_bash_file(temp_file_path)
     print(f"temp_bash_file: {temp_bash_file}")
-    os.system(f"bash {temp_bash_file}")
     
-
+    # os.system(f"bash {temp_bash_file}")
+    
+    saved_path = reconstruct_from_file(temp_file_path)
+    print(saved_path)
+    
 # demo = gr.Interface(
 #     predict,
 #     # gr.Dataframe(type="numpy", datatype="number", row_count=5, col_count=3),