//  [python]
def write_newly_entities_into_annotations_per_clip(self, first_frame_idx, out, targets, interim_size):
        """
        Write annotated masks for these objects that appear in the first frame into the annotation Dict
        Args:
            out: A dict to store the masks, boxes of newly entites
            targets: A list with [Dict, Dict, ..], which stores the annotated masks of target-objects
            first_frame_idx: the indix of the first frame in the current processing clip
        """
        pred_logits = out['pred_logits'].unsqueeze(1) # Q_newlyx1xK
        pred_masks = out['pred_masks']   # Q_newlyxTxHxW
        pred_embds = out['pred_embds']   # Q_newlyxTxC
        pred_embds = torch.mean(pred_embds, dim=1, keepdim=True)
        pred_boxes = out['pred_boxes']   # Q_newlyxTx4
        mask_quality_scores = out['mask_quality_scores']   # Q_newlyxTx4
        _num_instance_newly = pred_masks.shape[0]

        num_frames = pred_masks.shape[1]
        
        first_appear_frame_idxs_newly = torch.ones(_num_instance_newly, dtype=torch.long, device=self.device) * first_frame_idx
        if _num_instance_newly == 0:
            pred_masks = torch.zeros((0,self.num_frames, interim_size[0], interim_size[1]), device=pred_masks.device)
        else:
            pred_masks = F.interpolate(
                pred_masks, interim_size, mode='bilinear', align_corners=False
            ) 
        pred_occurrence = torch.ones([pred_masks.shape[0], pred_masks.shape[1]], device=pred_masks.device)

        assert len(targets) == 1, "Only support the batch size is 1"
        targets_per_video = targets[0]
        if 'masks' not in targets_per_video:
            pred_ids = torch.arange(_num_instance_newly, device=self.device)
            targets_per_video.update({
                "logits": pred_logits, 
                "masks": pred_masks.gt(0.),  
                "mask_logits": pred_masks,  
                "boxes": pred_boxes,  
                "embds": pred_embds, 
                "ids":   pred_ids,
                "first_appear_frame_idxs": first_appear_frame_idxs_newly,
                "mask_quality_scores": mask_quality_scores,
                "occurrence": pred_occurrence,
            })
        
        else:
            if _num_instance_newly == 0:
                return 

            gt_logits = targets_per_video["logits"]  # N, num_frames_prev-num_frames+1, K
            gt_masks = targets_per_video['masks']    # N, num_frames+1, H, W
            gt_mask_logits = targets_per_video['mask_logits']  # N, num_frames+1, H, W
            gt_boxes = targets_per_video['boxes']    # N, num_frames_prev, 4
            gt_embds = targets_per_video['embds']    # N, num_frames_prev, C
            gt_ids   = targets_per_video["ids"]      # N
            gt_mask_quality_scores = targets_per_video["mask_quality_scores"]
            gt_occurrence = targets_per_video["occurrence"]
            T_prev = gt_boxes.shape[1]

            # zero-vector padding to keep consistency with annotations
            gt_logits_pad = torch.zeros([_num_instance_newly, gt_logits.shape[1]-pred_logits.shape[1], gt_logits.shape[-1]], dtype=torch.float, device=self.device)
            mask_shape = [_num_instance_newly, gt_masks.shape[1]-num_frames, interim_size[0], interim_size[1]]
            gt_masks_pad = torch.zeros(mask_shape, dtype=torch.float, device=self.device)
            gt_boxes_pad = torch.zeros([_num_instance_newly, gt_boxes.shape[1]-num_frames, 4], dtype=torch.float32, device=self.device)
            gt_embds_pad = torch.zeros([_num_instance_newly, gt_embds.shape[1]-pred_embds.shape[1], self.hidden_dim], dtype=torch.float32, device=self.device)
            gt_occurrence_pad = torch.zeros([_num_instance_newly, gt_occurrence.shape[1]-pred_occurrence.shape[1]], device=self.device)
            
            gt_logits_newly = torch.cat([gt_logits_pad, pred_logits], dim=1)
            gt_masks_newly = torch.cat([gt_masks_pad, pred_masks], dim=1)
            gt_boxes_newly = torch.cat([gt_boxes_pad, pred_boxes], dim=1)
            gt_embds_newly = torch.cat([gt_embds_pad, pred_embds], dim=1)
            gt_ids_newly = torch.arange(_num_instance_newly, device=self.device) + len(gt_ids)
            gt_occurrence_newly = torch.cat([gt_occurrence_pad, pred_occurrence], dim=1)
            
            gt_logits = torch.cat([gt_logits, gt_logits_newly]) # N+N_newly, num_frames_prev-num_frames+1, 4
            gt_masks = torch.cat([gt_masks, gt_masks_newly.gt(0.)])      # N+N_newly, num_frames+1, H, W
            print(first_frame_idx, gt_mask_logits.shape, gt_masks_newly.shape, self.apply_cls_thres)
            gt_mask_logits = torch.cat([gt_mask_logits, gt_masks_newly]) # N+N_newly, num_frames+1, H, W
            gt_boxes = torch.cat([gt_boxes, gt_boxes_newly])    # N+N_newly, num_frames_prev, 4
            gt_embds = torch.cat([gt_embds, gt_embds_newly])    # N+N_newly, num_frames_prev, C
            gt_ids = torch.cat([gt_ids, gt_ids_newly])          # N+N_newly, 
            gt_occurrence = torch.cat([gt_occurrence, gt_occurrence_newly])
            first_appear_frame_idxs = torch.cat([
                targets_per_video["first_appear_frame_idxs"], first_appear_frame_idxs_newly
            ])
            
            gt_mask_quality_scores = torch.cat([gt_mask_quality_scores, mask_quality_scores])
            targets_per_video.update({
                "logits": gt_logits, 
                "masks": gt_masks,  
                "mask_logits": gt_mask_logits,
                "boxes": gt_boxes,  
                "embds": gt_embds, 
                "ids":   gt_ids,
                "first_appear_frame_idxs": first_appear_frame_idxs,
                "mask_quality_scores": gt_mask_quality_scores,
                "occurrence": gt_occurrence,
            })
            
            if "prompt_pe" in targets_per_video:
                # pad the corresponding prompt informations for parallel
                prompt_pe = targets_per_video["prompt_pe"]
                prompt_feats = targets_per_video["prompt_feats"]
                prompt_attn_masks = targets_per_video["prompt_attn_masks"]
                prompt_pe_pad = torch.zeros([_num_instance_newly, *prompt_pe.shape[1:]], device=self.device)
                prompt_feats_pad = torch.zeros([_num_instance_newly, *prompt_feats.shape[1:]], device=self.device)
                prompt_attn_masks_pad = torch.zeros(
                    [prompt_attn_masks.shape[0], prompt_attn_masks.shape[1], _num_instance_newly, prompt_attn_masks.shape[-1]], device=self.device
                ).bool()
                targets_per_video["prompt_pe"] = torch.cat([prompt_pe, prompt_pe_pad])
                targets_per_video["prompt_feats"] = torch.cat([prompt_feats, prompt_feats_pad])
                targets_per_video["prompt_attn_masks"] = torch.cat([prompt_attn_masks, prompt_attn_masks_pad], dim=-2)