| """ | |
| Implement the CAL + CAL (TEF) model mentioned in | |
| ``` | |
| @article{Escorcia2019TemporalLO, | |
| title={Temporal Localization of Moments in Video Collections with Natural Language}, | |
| author={Victor Escorcia and Mattia Soldan and Josef Sivic and Bernard Ghanem and Bryan Russell}, | |
| journal={ArXiv}, | |
| year={2019}, | |
| volume={abs/1907.12763} | |
| } | |
| ``` | |
| Methods: | |
| 1, Give top200 predictions for each query in CAL then using CAL (TEF) to re-rank. | |
| 2, This is approximated by re-ranking the top200 CAL using top1000 CAL(TEF) -- we assume they will be all covered. | |
| """ | |
| import torch | |
| import subprocess | |
| import numpy as np | |
| from tqdm import tqdm | |
| from utils.basic_utils import load_json, save_json | |
| def load_saved_res(pred_path): | |
| if pred_path.endswith(".json"): | |
| pred = load_json(pred_path) | |
| else: | |
| pred = torch.load(pred_path) | |
| vcmr_res = {e["desc_id"]: e for e in pred["VCMR"]} | |
| video2idx = pred["video2idx"] | |
| return vcmr_res, video2idx | |
| def main_mix_results(pred_path, tef_pred_path, save_path, max_after_nms=100): | |
| """ | |
| Args: | |
| pred_path: contains top-200 VCMR predictions | |
| tef_pred_path: contains top-1000 VCMR predictions | |
| save_path: | |
| max_after_nms: int, | |
| Returns: | |
| save | |
| """ | |
| vcmr_res, video2idx = load_saved_res(pred_path) | |
| tef_vcmr_res, video2idx = load_saved_res(tef_pred_path) | |
| reranked_vcmr_res = {} | |
| num_valid = [] | |
| for desc_id, preds in tqdm(vcmr_res.items(), desc="Loop over the predictions"): | |
| tef_preds = tef_vcmr_res[desc_id]["predictions"] | |
| pred_moments = set([tuple(e[:3]) for e in preds["predictions"]]) | |
| reranked_moments = [e for e in tef_preds if tuple(e[:3]) in pred_moments][:max_after_nms] | |
| num_valid += [len(reranked_moments)] | |
| if len(reranked_moments) != 100: | |
| reranked_moments += reranked_moments[:100 - len(reranked_moments)] | |
| reranked_vcmr_res[desc_id] = dict( | |
| predictions=reranked_moments, | |
| desc_id=desc_id, | |
| desc=preds["desc"] | |
| ) | |
| print("There are {} moments founded on average".format(np.mean(num_valid))) | |
| reranked_predictions = dict( | |
| VCMR=list(reranked_vcmr_res.values()), | |
| video2idx=video2idx | |
| ) | |
| save_json(reranked_predictions, save_path) | |
| if __name__ == '__main__': | |
| import argparse | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--pred_path", type=str, help="path to prediction res") | |
| parser.add_argument("--tef_pred_path", type=str, help="path to TEF prediction res") | |
| parser.add_argument("--save_path", type=str, help="path to save the re-ranked predictions, same dir as --pred_path") | |
| parser.add_argument("--gt_path", type=str, help="path to ground truth file") | |
| args = parser.parse_args() | |
| main_mix_results(args.pred_path, args.tef_pred_path, args.save_path) | |
| metrics_path = args.save_path.replace(".json", "_metrics.json") | |
| eval_cmd = "python standalone_eval/eval.py --submission_path " + args.save_path + " --gt_path " + args.gt_path + \ | |
| " --save_path " + metrics_path | |
| results = subprocess.run(eval_cmd, shell=True) | |