first commit

initial version

features/Chinese_batch_frame_output_list.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7ba331cd1db032987eadfc624172f34172519047283d72381d57a488dab27c2
+size 36864128

features/Chinese_batch_visual_output_list.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c55e9f579a5fb77acec9e26e87e63f8284ff63e0fa933321db27ea3e2b569c5b
+size 3072128

features/English_batch_frame_output_list.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec6b4ac3f600f7cd07974f929ff926d1a03b84b2f52a5a6a250867aacfc9e39c
+size 36864128

features/English_batch_visual_output_list.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01edc18afdfd18d495d1bc4672c71d8d5088f77a40a98c359921f03f09ae15d8
+size 3072128

main_task_retrieval.py

@@ -0,0 +1,639 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+import os
+
+import torch
+from torch.utils.data import (SequentialSampler)
+import numpy as np
+import random
+from thop import profile
+
+from metrics import logging_rank
+import time
+import argparse
+from sklearn import preprocessing
+from transformers import BertTokenizer, AutoTokenizer, AutoModel
+from tensorboardX import SummaryWriter
+from modules.file_utils import PYTORCH_PRETRAINED_BERT_CACHE
+from modules.tokenization_clip import SimpleTokenizer as ClipTokenizer
+from modules.modeling import BirdModel_VT, BirdPreTrainedModel, BirdModel
+from modules.optimization import BertAdam
+from dataloaders.dataloader import DATALOADER_DICT
+from modules.until_module import get_dual_matrix
+from util import parallel_apply, get_logger
+from torch.cuda.amp import autocast, GradScaler
+
+torch.distributed.init_process_group(backend="nccl")
+
+global logger
+
+
+def get_args(description='CLIP4Clip on Retrieval Task'):
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument("--do_pretrain", action='store_true', help="Whether to run training.")
+    parser.add_argument("--do_train", action='store_true', help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true', help="Whether to run eval on the dev set.")
+    parser.add_argument("--do_params", action='store_true', help="text the params of the model.")
+    parser.add_argument("--use_frame_fea", action='store_true', help="whether use frame feature matching text")
+    parser.add_argument('--task', type=str, default="retrieval", choices=["retrieval_VT", "retrieval"],
+                        help="choose downstream task.")
+    parser.add_argument('--dataset', type=str, default="bird", choices=["bird", "msrvtt", "vatex", "msvd"],
+                        help="choose dataset.")
+    parser.add_argument('--num_thread_reader', type=int, default=1, help='')
+    parser.add_argument('--lr', type=float, default=0.0001, help='initial learning rate')
+    parser.add_argument('--text_lr', type=float, default=0.00001, help='text encoder learning rate')
+    parser.add_argument('--epochs', type=int, default=20, help='upper epoch limit')
+    parser.add_argument('--batch_size', type=int, default=256, help='batch size')
+    parser.add_argument('--batch_size_val', type=int, default=3500, help='batch size eval')
+    parser.add_argument('--lr_decay', type=float, default=0.9, help='Learning rate exp epoch decay')
+    parser.add_argument('--weight_decay', type=float, default=0.2, help='Learning rate exp epoch decay')
+    parser.add_argument('--n_display', type=int, default=100, help='Information display frequence')
+    parser.add_argument('--seed', type=int, default=42, help='random seed')
+    parser.add_argument('--max_words', type=int, default=32, help='')
+    parser.add_argument('--max_frames', type=int, default=12, help='')
+    parser.add_argument('--top_frames', type=int, default=3, help='')
+    parser.add_argument('--frame_sample', type=str, default="uniform", choices=["uniform", "random", "uniform_random"],
+                        help='frame sample strategy')
+    parser.add_argument('--frame_sample_len', type=str, default="fix", choices=["dynamic", "fix"],
+                        help='use dynamic frame length of fix frame length')
+    parser.add_argument('--language', type=str, default="chinese", choices=["chinese", "english"],
+                        help='language for text encoder')
+    parser.add_argument('--use_temp', action='store_true', help='whether to use temporal transformer')
+
+    parser.add_argument("--logdir", default=None, type=str, required=False, help="log dir for tensorboardX writer")
+    parser.add_argument("--output_dir", default=None, type=str, required=True,
+                        help="The output directory where the model predictions and checkpoints will be written.")
+    parser.add_argument("--cross_model", default="cross-base", type=str, required=False, help="Cross module")
+    parser.add_argument("--init_model", default=None, type=str, required=False, help="Initial model.")
+    parser.add_argument("--warmup_proportion", default=0.1, type=float,
+                        help="Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% of training.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument('--n_gpu', type=int, default=1, help="Changed in the execute process.")
+
+    parser.add_argument("--cache_dir", default="", type=str,
+                        help="Where do you want to store the pre-trained models downloaded from s3")
+
+    parser.add_argument('--enable_amp', action='store_true', help="whether to use pytorch amp")
+
+    parser.add_argument("--world_size", default=0, type=int, help="distribted training")
+    parser.add_argument("--local_rank", default=0, type=int, help="distribted training")
+    parser.add_argument("--rank", default=0, type=int, help="distribted training")
+    parser.add_argument('--coef_lr', type=float, default=1., help='coefficient for bert branch.')
+
+    args = parser.parse_args()
+
+    # Check paramenters
+    if args.gradient_accumulation_steps < 1:
+        raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
+            args.gradient_accumulation_steps))
+    if not args.do_train and not args.do_eval and not args.do_params:
+        raise ValueError("At least one of `do_train` or `do_eval` or 'do_params' must be True.")
+
+    args.batch_size = int(args.batch_size / args.gradient_accumulation_steps)
+
+    return args
+
+
+def set_seed_logger(args):
+    global logger
+    # predefining random initial seeds
+    random.seed(args.seed)
+    os.environ['PYTHONHASHSEED'] = str(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    torch.cuda.manual_seed(args.seed)
+    torch.cuda.manual_seed_all(args.seed)  # if you are using multi-GPU.
+    torch.backends.cudnn.benchmark = False
+    torch.backends.cudnn.deterministic = True
+
+    world_size = torch.distributed.get_world_size()
+    torch.cuda.set_device(args.local_rank)
+    args.world_size = world_size
+    rank = torch.distributed.get_rank()
+    args.rank = rank
+
+    if not os.path.exists(args.output_dir):
+        os.makedirs(args.output_dir, exist_ok=True)
+
+    logger = get_logger(os.path.join(args.output_dir, "log.txt"))
+    if args.local_rank == 0:
+        if args.logdir:
+            args.writer = SummaryWriter(args.logdir)
+        logger.info("Effective parameters:")
+        for key in sorted(args.__dict__):
+            logger.info("  <<< {}: {}".format(key, args.__dict__[key]))
+
+    return args
+
+
+def init_device(args, local_rank):
+    global logger
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu", local_rank)
+
+    n_gpu = torch.cuda.device_count()
+    logger.info("device: {} n_gpu: {}".format(device, n_gpu))
+    args.n_gpu = n_gpu
+
+    if args.batch_size % args.n_gpu != 0 or args.batch_size_val % args.n_gpu != 0:
+        raise ValueError(
+            "Invalid batch_size/batch_size_val and n_gpu parameter: {}%{} and {}%{}, should be == 0".format(
+                args.batch_size, args.n_gpu, args.batch_size_val, args.n_gpu))
+
+    return device, n_gpu
+
+
+def init_model(args, device, n_gpu, local_rank):
+    if args.init_model:
+        model_state_dict = torch.load(args.init_model, map_location='cpu')
+    else:
+        model_state_dict = None
+
+    # Prepare model
+    cache_dir = args.cache_dir if args.cache_dir else os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed')
+    if args.task == "retrieval_VT":
+        model = BirdModel_VT.from_pretrained(args.cross_model, cache_dir=cache_dir, state_dict=model_state_dict,
+                                             task_config=args)
+    elif args.task == "retrieval":
+        model = BirdModel.from_pretrained(args.cross_model, cache_dir=cache_dir, state_dict=model_state_dict,
+                                          task_config=args)
+    else:
+        raise Exception('wrong task! task should in [retrieve_VT, retrieve]')
+    # args.writer.add_graph(model)
+    model.to(device)
+
+    return model
+
+
+def prep_optimizer(args, model, num_train_optimization_steps, device, n_gpu, local_rank, coef_lr=1.):
+    if hasattr(model, 'module'):
+        model = model.module
+
+    param_optimizer = list(model.named_parameters())
+    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
+
+    decay_param_tp = [(n, p) for n, p in param_optimizer if not any(nd in n for nd in no_decay)]
+    no_decay_param_tp = [(n, p) for n, p in param_optimizer if any(nd in n for nd in no_decay)]
+
+    decay_clip_param_tp = [(n, p) for n, p in decay_param_tp if "visual_encoder.visual." in n]
+    decay_chinesebert_param_tp = [(n, p) for n, p in decay_param_tp if "text_encoder." in n]
+    decay_noclip_param_tp = [(n, p) for n, p in decay_param_tp if
+                             ("visual_encoder.visual." not in n) and ("text_encoder." not in n)]
+
+    no_decay_clip_param_tp = [(n, p) for n, p in no_decay_param_tp if "visual_encoder.visual." in n]
+    no_decay_text_param_tp = [(n, p) for n, p in no_decay_param_tp if "text_encoder." in n]
+    no_decay_noclip_param_tp = [(n, p) for n, p in no_decay_param_tp if
+                                ("visual_encoder.visual." not in n) and ("text_encoder." not in n)]
+
+    weight_decay = args.weight_decay
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in decay_clip_param_tp], 'weight_decay': weight_decay, 'lr': args.lr * coef_lr},
+        {'params': [p for n, p in decay_chinesebert_param_tp], 'weight_decay': weight_decay, 'lr': args.text_lr},
+        {'params': [p for n, p in decay_noclip_param_tp], 'weight_decay': weight_decay},
+        {'params': [p for n, p in no_decay_clip_param_tp], 'weight_decay': 0.0, 'lr': args.lr * coef_lr},
+        {'params': [p for n, p in no_decay_text_param_tp], 'weight_decay': 0.0, 'lr': args.text_lr},
+        {'params': [p for n, p in no_decay_noclip_param_tp], 'weight_decay': 0.0}
+    ]
+
+    scheduler = None
+    optimizer = BertAdam(optimizer_grouped_parameters, lr=args.lr, warmup=args.warmup_proportion,
+                         schedule='warmup_cosine', b1=0.9, b2=0.98, e=1e-6,
+                         t_total=num_train_optimization_steps, weight_decay=weight_decay,
+                         max_grad_norm=1.0)
+
+    model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[local_rank],
+                                                      output_device=local_rank, find_unused_parameters=True)
+    # if args.local_rank == 0:
+    #     for name, parameters in model.named_parameters():
+    #         logger.info("name:{} requires_grad:{} size:{}".format(name, parameters.requires_grad, parameters.size()))
+    return optimizer, scheduler, model
+
+
+def save_model(epoch, args, model, type_name=""):
+    # Only save the model it-self
+    model_to_save = model.module if hasattr(model, 'module') else model
+    output_model_file = os.path.join(
+        args.output_dir, "pytorch_model.bin.{}{}".format("" if type_name == "" else type_name + ".", epoch))
+    torch.save(model_to_save.state_dict(), output_model_file)
+    logger.info("Model saved to %s", output_model_file)
+    return output_model_file
+
+
+def load_model(epoch, args, n_gpu, device, model_file=None):
+    if model_file is None or len(model_file) == 0:
+        model_file = os.path.join(args.output_dir, "pytorch_model.bin.{}".format(epoch))
+    if os.path.exists(model_file):
+        model_state_dict = torch.load(model_file, map_location='cpu')
+        if args.local_rank == 0:
+            logger.info("Model loaded from %s", model_file)
+        # Prepare model
+        cache_dir = args.cache_dir if args.cache_dir else os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
+                                                                       'distributed')
+        if args.task == "retrieval":
+            model = BirdModel.from_pretrained(args.cross_model, cache_dir=cache_dir, state_dict=model_state_dict,
+                                              task_config=args)
+        elif args.task == "retrieval_VT":
+            model = BirdModel_VT.from_pretrained(args.cross_model, cache_dir=cache_dir, state_dict=model_state_dict,
+                                                 task_config=args)
+        else:
+            model = None
+
+        model.to(device)
+    else:
+        model = None
+    return model
+
+
+def train_epoch(epoch, args, model, train_dataloader, device, n_gpu, optimizer, scheduler, scaler, global_step, local_rank=0):
+    global logger
+    torch.cuda.empty_cache()
+    model.train()
+    log_step = args.n_display
+    start_time = time.time()
+    total_loss = 0
+    load_start_time = time.time()
+    for step, batch in enumerate(train_dataloader):
+        load_finish_time = time.time()
+        if global_step % log_step == 0 and local_rank == 0:
+            logger.info("data loader time:{}".format(load_finish_time - load_start_time))
+        global_step += 1
+        if n_gpu == 1:
+            # multi-gpu does scattering it-self
+            batch = tuple(t.to(device=device, non_blocking=True) for t in batch)
+
+        with autocast(enabled=args.enable_amp):
+            if args.task == "retrieval_VT":
+                query_ids, query_mask, video_data, video_frame, title_ids, title_mask, idx = batch
+                loss = model(query_ids, query_mask, video_data, video_frame, title_ids, title_mask, idx, global_step)
+            elif args.task == "retrieval":
+                query_ids, query_mask, video_data, video_frame, idx = batch
+                loss = model(query_ids, query_mask, video_data, video_frame, idx, global_step)
+            else:
+                raise ValueError("wrong task type:{}".format(args.task))
+            if n_gpu > 1:
+                loss = loss.mean()  # mean() to average on multi-gpu.
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+        forward_time = time.time()
+        if args.enable_amp:
+            scaler.scale(loss).backward()
+        else:
+            loss.backward()
+        total_loss += float(loss)
+        backward_time = time.time()
+        if global_step % log_step == 0 and local_rank == 0:
+            logger.info("forward_time:{},backward_time:{}".format(forward_time - load_finish_time, backward_time - forward_time))
+
+        if (step + 1) % args.gradient_accumulation_steps == 0:
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+
+            if scheduler is not None:
+                scheduler.step()  # Update learning rate schedule
+
+            if args.enable_amp:
+                scaler.step(optimizer)
+                scaler.update()
+            else:
+                optimizer.step()
+
+            optimizer.zero_grad()
+
+            if global_step % log_step == 0 and local_rank == 0:
+                logger.info("Epoch: %d/%s, Step: %d/%d, Lr: %s, Loss: %f, Time/step: %f", epoch + 1,
+                            args.epochs, step + 1,
+                            len(train_dataloader),
+                            "-".join([str('%.9f' % itm) for itm in sorted(list(set(optimizer.get_lr())))]),
+                            float(loss),
+                            (time.time() - start_time) / (log_step * args.gradient_accumulation_steps))
+                if args.logdir:
+                    # args.writer.add_scalar('loss', loss.item(), global_step=global_step)
+                    args.writer.add_scalars('lr', {"lr%d" % i: itm for i, itm in enumerate(sorted(list(set(optimizer.get_lr()))))},
+                                            global_step=global_step)
+                start_time = time.time()
+        load_start_time = time.time()
+    total_loss = total_loss / len(train_dataloader)
+    return total_loss, global_step
+
+
+def _run_on_single_gpu(model, batch_query_output_list, batch_visual_output_list, batch_title_output_list,
+                       batch_frame_output_list):
+    sim_matrix = []
+    sim_matrix_title = []
+    sim_matrix_frame = []
+    for idx1, query_output in enumerate(batch_query_output_list):
+        each_row = []
+        title_each_row = []
+        frame_each_row = []
+        for idx2, (visual_output, title_output, frame_output) in enumerate(zip(batch_visual_output_list,
+                                                                               batch_title_output_list, batch_frame_output_list)):
+            b1b2_logits = model.loose_similarity(query_output, visual_output)
+            title_logits = model.loose_similarity(query_output, title_output)
+            frame_logits = model.loose_similarity(query_output, frame_output)
+            frame_logits = torch.topk(frame_logits, k=model.top_frames, dim=2)[0]
+            frame_logits = torch.mean(frame_logits, dim=2)
+            b1b2_logits = b1b2_logits.cpu().detach().numpy()
+            title_logits = title_logits.cpu().detach().numpy()
+            frame_logits = frame_logits.cpu().detach().numpy()
+            each_row.append(b1b2_logits)
+            title_each_row.append(title_logits)
+            frame_each_row.append(frame_logits)
+            # logger.info("b1b2_logits:{}".format(b1b2_logits.shape))
+            # logger.info("frame_logits:{}".format(frame_logits.shape))
+
+        each_row = np.concatenate(tuple(each_row), axis=-1)
+        # logger.info("each_row:{}".format(each_row.shape))
+        title_each_row = np.concatenate(tuple(title_each_row), axis=-1)
+        # frame_each_row = np.concatenate(tuple(frame_each_row), axis=-1)
+        frame_each_row = np.concatenate(tuple(frame_each_row), axis=1)
+        # logger.info("frame_each_row:{}".format(frame_each_row.shape))
+        # sim_matrix.append(preprocessing.scale(each_row, axis=1))
+        sim_matrix.append(each_row)
+        sim_matrix_title.append(title_each_row)
+        sim_matrix_frame.append(frame_each_row)
+    # logger.info("sim_matrix:{}".format(sim_matrix))
+    return sim_matrix, sim_matrix_title, sim_matrix_frame
+
+
+def eval_epoch(args, model, test_dataloader, device, n_gpu):
+    torch.cuda.empty_cache()
+    if hasattr(model, 'module'):
+        model = model.module.to(device)
+    else:
+        model = model.to(device)
+
+    model.eval()
+    logger.info("args.task:{}".format(args.task))
+
+    # if multi_sentence_ == True: compute the similarity with multi-sentences retrieval
+    multi_sentence_ = False
+
+    cut_off_points_, sentence_num_, video_num_ = [], -1, -1
+    if hasattr(test_dataloader.dataset, 'multi_sentence_per_video') \
+            and test_dataloader.dataset.multi_sentence_per_video:
+        multi_sentence_ = True
+        cut_off_points_ = test_dataloader.dataset.cut_off_points  # used to tag the label when calculate the metric
+        sentence_num_ = test_dataloader.dataset.sentence_num  # used to cut the sentence representation
+        video_num_ = test_dataloader.dataset.video_num  # used to cut the video representation
+        cut_off_points_ = [itm - 1 for itm in cut_off_points_]
+    logger.info("multi_sentence_:{}".format(multi_sentence_))
+
+    with torch.no_grad():
+        batch_query_output_list, batch_visual_output_list = [], []
+        batch_title_output_list = []
+        batch_frame_output_list = []
+        total_video_num = 0
+        # ----------------------------
+        # 1. cache the features
+        # ----------------------------
+        for bid, batch in enumerate(test_dataloader):
+            batch = tuple(t.to(device) for t in batch)
+            if args.task == "retrieval_VT":
+                query_ids, query_mask, video, video_frame, title_ids, title_mask = batch
+            elif args.task == "retrieval":
+                query_ids, query_mask, video, video_frame = batch
+            else:
+                raise ValueError("wrong task type:{}".format(args.task))
+
+            print("bid:{}/{}".format(bid, len(test_dataloader)), end="\r")
+            if multi_sentence_:
+                # multi-sentences retrieval means: one frame clip has two or more descriptions.
+                b, *_t = video.shape
+                # logger.info("query_ids.shape:{}".format(query_ids.shape))
+                # logger.info("video.shape:{}".format(video.shape))
+                query_output = model.text_encoder(query_ids, query_mask)
+                batch_query_output_list.append(query_output)
+                title_output = torch.zeros_like(query_output)
+                batch_title_output_list.append(title_output)
+                s_, e_ = total_video_num, total_video_num + b
+                filter_inds = [itm - s_ for itm in cut_off_points_ if s_ <= itm < e_]
+
+                if len(filter_inds) > 0:
+                    video = video[filter_inds, ...]
+                    visual_output, frame_output = model.visual_encoder(video, video_frame)
+                    # frame_output = torch.mean(frame_output, dim=1)
+                    batch_visual_output_list.append(visual_output)
+                    batch_frame_output_list.append(frame_output)
+                total_video_num += b
+            else:
+                query_output = model.text_encoder(query_ids, query_mask)
+                visual_output, frame_output = model.visual_encoder(video, video_frame)
+                # frame_output = torch.mean(frame_output, dim=1)
+                if args.task == "retrieval_VT":
+                    title_output = model.text_encoder(title_ids, title_mask)
+                    logger.info("title_output.shape:{}".format(title_output.shape))
+                elif args.task == "retrieval":
+                    title_output = torch.zeros_like(query_output)
+                else:
+                    raise ValueError("wrong task type:{}".format(args.task))
+
+                # logger.info("query_output.shape:{}".format(query_output.shape))
+                # logger.info("weight_VTM:{},weight_FTM:{},exp:{}".format(model.weight_VTM, model.weight_FTM,
+                #                                                         model.text_encoder.logit_scale.exp()))
+                logger.info("visual_output.shape:{}".format(visual_output.shape))
+                logger.info("frame_output.shape:{}".format(frame_output.shape))
+
+                batch_query_output_list.append(query_output)
+                batch_visual_output_list.append(visual_output)
+                batch_title_output_list.append(title_output)
+                batch_frame_output_list.append(frame_output)
+
+        # ----------------------------------
+        # 2. calculate the similarity
+        # ----------------------------------
+        logger.info("n_gpu:{}".format(n_gpu))
+        # logger.info("model.weight_sum:{}".format(model.weight_sum))
+        if n_gpu > 1:
+            device_ids = list(range(n_gpu))
+            batch_t_output_splits = []
+            batch_v_output_splits = []
+            batch_title_output_splits = []
+            batch_frame_output_splits = []
+            bacth_len = len(batch_query_output_list)
+            split_len = (bacth_len + n_gpu - 1) // n_gpu
+            for dev_id in device_ids:
+                s_, e_ = dev_id * split_len, (dev_id + 1) * split_len
+                if dev_id == 0:
+                    batch_t_output_splits.append(batch_query_output_list[s_:e_])
+                    batch_v_output_splits.append(batch_visual_output_list)
+                    batch_title_output_splits.append(batch_title_output_list)
+                    batch_frame_output_splits.append(batch_frame_output_list)
+                else:
+                    devc = torch.device('cuda:{}'.format(str(dev_id)))
+
+                    devc_batch_list = [b.to(devc) for b in batch_query_output_list[s_:e_]]
+                    batch_t_output_splits.append(devc_batch_list)
+                    devc_batch_list = [b.to(devc) for b in batch_visual_output_list]
+                    batch_v_output_splits.append(devc_batch_list)
+                    devc_batch_list = [b.to(devc) for b in batch_title_output_list]
+                    batch_title_output_splits.append(devc_batch_list)
+                    devc_batch_list = [b.to(devc) for b in batch_frame_output_list]
+                    batch_frame_output_splits.append(devc_batch_list)
+
+            parameters_tuple_list = [(batch_t_output_splits[dev_id], batch_v_output_splits[dev_id],
+                                      batch_title_output_splits[dev_id], batch_frame_output_splits[dev_id]) for dev_id in device_ids]
+            parallel_outputs_tuple = parallel_apply(_run_on_single_gpu, model, parameters_tuple_list, device_ids)
+            sim_matrix = []
+            sim_matrix_title = []
+            sim_matrix_frame = []
+            for idx in range(len(parallel_outputs_tuple)):
+                parallel_outputs, parallel_outputs_title, parallel_outputs_frame = parallel_outputs_tuple[idx]
+                sim_matrix += parallel_outputs
+                sim_matrix_title += parallel_outputs_title
+                sim_matrix_frame += parallel_outputs_frame
+            sim_matrix = np.concatenate(tuple(sim_matrix), axis=0)
+            sim_matrix_title = np.concatenate(tuple(sim_matrix_title), axis=0)
+            sim_matrix_frame = np.concatenate(tuple(sim_matrix_frame), axis=0)
+        else:
+            sim_matrix_tuple = _run_on_single_gpu(model, batch_query_output_list, batch_visual_output_list,
+                                                  batch_title_output_list, batch_frame_output_list)
+            sim_matrix, sim_matrix_title, sim_matrix_frame = sim_matrix_tuple
+            sim_matrix = np.concatenate(tuple(sim_matrix), axis=0)
+            sim_matrix_title = np.concatenate(tuple(sim_matrix_title), axis=0)
+            sim_matrix_frame = np.concatenate(tuple(sim_matrix_frame), axis=0)
+
+            batch_visual_output_list = torch.cat(batch_visual_output_list, dim=0)
+            batch_frame_output_list = torch.cat(batch_frame_output_list, dim=0)
+            batch_visual_output_list = batch_visual_output_list.cpu().detach().numpy()
+            batch_frame_output_list = batch_frame_output_list.cpu().detach().numpy()
+            # np.save("/ai/swxdisk/data/vatex/features/Chinese_batch_visual_output_list", batch_visual_output_list)
+            # np.save("/ai/swxdisk/data/vatex/features/Chinese_batch_frame_output_list", batch_frame_output_list)
+            np.save("/ai/swxdisk/data/vatex/features/English_batch_visual_output_list", batch_visual_output_list)
+            np.save("/ai/swxdisk/data/vatex/features/English_batch_frame_output_list", batch_frame_output_list)
+
+        # logger.info("sim_matrix:{}".format(sim_matrix.shape))
+        # logger.info("sim_matrix_frame:{}".format(sim_matrix_frame.shape))
+        # np.save("/ai/swxdisk/data/msrvtt/visualize/sim_matrix", sim_matrix)
+        # np.save("/ai/swxdisk/data/msrvtt/visualize/sim_matrix_frame_top2", sim_matrix_frame)
+        # sim_matrix_frame = np.topk(sim_matrix_frame, k=model.top_frames, dim=2)[0]
+        # sim_matrix_frame = np.mean(sim_matrix_frame, dim=2)
+        if args.use_frame_fea:
+            sim_matrix += sim_matrix_frame
+
+        if args.task == "retrieval_VT":
+            # logger.info("sim_matrix_title:{}".format(sim_matrix_title))
+            weight_title = model.weight_title
+            sim_matrix += weight_title * sim_matrix_title
+            # sim_matrix = weight_title * sim_matrix_title
+
+    logger.info("sim matrix size:  {}".format(np.array(sim_matrix).shape))
+    # sim_matrix = get_dual_matrix(sim_matrix)
+
+    tv_metrics = logging_rank(sim_matrix, multi_sentence_, cut_off_points_, logger)
+    return tv_metrics
+
+
+def main():
+    global logger
+    args = get_args()
+    args = set_seed_logger(args)
+    device, n_gpu = init_device(args, args.local_rank)
+
+    # get text pretrained path
+    pretrained_text = "hfl/chinese-roberta-wwm-ext"
+    args.pretrained_text = pretrained_text
+    if args.language == "chinese":
+        tokenizer = BertTokenizer.from_pretrained(pretrained_text)
+    else:
+        tokenizer = ClipTokenizer()
+
+    model = init_model(args, device, n_gpu, args.local_rank)
+    ## ####################################
+    # freeze testing
+    ## ####################################
+    '''
+    assert args.freeze_layer_num <= 12 and args.freeze_layer_num >= -1
+    if hasattr(model, "visual_encoder") and args.freeze_layer_num > -1:
+        for name, param in model.visual_encoder.named_parameters():
+            # top layers always need to train
+            if name.find("ln_final.") == 0 or name.find("text_projection") == 0 or name.find("logit_scale") == 0 \
+                    or name.find("visual.ln_post.") == 0 or name.find("visual.proj") == 0:
+                continue  # need to train
+            elif name.find("visual.transformer.resblocks.") == 0 or name.find("transformer.resblocks.") == 0:
+                layer_num = int(name.split(".resblocks.")[1].split(".")[0])
+                if layer_num >= args.freeze_layer_num:
+                    continue  # need to train
+
+            if args.linear_patch == "3d" and name.find("conv2."):
+                continue
+            else:
+                # paramenters which < freeze_layer_num will be freezed
+                param.requires_grad = False
+    '''
+    assert args.dataset in DATALOADER_DICT
+    test_dataloader, test_length = DATALOADER_DICT[args.dataset]["test"](args, tokenizer)
+
+    if args.local_rank == 0:
+        logger.info("***** Running test *****")
+        logger.info("  Num examples = %d", test_length)
+        logger.info("  Batch size = %d", args.batch_size_val)
+        logger.info("  Num steps = %d", len(test_dataloader))
+
+    if args.do_train:
+        train_dataloader, train_length, train_sampler = DATALOADER_DICT[args.dataset]["train"](args, tokenizer)
+
+        num_train_optimization_steps = (int(len(train_dataloader) + args.gradient_accumulation_steps - 1)
+                                        / args.gradient_accumulation_steps) * args.epochs
+        # logger.info("train_dataloader len = {}".format(len(train_dataloader)))
+        # logger.info("gradient_accumulation_steps = {}".format(args.gradient_accumulation_steps))
+        coef_lr = args.coef_lr
+        optimizer, scheduler, model = prep_optimizer(args, model, num_train_optimization_steps, device, n_gpu,
+                                                     args.local_rank, coef_lr=coef_lr)
+
+        if args.local_rank == 0:
+            logger.info("***** Running training *****")
+            logger.info("  Num examples = %d", train_length)
+            logger.info("  Batch size = %d", args.batch_size)
+            logger.info("  Num steps = %d", num_train_optimization_steps * args.gradient_accumulation_steps)
+
+        best_score = 0.00001
+        best_output_model_file = "None"
+        global_step = 0
+        if args.enable_amp:
+            scaler = GradScaler()
+        else:
+            scaler = None
+        for epoch in range(args.epochs):
+            train_sampler.set_epoch(epoch)
+            tr_loss, global_step = train_epoch(epoch, args, model, train_dataloader, device, n_gpu, optimizer,
+                                               scheduler, scaler, global_step, local_rank=args.local_rank)
+            if args.local_rank == 0:
+                logger.info("Epoch %d/%s Finished, Train Loss: %f", epoch + 1, args.epochs, tr_loss)
+                # for name, param in model.named_parameters():
+                # args.writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch)
+                # writer.add_histogram(name + '/grad', param.requires_grad_().clone().cpu().data.numpy(), epoch)
+                if epoch % 1 == 0:
+                    ## Uncomment if want to save checkpoint
+                    output_model_file = save_model(epoch, args, model, type_name="")
+                    # if epoch == 100:
+                    metrics = eval_epoch(args, model, test_dataloader, device, n_gpu)
+                    if args.logdir:
+                        args.writer.add_scalars('metrics', {'R1': metrics["R1"], 'R5': metrics["R5"],
+                                                            'R10': metrics["R10"]}, global_step=epoch)
+                    if best_score < metrics["R1"]:
+                        best_score = metrics["R1"]
+                        best_output_model_file = output_model_file
+                    logger.info("The best model is: {}, the R1 is: {:.4f}".format(best_output_model_file, best_score))
+
+    elif args.do_eval:
+        if args.local_rank == 0:
+            eval_epoch(args, model, test_dataloader, device, n_gpu)
+    elif args.do_params:
+        logger.info("do_params begin!")
+        # total = sum([param.nelement() for param in model.parameters()])
+        total = sum(p.numel() for p in model.parameters())
+        logger.info("Number of parameter: %.2fM" % (total / 1e6))
+        for bid, batch in enumerate(test_dataloader):
+            batch = tuple(t.to(device) for t in batch)
+            query_ids, query_mask, pos_video_data, pos_title_ids, pos_title_mask, = batch
+            flops, params = profile(model, (query_ids, query_mask, pos_video_data, pos_title_ids, pos_title_mask,))
+            print('flops: %.2f G, params: %.2f M' % (flops / 1e9, params / 1e6))
+            break
+    if args.local_rank == 0 and args.logdir:
+        args.writer.close()
+
+
+if __name__ == "__main__":
+    main()

models/Chinese_vatex.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:064f9035ddb61431c18584443be4d7375adf8eda396c4ad21f32e73477882bc2
+size 637216411

models/English_vatex.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29cf3b4e70167178007b621a4e27eb5b400caa7df2f5d911d31827000fa10ac3
+size 404112709

modules/__init__.py

@@ -0,0 +1 @@
+#encoding:utf-8

modules/bpe_simple_vocab_16e6.txt.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
+size 1356917

modules/cross-base/bert_base_6layer_6conect.json

@@ -0,0 +1,30 @@
+{
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 512,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "type_vocab_size": 2,
+  "vocab_size": 30522,
+  "v_feature_size": 2048,
+  "v_target_size": 1601,
+  "v_hidden_size": 768,
+  "v_num_hidden_layers":6,
+  "v_num_attention_heads":8,
+  "v_intermediate_size":768,
+  "bi_hidden_size":768,
+  "bi_num_attention_heads":8,
+  "bi_intermediate_size": 768,
+  "bi_attention_type":1,
+  "v_attention_probs_dropout_prob":0.1,
+  "v_hidden_act":"gelu",
+  "v_hidden_dropout_prob":0.1,
+  "v_initializer_range":0.02,
+  "v_biattention_id":[0, 1, 2, 3, 4, 5],
+  "t_biattention_id":[6, 7, 8, 9, 10, 11],
+  "pooling_method": "mul"
+}

modules/cross-base/cross_config.json

@@ -0,0 +1,18 @@
+{
+  "max_position_embeddings": 48,
+  "chinese_hidden_size": 768,
+  "temporal_hidden_size": 512,
+  "temporal_attention_heads": 8,
+  "temporal_hidden_layers": 4,
+  "mlm_probability": 0.15,
+  "co_attention_layers": 4,
+  "proj_num_layers": 2,
+  "pred_num_layers": 2,
+  "weight_FAM": 0.05,
+  "weight_VTM": 0.45,
+  "weight_FTM": 0.45,
+  "weight_MLM": 0.05,
+  "weight_VTM_finetune": 0.85,
+  "weight_FTM_finetune": 0.15,
+  "pretrained_clip_name": "ViT-B/32"
+}

modules/file_utils.py

@@ -0,0 +1,239 @@
+"""
+Utilities for working with the local dataset cache.
+This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp
+Copyright by the AllenNLP authors.
+"""
+
+import os
+import logging
+import shutil
+import tempfile
+import json
+from urllib.parse import urlparse
+from pathlib import Path
+from typing import Optional, Tuple, Union, IO, Callable, Set
+from hashlib import sha256
+from functools import wraps
+
+from tqdm import tqdm
+
+import boto3
+from botocore.exceptions import ClientError
+import requests
+
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
+PYTORCH_PRETRAINED_BERT_CACHE = Path(os.getenv('PYTORCH_PRETRAINED_BERT_CACHE',
+                                               Path.home() / '.pytorch_pretrained_bert'))
+
+
+def url_to_filename(url: str, etag: str = None) -> str:
+    """
+    Convert `url` into a hashed filename in a repeatable way.
+    If `etag` is specified, append its hash to the url's, delimited
+    by a period.
+    """
+    url_bytes = url.encode('utf-8')
+    url_hash = sha256(url_bytes)
+    filename = url_hash.hexdigest()
+
+    if etag:
+        etag_bytes = etag.encode('utf-8')
+        etag_hash = sha256(etag_bytes)
+        filename += '.' + etag_hash.hexdigest()
+
+    return filename
+
+
+def filename_to_url(filename: str, cache_dir: Union[str, Path] = None) -> Tuple[str, str]:
+    """
+    Return the url and etag (which may be ``None``) stored for `filename`.
+    Raise ``FileNotFoundError`` if `filename` or its stored metadata do not exist.
+    """
+    if cache_dir is None:
+        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
+    if isinstance(cache_dir, Path):
+        cache_dir = str(cache_dir)
+
+    cache_path = os.path.join(cache_dir, filename)
+    if not os.path.exists(cache_path):
+        raise FileNotFoundError("file {} not found".format(cache_path))
+
+    meta_path = cache_path + '.json'
+    if not os.path.exists(meta_path):
+        raise FileNotFoundError("file {} not found".format(meta_path))
+
+    with open(meta_path) as meta_file:
+        metadata = json.load(meta_file)
+    url = metadata['url']
+    etag = metadata['etag']
+
+    return url, etag
+
+
+def cached_path(url_or_filename: Union[str, Path], cache_dir: Union[str, Path] = None) -> str:
+    """
+    Given something that might be a URL (or might be a local path),
+    determine which. If it's a URL, download the file and cache it, and
+    return the path to the cached file. If it's already a local path,
+    make sure the file exists and then return the path.
+    """
+    if cache_dir is None:
+        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
+    if isinstance(url_or_filename, Path):
+        url_or_filename = str(url_or_filename)
+    if isinstance(cache_dir, Path):
+        cache_dir = str(cache_dir)
+
+    parsed = urlparse(url_or_filename)
+
+    if parsed.scheme in ('http', 'https', 's3'):
+        # URL, so get it from the cache (downloading if necessary)
+        return get_from_cache(url_or_filename, cache_dir)
+    elif os.path.exists(url_or_filename):
+        # File, and it exists.
+        return url_or_filename
+    elif parsed.scheme == '':
+        # File, but it doesn't exist.
+        raise FileNotFoundError("file {} not found".format(url_or_filename))
+    else:
+        # Something unknown
+        raise ValueError("unable to parse {} as a URL or as a local path".format(url_or_filename))
+
+
+def split_s3_path(url: str) -> Tuple[str, str]:
+    """Split a full s3 path into the bucket name and path."""
+    parsed = urlparse(url)
+    if not parsed.netloc or not parsed.path:
+        raise ValueError("bad s3 path {}".format(url))
+    bucket_name = parsed.netloc
+    s3_path = parsed.path
+    # Remove '/' at beginning of path.
+    if s3_path.startswith("/"):
+        s3_path = s3_path[1:]
+    return bucket_name, s3_path
+
+
+def s3_request(func: Callable):
+    """
+    Wrapper function for s3 requests in order to create more helpful error
+    messages.
+    """
+
+    @wraps(func)
+    def wrapper(url: str, *args, **kwargs):
+        try:
+            return func(url, *args, **kwargs)
+        except ClientError as exc:
+            if int(exc.response["Error"]["Code"]) == 404:
+                raise FileNotFoundError("file {} not found".format(url))
+            else:
+                raise
+
+    return wrapper
+
+
+@s3_request
+def s3_etag(url: str) -> Optional[str]:
+    """Check ETag on S3 object."""
+    s3_resource = boto3.resource("s3")
+    bucket_name, s3_path = split_s3_path(url)
+    s3_object = s3_resource.Object(bucket_name, s3_path)
+    return s3_object.e_tag
+
+
+@s3_request
+def s3_get(url: str, temp_file: IO) -> None:
+    """Pull a file directly from S3."""
+    s3_resource = boto3.resource("s3")
+    bucket_name, s3_path = split_s3_path(url)
+    s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file)
+
+
+def http_get(url: str, temp_file: IO) -> None:
+    req = requests.get(url, stream=True)
+    content_length = req.headers.get('Content-Length')
+    total = int(content_length) if content_length is not None else None
+    progress = tqdm(unit="B", total=total)
+    for chunk in req.iter_content(chunk_size=1024):
+        if chunk: # filter out keep-alive new chunks
+            progress.update(len(chunk))
+            temp_file.write(chunk)
+    progress.close()
+
+
+def get_from_cache(url: str, cache_dir: Union[str, Path] = None) -> str:
+    """
+    Given a URL, look for the corresponding dataset in the local cache.
+    If it's not there, download it. Then return the path to the cached file.
+    """
+    if cache_dir is None:
+        cache_dir = PYTORCH_PRETRAINED_BERT_CACHE
+    if isinstance(cache_dir, Path):
+        cache_dir = str(cache_dir)
+
+    os.makedirs(cache_dir, exist_ok=True)
+
+    # Get eTag to add to filename, if it exists.
+    if url.startswith("s3://"):
+        etag = s3_etag(url)
+    else:
+        response = requests.head(url, allow_redirects=True)
+        if response.status_code != 200:
+            raise IOError("HEAD request failed for url {} with status code {}"
+                          .format(url, response.status_code))
+        etag = response.headers.get("ETag")
+
+    filename = url_to_filename(url, etag)
+
+    # get cache path to put the file
+    cache_path = os.path.join(cache_dir, filename)
+
+    if not os.path.exists(cache_path):
+        # Download to temporary file, then copy to cache dir once finished.
+        # Otherwise you get corrupt cache entries if the download gets interrupted.
+        with tempfile.NamedTemporaryFile() as temp_file:
+            logger.info("%s not found in cache, downloading to %s", url, temp_file.name)
+
+            # GET file object
+            if url.startswith("s3://"):
+                s3_get(url, temp_file)
+            else:
+                http_get(url, temp_file)
+
+            # we are copying the file before closing it, so flush to avoid truncation
+            temp_file.flush()
+            # shutil.copyfileobj() starts at the current position, so go to the start
+            temp_file.seek(0)
+
+            logger.info("copying %s to cache at %s", temp_file.name, cache_path)
+            with open(cache_path, 'wb') as cache_file:
+                shutil.copyfileobj(temp_file, cache_file)
+
+            logger.info("creating metadata file for %s", cache_path)
+            meta = {'url': url, 'etag': etag}
+            meta_path = cache_path + '.json'
+            with open(meta_path, 'w') as meta_file:
+                json.dump(meta, meta_file)
+
+            logger.info("removing temp file %s", temp_file.name)
+
+    return cache_path
+
+
+def read_set_from_file(filename: str) -> Set[str]:
+    '''
+    Extract a de-duped collection (set) of text from a file.
+    Expected file format is one item per line.
+    '''
+    collection = set()
+    with open(filename, 'r', encoding='utf-8') as file_:
+        for line in file_:
+            collection.add(line.rstrip())
+    return collection
+
+
+def get_file_extension(path: str, dot=True, lower: bool = True):
+    ext = os.path.splitext(path)[1]
+    ext = ext if dot else ext[1:]
+    return ext.lower() if lower else ext

modules/modeling.py

@@ -0,0 +1,507 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import logging
+import torch
+from torch import nn
+import numpy as np
+import torch.nn.functional as F
+from transformers import AutoConfig, AutoModel, BertTokenizer
+from modules.tokenization_clip import SimpleTokenizer as ClipTokenizer
+from modules.until_module import PreTrainedModel, AllGather, CrossEn, Dual_CrossEn
+from modules.module_cross import TextEncoder, VisualEncoder, CrossConfig, BertLMPredictionHead
+
+logger = logging.getLogger(__name__)
+allgather = AllGather.apply
+
+
+class CLIP4ClipPreTrainedModel(PreTrainedModel, nn.Module):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+
+    def __init__(self, cross_config, *inputs, **kwargs):
+        super(CLIP4ClipPreTrainedModel, self).__init__(cross_config)
+        self.cross_config = cross_config
+
+    @classmethod
+    def from_pretrained(cls, cross_model_name, state_dict=None, cache_dir=None, type_vocab_size=2, *inputs, **kwargs):
+
+        task_config = None
+        if "task_config" in kwargs.keys():
+            task_config = kwargs["task_config"]
+            if not hasattr(task_config, "local_rank"):
+                task_config.__dict__["local_rank"] = 0
+            elif task_config.local_rank == -1:
+                task_config.local_rank = 0
+
+        cross_config, _ = CrossConfig.get_config(cross_model_name, cache_dir, type_vocab_size, state_dict=None,
+                                                 task_config=task_config)
+
+        model = cls(cross_config, *inputs, **kwargs)
+
+        if state_dict is not None:
+            model = cls.init_preweight(model, state_dict, task_config=task_config)
+
+        return model
+
+
+def show_log(task_config, info):
+    if task_config is None or task_config.local_rank == 0:
+        logger.warning(info)
+
+
+def update_attr(target_name, target_config, target_attr_name, source_config, source_attr_name, default_value=None):
+    if hasattr(source_config, source_attr_name):
+        if default_value is None or getattr(source_config, source_attr_name) != default_value:
+            setattr(target_config, target_attr_name, getattr(source_config, source_attr_name))
+            show_log(source_config, "Set {}.{}: {}.".format(target_name,
+                                                            target_attr_name, getattr(target_config, target_attr_name)))
+    return target_config
+
+
+def check_attr(target_name, task_config):
+    return hasattr(task_config, target_name) and task_config.__dict__[target_name]
+
+
+class BirdPreTrainedModel(CLIP4ClipPreTrainedModel):
+    def __init__(self, cross_config, task_config):
+        super(BirdPreTrainedModel, self).__init__(cross_config)
+        self.task_config = task_config
+        self.rank = task_config.local_rank
+        self.mlm_probability = cross_config.mlm_probability
+        self.top_frames = task_config.top_frames
+        # self.weight_sum = torch.nn.Parameter(torch.tensor([0.5], dtype=torch.float32), requires_grad=True)
+        self.weight_FAM = cross_config.weight_FAM
+        self.weight_VTM = cross_config.weight_VTM
+        self.weight_FTM = cross_config.weight_FTM
+        self.weight_MLM = cross_config.weight_MLM
+        self.contrast_momentum = task_config.contrast_momentum
+        self.contrast_temperature = task_config.contrast_temperature
+        self.contrast_num_negative = task_config.contrast_num_negative
+        ################## chinese text Encoder
+        if self.task_config.language == "chinese":
+            self.tokenizer = BertTokenizer.from_pretrained(self.task_config.pretrained_text)
+        else:
+            self.tokenizer = ClipTokenizer()
+        if self.rank == 0:
+            logger.info("voacb_size:{}".format(self.tokenizer.vocab_size))
+        t_config = AutoConfig.from_pretrained(self.task_config.pretrained_text)
+        self.text_encoder = TextEncoder(self.task_config, cross_config)
+        self.text_encoder_k = TextEncoder(self.task_config, cross_config)
+        self.t_projector = MLP(num_layers=cross_config.proj_num_layers)
+        self.t_projector_k = MLP(num_layers=cross_config.proj_num_layers)
+        nn.SyncBatchNorm.convert_sync_batchnorm(self.t_projector)
+        nn.SyncBatchNorm.convert_sync_batchnorm(self.t_projector_k)
+        # for MLM
+        t_config.hidden_size = cross_config.temporal_hidden_size
+        t_config.vocab_size = self.tokenizer.vocab_size
+        self.cls = BertLMPredictionHead(t_config)
+        ################## visual_encoder
+        self.visual_encoder = VisualEncoder(self.task_config, cross_config)
+        self.visual_encoder_k = VisualEncoder(self.task_config, cross_config)
+        self.v_projector = MLP(num_layers=cross_config.proj_num_layers)
+        self.v_projector_k = MLP(num_layers=cross_config.proj_num_layers)
+        self.v_predictor = MLP(num_layers=cross_config.pred_num_layers)
+        nn.SyncBatchNorm.convert_sync_batchnorm(self.v_projector)
+        nn.SyncBatchNorm.convert_sync_batchnorm(self.v_projector_k)
+        nn.SyncBatchNorm.convert_sync_batchnorm(self.v_predictor)
+        ################# momemtun mdoel pairs
+        self.model_pairs = [[self.visual_encoder, self.visual_encoder_k],
+                            [self.text_encoder, self.text_encoder_k],
+                            [self.v_projector, self.v_projector_k],
+                            [self.t_projector, self.t_projector_k],
+                            ]
+        self.copy_params()
+        ################## create queue
+        self.register_buffer("queue_v_cross_ng", torch.randn(cross_config.temporal_hidden_size, self.contrast_num_negative))
+        self.register_buffer("queue_frame_proj_ng", torch.randn(cross_config.temporal_hidden_size,
+                                                                self.contrast_num_negative * self.task_config.max_frames))
+        self.register_buffer("queue_frame_cross_ng", torch.randn(cross_config.temporal_hidden_size,
+                                                                 self.contrast_num_negative * self.task_config.max_frames))
+        self.register_buffer("queue_title_cross_ng", torch.randn(cross_config.temporal_hidden_size, self.contrast_num_negative))
+        self.register_buffer("queue_tag_cross_ng", torch.randn(cross_config.temporal_hidden_size, self.contrast_num_negative))
+        self.queue_v_cross_ng = F.normalize(self.queue_v_cross_ng, dim=0)
+        self.queue_frame_proj_ng = F.normalize(self.queue_frame_proj_ng, dim=0)
+        self.queue_frame_cross_ng = F.normalize(self.queue_frame_cross_ng, dim=0)
+        self.queue_title_cross_ng = F.normalize(self.queue_title_cross_ng, dim=0)
+        self.queue_tag_cross_ng = F.normalize(self.queue_tag_cross_ng, dim=0)
+
+        self.register_buffer("queue_ptr", torch.zeros(1, dtype=torch.long))
+
+        ################## loss function
+        self.loss_fct = CrossEn()
+        self.loss_fct_dual = Dual_CrossEn()
+
+
+        # self.apply(self.init_weights)
+
+    def get_mlm_loss(self, input_ids, input_mask):
+        to_mask_input_ids = input_ids.clone()
+        input_labels = to_mask_input_ids.clone()
+        input_probability_matrix = torch.full(input_labels.shape, self.mlm_probability)
+        masked_input_ids, input_labels = self.mask(to_mask_input_ids, self.tokenizer.vocab_size,
+                                                   input_mask.device, targets=input_labels,
+                                                   probability_matrix=input_probability_matrix)
+        masked_input_output = self.text_encoder(masked_input_ids, input_mask, return_hidden=True)
+        mlm_input_loss = self.calculate_mlm_loss(masked_input_output, input_labels)
+        return mlm_input_loss
+
+    def calculate_mlm_loss(self, sequence_output_mlm, labels):
+
+        mlm_scores = self.cls(sequence_output_mlm)
+        # logger.info("sequence_output_mlm.shape:{}".format(sequence_output_mlm.shape))
+        # logger.info("mlm_scores.shape:{}".format(mlm_scores.shape))
+        # logger.info("labels.shape:{}".format(labels.shape))
+        mlm_loss = F.cross_entropy(mlm_scores.view(-1, self.tokenizer.vocab_size),
+                                   labels.view(-1), ignore_index=-100)
+        return mlm_loss
+
+    def mask(self, input_ids, vocab_size, device, targets=None, masked_indices=None, probability_matrix=None):
+        if masked_indices is None:
+            masked_indices = torch.bernoulli(probability_matrix).bool()
+
+        masked_indices[input_ids == self.tokenizer.pad_token_id] = False
+        masked_indices[input_ids == self.tokenizer.cls_token_id] = False
+        # logger.info("masked_indices:{}".format(masked_indices))
+        # logger.info("masked_indices.shape:{}".format(masked_indices.shape))
+        if targets is not None:
+            targets[~masked_indices] = -100  # We only compute loss on masked tokens
+
+        # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])
+        indices_replaced = torch.bernoulli(torch.full(input_ids.shape, 0.8)).bool() & masked_indices
+        input_ids[indices_replaced] = self.tokenizer.mask_token_id
+
+        # 10% of the time, we replace masked input tokens with random word
+        indices_random = torch.bernoulli(torch.full(input_ids.shape, 0.5)).bool() & masked_indices & ~indices_replaced
+        random_words = torch.randint(vocab_size, input_ids.shape, dtype=torch.long).to(device)
+        input_ids[indices_random] = random_words[indices_random]
+        # The rest of the time (10% of the time) we keep the masked input tokens unchanged
+
+        if targets is not None:
+            return input_ids, targets
+        else:
+            return input_ids
+
+    def loose_similarity(self, sequence_output, visual_output):
+        sequence_output, visual_output = sequence_output.contiguous(), visual_output.contiguous()
+
+        visual_output = visual_output.squeeze()
+        visual_output = visual_output / visual_output.norm(dim=-1, keepdim=True)
+
+        sequence_output = sequence_output.squeeze()
+        sequence_output = sequence_output / sequence_output.norm(dim=-1, keepdim=True)
+
+        logit_scale = self.text_encoder.logit_scale.exp()
+        logit_scale.data = torch.clamp(logit_scale.data, max=100)
+        # if self.rank == 0:
+        #     logger.info("logit_scale:{},dtype:{}".format(logit_scale, logit_scale.dtype))
+        #     logger.info("sequence_output.shape:{}".format(sequence_output.shape))
+        #     logger.info("visual_output.shape:{}".format(visual_output.shape))
+        if len(visual_output.shape) == 2:
+            retrieve_logits = logit_scale * torch.matmul(sequence_output, visual_output.t())
+        else:
+            visual_temp = visual_output.permute(0, 2, 1)
+            retrieve_logits = logit_scale * torch.matmul(sequence_output, visual_temp)
+            if len(retrieve_logits.shape) == 3:
+                retrieve_logits = retrieve_logits.permute(1, 0, 2)
+
+        return retrieve_logits
+
+    @torch.no_grad()
+    def copy_params(self):
+        for model_pair in self.model_pairs:
+            for param, param_k in zip(model_pair[0].parameters(), model_pair[1].parameters()):
+                param_k.data.copy_(param.data)  # initialize
+                param_k.requires_grad = False  # not update by gradient
+
+    @torch.no_grad()
+    def _momentum_update(self):
+        for model_pair in self.model_pairs:
+            for param, param_k in zip(model_pair[0].parameters(), model_pair[1].parameters()):
+                param_k.data = param_k.data * self.contrast_momentum + param.data * (1. - self.contrast_momentum)
+
+    @torch.no_grad()
+    def _dequeue_and_enqueue(self, v_fea_k, tag_fea_k, title_fea_k, frame_fea_k, frame_proj_k):
+
+        # gather keys before updating queue
+        # [bs,hidden]
+        v_fea_k = F.normalize(v_fea_k, dim=1)
+        tag_fea_k = F.normalize(tag_fea_k, dim=1)
+        title_fea_k = F.normalize(title_fea_k, dim=1)
+        # [bs,frame,hidden]
+        frame_fea_k = F.normalize(frame_fea_k, dim=2)
+        frame_proj_k = F.normalize(frame_proj_k, dim=2)
+
+        batch_size = v_fea_k.size(0)
+        frame_num = frame_fea_k.size(1)
+        frame_fea_k = frame_fea_k.view(-1, frame_fea_k.size(-1))
+        frame_proj_k = frame_proj_k.view(-1, frame_proj_k.size(-1))
+
+        ptr = int(self.queue_ptr)
+        # if self.rank == 0:
+        #     logger.info(
+        #         "begin>>>>: ptr:{},batch_size:{},frame_num:{},queue_size:{}".format(ptr, batch_size, frame_num, self.contrast_num_negative))
+        #     logger.info("v1_self_k.shape:{},tag_cross_k.shape:{},frame_proj_k.shape:{}".format(v_fea_k.shape, tag_fea_k.shape, frame_proj_k.shape))
+
+        # replace the keys at ptr (dequeue and enqueue)
+        self.queue_v_cross_ng[:, ptr:ptr + batch_size] = v_fea_k.T
+        self.queue_tag_cross_ng[:, ptr:ptr + batch_size] = tag_fea_k.T
+        self.queue_title_cross_ng[:, ptr:ptr + batch_size] = title_fea_k.T
+
+        self.queue_frame_proj_ng[:, ptr * frame_num:(ptr + batch_size) * frame_num] = frame_proj_k.T
+        self.queue_frame_cross_ng[:, ptr * frame_num:(ptr + batch_size) * frame_num] = frame_fea_k.T
+        # move pointer
+        ptr = (ptr + batch_size) % self.contrast_num_negative
+
+        # if self.rank == 0:
+        #     logger.info("end>>>>: ptr:{}".format(ptr))
+        self.queue_ptr[0] = ptr
+
+    def contrastive_loss(self, q, k, queue):
+
+        q = q.squeeze()
+        q = F.normalize(q, dim=1)
+        k = k.squeeze()
+        k = F.normalize(k, dim=1)
+
+        bs = q.size(0)
+        # logger.info("q.dtype:{},k.dtype:{}".format(q.dtype, k.dtype))
+        # positive logits: Nx1
+        # >>>>>>got error in apex:amp level=01!!!!!!!!!
+        # l_pos = torch.einsum('nc,nc->n', [q, k]).unsqueeze(-1)
+        l_pos = torch.matmul(q, k.T)
+        l_pos = torch.diag(l_pos).reshape([bs, -1])
+        # negative logits: NxK
+        # l_neg = torch.einsum('nc,ck->nk', [q, queue.clone().detach()])
+        l_neg = torch.matmul(q, queue.clone().detach())
+        # logits: Nx(1+K)
+        logits = torch.cat([l_pos, l_neg], dim=1)
+        # if self.rank == 0:
+        #     logger.info("logits.shape:{}".format(logits.shape))
+        # apply temperature
+        logits /= self.contrast_temperature
+
+        # labels: positive key indicators
+        labels = torch.zeros(logits.shape[0], dtype=torch.long).cuda()
+
+        return F.cross_entropy(logits, labels)
+
+    def frame_self_loss(self, frame_fea, frame_fea_k, queue_frame_ng):
+        loss = 0.
+
+        for i in range(frame_fea.size(1) - 1):
+            frame_loss = self.contrastive_loss(frame_fea[:, i, :], frame_fea_k[:, i+1, :], queue_frame_ng) \
+                        + self.contrastive_loss(frame_fea[:, i+1, :], frame_fea_k[:, i, :], queue_frame_ng)
+            loss += frame_loss
+        loss = loss / (frame_fea.size(1) - 1)
+        return loss
+
+    def frame_cross_loss(self, frame_fea, frame_fea_k, queue_frame_ng, text_fea, text_fea_k, queue_text_ng):
+        loss = 0.
+        for i in range(frame_fea.size(1)):
+            frame_loss = self.contrastive_loss(text_fea, frame_fea_k[:, i, :], queue_frame_ng) + \
+                         self.contrastive_loss(frame_fea[:, i, :], text_fea_k, queue_text_ng)
+            loss += frame_loss
+        loss = loss / frame_fea.size(1)
+        return loss
+
+    def forward(self, video_data, video_frame, tag_ids, tag_mask, title_ids, title_mask, global_step):
+        tag_ids = tag_ids.view(-1, tag_ids.shape[-1])
+        tag_mask = tag_mask.view(-1, tag_mask.shape[-1])
+        title_ids = title_ids.view(-1, title_ids.shape[-1])
+        title_mask = title_mask.view(-1, title_mask.shape[-1])
+        # bs x frames x 3 x H x W
+        video = torch.as_tensor(video_data)
+
+        if self.rank == 0 and global_step % self.task_config.n_display == 0:
+            logger.info("video1.shape:{}, dtype:{}, device:{}".format(video.shape, video.dtype, video.device))
+
+        if self.training:
+            # loss = 0.0
+            v_fea, frame_fea = self.visual_encoder(video, video_frame)
+            if self.task_config.dataset == "bird":
+                tag_fea = self.text_encoder(tag_ids, tag_mask)
+            title_fea = self.text_encoder(title_ids, title_mask)
+
+            # for video self supervised learning
+            # [bs,hidden_size]
+            bs, frame, hidden = frame_fea.shape
+            frame_fea = frame_fea.view(-1, hidden)
+            frame_proj = self.v_projector(frame_fea)
+            frame_pred = self.v_predictor(frame_proj)
+            frame_fea = frame_fea.view(bs, frame, hidden)
+            frame_proj = frame_proj.view(bs, frame, hidden)
+            frame_pred = frame_pred.view(bs, frame, hidden)
+            if self.rank == 0 and global_step % self.task_config.n_display == 0:
+                logger.info("v_fea.shape:{},device:{}".format(v_fea.shape, v_fea.device))
+                logger.info("frame_fea.shape:{},device:{}".format(frame_fea.shape, frame_fea.device))
+                logger.info("frame_proj.shape:{},device:{}".format(frame_proj.shape, frame_proj.device))
+                logger.info("title_fea.shape:{}".format(title_fea.shape))
+                logger.info("queue_v_cross_ng.shape:{}".format(self.queue_v_cross_ng.shape))
+            # compute key features
+            with torch.no_grad():  # no gradient to keys
+                self._momentum_update()  # update the key encoder
+
+                tag_fea_k = self.text_encoder_k(tag_ids, tag_mask)
+                title_fea_k = self.text_encoder_k(title_ids, title_mask)
+                #
+                v_fea_k, frame_fea_k = self.visual_encoder_k(video, video_frame)
+                frame_fea_k = frame_fea_k.view(-1, hidden)
+                frame_proj_k = self.v_projector_k(frame_fea_k)
+                frame_fea_k = frame_fea_k.view(bs, frame, hidden)
+                frame_proj_k = frame_proj_k.view(bs, frame, hidden)
+
+            # compute loss
+            if self.rank == 0 and global_step % self.task_config.n_display == 0:
+                logger.info(
+                    "dtype: v_fea:{},v_fea_k:{},title_fea:{}".format(v_fea.dtype, v_fea_k.dtype, title_fea.dtype))
+            # single video modality: video queue loss
+            loss_FAM = self.frame_self_loss(frame_pred, frame_proj_k, self.queue_frame_proj_ng)
+            # cross modality: cross queue loss
+            v_title_queue_loss = self.contrastive_loss(v_fea, title_fea_k, self.queue_title_cross_ng) \
+                                 + self.contrastive_loss(title_fea, v_fea_k, self.queue_v_cross_ng)
+            if self.task_config.dataset == "bird":
+                v_tag_queue_loss = self.contrastive_loss(v_fea, tag_fea_k, self.queue_tag_cross_ng) \
+                                   + self.contrastive_loss(tag_fea, v_fea_k, self.queue_v_cross_ng)
+                loss_VTM = (v_tag_queue_loss + v_title_queue_loss) / 2
+            else:
+                loss_VTM = v_title_queue_loss
+
+            loss_FTM = 0.
+            if self.task_config.use_frame_fea:
+                frame_title_loss = self.frame_cross_loss(frame_fea, frame_fea_k, self.queue_frame_cross_ng, title_fea,
+                                                         title_fea_k, self.queue_title_cross_ng)
+                if self.task_config.dataset == "bird":
+                    frame_tag_loss = self.frame_cross_loss(frame_fea, frame_fea_k, self.queue_frame_cross_ng, tag_fea,
+                                                           tag_fea_k, self.queue_tag_cross_ng)
+                    loss_FTM += (frame_tag_loss + frame_title_loss) / 2
+                else:
+                    loss_FTM = frame_title_loss
+
+            # single text modality: text queue loss
+            # t_queue_loss = self.contrastive_loss(title_fea, tag_fea_k, self.queue_tag_cross_ng) \
+            #                + self.contrastive_loss(tag_fea, title_fea_k, self.queue_v_cross_ng)
+
+            # dequeue_and_enqueue
+            self._dequeue_and_enqueue(v_fea_k, tag_fea_k, title_fea_k, frame_fea_k, frame_proj_k)
+
+            # mlm loss
+
+            mlm_title_loss = self.get_mlm_loss(title_ids, title_mask)
+            if self.task_config.dataset == "bird":
+                mlm_tag_loss = self.get_mlm_loss(tag_ids, tag_mask)
+                loss_MLM = (mlm_tag_loss + mlm_title_loss) / 2
+            else:
+                loss_MLM = mlm_title_loss
+
+            # total loss
+            loss = self.weight_FAM * loss_FAM + self.weight_VTM * loss_VTM + self.weight_FTM * loss_FTM + self.weight_MLM * loss_MLM
+            if self.rank == 0:
+                if global_step % self.task_config.n_display == 0:
+                    logger.info("loss:{},loss_FAM:{},loss_VTM:{},loss_FTM:{},loss_MLM:{}"
+                                "".format(loss, loss_FAM, loss_VTM, loss_FTM, loss_MLM))
+                if self.task_config.logdir:
+                    loss_item = {"loss": float(loss), "loss_FAM": float(loss_FAM), "loss_VTM": float(loss_VTM),
+                                 "loss_FTM": float(loss_FTM), "loss_MLM": float(loss_MLM)}
+                    self.task_config.writer.add_scalars('loss', loss_item, global_step=global_step)
+                    # self.task_config.writer.add_scalar('loss', video_cross_loss, global_step=global_step)
+            return loss
+        else:
+            return None
+
+
+class BirdModel(BirdPreTrainedModel):
+    def __init__(self, cross_config, task_config):
+        super(BirdPreTrainedModel, self).__init__(cross_config)
+        self.task_config = task_config
+        self.rank = task_config.local_rank
+        # self.weight_sim = torch.nn.Parameter(torch.tensor([0.9], dtype=torch.float32), requires_grad=True)
+        self.weight_VTM_finetune = cross_config.weight_VTM_finetune
+        self.weight_FTM_finetune = cross_config.weight_FTM_finetune
+        self.top_frames = task_config.top_frames
+        ################## text Encoder
+        self.text_encoder = TextEncoder(self.task_config, cross_config)
+        ################## visual_encoder
+        self.visual_encoder = VisualEncoder(self.task_config, cross_config)
+        ################## loss function
+        self.loss_fct = CrossEn()
+        self.loss_fct_dual = Dual_CrossEn()
+
+    def frame_loss(self, query_output, frame_output):
+        frame_num = frame_output.size(1)
+        loss = 0.
+        for i in range(frame_num):
+            frame_single = frame_output[:, i, :].squeeze()
+            sim_matrix = self.loose_similarity(query_output, frame_single)
+            sim_loss = self.loss_fct(sim_matrix) + self.loss_fct(sim_matrix.T)
+            loss += sim_loss / frame_num
+        # logger.info("frame_output.shape:{},dtype:{}".format(frame_output.shape, frame_output.dtype))
+        # logger.info("query_output.shape:{},dtype:{}".format(query_output.shape, frame_output.dtype))
+        # sim_matrix = self.loose_similarity(query_output, frame_output)
+        # sim_matrix = torch.topk(sim_matrix, k=self.top_frames, dim=2)[0]
+        # sim_matrix = torch.mean(sim_matrix, dim=2)
+        # sim_loss = self.loss_fct(sim_matrix) + self.loss_fct(sim_matrix.T)
+        # loss += sim_loss
+        return loss
+
+    def forward(self, query_ids, query_mask, video_data, video_frame, idx, global_step):
+        query_ids = query_ids.view(-1, query_ids.shape[-1])
+        query_mask = query_mask.view(-1, query_mask.shape[-1])
+        # T x 3 x H x W
+        video = torch.as_tensor(video_data)
+        # if self.rank == 0:
+        #     logger.info("video.shape:{}, dtype:{}".format(video.shape, video.dtype))
+        if self.training:
+            loss = 0.0
+            query_output = self.text_encoder(query_ids, query_mask)
+            visual_output, frame_output = self.visual_encoder(video, video_frame)
+            # if self.rank == 0:
+            #     logger.info("query_output.shape:{},dtype:{}".format(query_output.shape, query_output.dtype))
+            #     logger.info("visual_output.shape:{},dtype:{}".format(visual_output.shape, visual_output.dtype))
+            #     logger.info("frame_output.shape:{},dtype:{}".format(frame_output.shape, frame_output.dtype))
+
+            # frame loss
+            if self.task_config.use_frame_fea:
+                frame_loss = self.frame_loss(query_output, frame_output)
+                loss += self.weight_FTM_finetune * frame_loss
+            # video loss
+            sim_matrix = self.loose_similarity(query_output, visual_output)
+            sim_loss = self.loss_fct(sim_matrix) + self.loss_fct(sim_matrix.T)
+            loss += self.weight_VTM_finetune * sim_loss
+            # loss += sim_loss
+
+            if self.task_config.local_rank == 0:
+                if global_step % self.task_config.n_display == 0:
+                    logger.info(
+                        "loss:{},frame_loss:{},sim_loss:{},type:{},sim_matrix.shape:{}".format(loss, loss - sim_loss,
+                                                                                sim_loss, sim_loss.dtype, sim_matrix.shape))
+
+                if self.task_config.logdir:
+                    self.task_config.writer.add_scalar('loss', float(loss), global_step=global_step)
+            return loss
+        else:
+            return None
+
+
+class MLP(nn.Module):
+    def __init__(self, in_dim=512, inner_dim=4096, out_dim=512, num_layers=2):
+        super(MLP, self).__init__()
+
+        # hidden layers
+        linear_hidden = [nn.Identity()]
+        for i in range(num_layers - 1):
+            linear_hidden.append(nn.Linear(in_dim if i == 0 else inner_dim, inner_dim))
+            linear_hidden.append(nn.BatchNorm1d(inner_dim))
+            linear_hidden.append(nn.ReLU(inplace=True))
+        self.linear_hidden = nn.Sequential(*linear_hidden)
+
+        self.linear_out = nn.Linear(in_dim if num_layers == 1 else inner_dim,
+                                    out_dim) if num_layers >= 1 else nn.Identity()
+
+    def forward(self, x):
+        x = self.linear_hidden(x)
+        x = self.linear_out(x)
+
+        return x

modules/module_clip.py

@@ -0,0 +1,579 @@
+"""
+Adapted from: https://github.com/openai/CLIP/blob/main/clip/clip.py
+"""
+from collections import OrderedDict
+from typing import Tuple, Union
+
+import hashlib
+import os
+import urllib
+import warnings
+from tqdm import tqdm
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+import logging
+logger = logging.getLogger(__name__)
+
+_MODELS = {
+    "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt",
+    "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt",
+    "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt",
+    "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt",
+    "RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt",
+    "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
+    "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt",
+    "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt",
+}
+
+def _download(url: str, root: str = os.path.expanduser("~/.cache/visual_encoder")):
+    os.makedirs(root, exist_ok=True)
+    filename = os.path.basename(url)
+
+    expected_sha256 = url.split("/")[-2]
+    download_target = os.path.join(root, filename)
+
+    if os.path.exists(download_target) and not os.path.isfile(download_target):
+        raise RuntimeError(f"{download_target} exists and is not a regular file")
+
+    if os.path.isfile(download_target):
+        if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
+            return download_target
+        else:
+            warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")
+
+    with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
+        with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True) as loop:
+            while True:
+                buffer = source.read(8192)
+                if not buffer:
+                    break
+
+                output.write(buffer)
+                loop.update(len(buffer))
+
+    if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
+        raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match")
+
+    return download_target
+
+def available_models():
+    """Returns the names of available CLIP models"""
+    return list(_MODELS.keys())
+
+# =============================
+
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1):
+        super().__init__()
+
+        # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
+        self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+
+        self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+
+        self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
+
+        self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = None
+        self.stride = stride
+
+        if stride > 1 or inplanes != planes * Bottleneck.expansion:
+            # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
+            self.downsample = nn.Sequential(OrderedDict([
+                ("-1", nn.AvgPool2d(stride)),
+                ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),
+                ("1", nn.BatchNorm2d(planes * self.expansion))
+            ]))
+
+    def forward(self, x: torch.Tensor):
+        identity = x
+
+        out = self.relu(self.bn1(self.conv1(x)))
+        out = self.relu(self.bn2(self.conv2(out)))
+        out = self.avgpool(out)
+        out = self.bn3(self.conv3(out))
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu(out)
+        return out
+
+
+class AttentionPool2d(nn.Module):
+    def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
+        super().__init__()
+        self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
+        # print("spacial_dim:{},embed_dim:{}".format(spacial_dim, embed_dim))
+        # print("self.positional_embedding.shape:{}".format(self.positional_embedding.shape))
+        # print("self.positional_embedding[:, None, :].shape:{}".format(self.positional_embedding[:, None, :].shape))
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
+        self.num_heads = num_heads
+
+    def forward(self, x):
+        # logger.info("x1.shape:{}".format(x.shape))
+        x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute(2, 0, 1)  # NCHW -> (HW)NC
+        x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0)  # (HW+1)NC
+        x = x + self.positional_embedding[:, None, :].to(x.dtype)  # (HW+1)NC
+        # logger.info("x2.shape:{}".format(x.shape))
+        x, _ = F.multi_head_attention_forward(
+            query=x, key=x, value=x,
+            embed_dim_to_check=x.shape[-1],
+            num_heads=self.num_heads,
+            q_proj_weight=self.q_proj.weight,
+            k_proj_weight=self.k_proj.weight,
+            v_proj_weight=self.v_proj.weight,
+            in_proj_weight=None,
+            in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
+            bias_k=None,
+            bias_v=None,
+            add_zero_attn=False,
+            dropout_p=0,
+            out_proj_weight=self.c_proj.weight,
+            out_proj_bias=self.c_proj.bias,
+            use_separate_proj_weight=True,
+            training=self.training,
+            need_weights=False
+        )
+        # logger.info("x3.shape:{}".format(x.shape))
+        return x[0]
+        # return x
+
+
+class ModifiedResNet(nn.Module):
+    """
+    A ResNet class that is similar to torchvision's but contains the following changes:
+    - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
+    - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
+    - The final pooling layer is a QKV attention instead of an average pool
+    """
+
+    def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):
+        super().__init__()
+        self.output_dim = output_dim
+        self.input_resolution = input_resolution
+
+        # the 3-layer stem
+        self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(width // 2)
+        self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(width // 2)
+        self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(width)
+        self.avgpool = nn.AvgPool2d(2)
+        self.relu = nn.ReLU(inplace=True)
+
+        # residual layers
+        self._inplanes = width  # this is a *mutable* variable used during construction
+        self.layer1 = self._make_layer(width, layers[0])
+        self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
+        self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
+        self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
+
+        embed_dim = width * 32  # the ResNet feature dimension
+        self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)
+
+    def _make_layer(self, planes, blocks, stride=1):
+        layers = [Bottleneck(self._inplanes, planes, stride)]
+
+        self._inplanes = planes * Bottleneck.expansion
+        for _ in range(1, blocks):
+            layers.append(Bottleneck(self._inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        def stem(x):
+            for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), (self.conv3, self.bn3)]:
+                x = self.relu(bn(conv(x)))
+            x = self.avgpool(x)
+            return x
+
+        x = x.type(self.conv1.weight.dtype)
+        x = stem(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        x = self.attnpool(x)
+
+        return x
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+class QuickGELU(nn.Module):
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, n_head: int, attn_mask=None):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, d_model * 4)),
+            ("gelu", QuickGELU()),
+            ("c_proj", nn.Linear(d_model * 4, d_model))
+        ]))
+        self.ln_2 = LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+    def attention(self, x: torch.Tensor):
+        attn_mask_ = self.attn_mask
+        if self.attn_mask is not None and hasattr(self.attn_mask, '__call__'):
+            attn_mask_ = self.attn_mask(x.size(0))   # LND
+
+        attn_mask_ = attn_mask_.to(dtype=x.dtype, device=x.device) if attn_mask_ is not None else None
+        return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0]
+
+    def forward(self, x_tuple:tuple):
+        x, video_frame = x_tuple
+        x = x + self.attention(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return (x, video_frame)
+
+
+class Transformer(nn.Module):
+    def __init__(self, width: int, layers: int, heads: int, attn_mask = None):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)])
+
+    def forward(self, x: torch.Tensor, video_frame=-1):
+        return self.resblocks((x, video_frame))[0]
+
+
+class VisualTransformer(nn.Module):
+    def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int,
+                 linear_patch: str = '2d',):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.output_dim = output_dim
+
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
+
+        scale = width ** -0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
+        self.ln_pre = LayerNorm(width)
+
+        self.transformer = Transformer(width, layers, heads)
+
+        self.ln_post = LayerNorm(width)
+        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
+
+        # For 3D
+        assert linear_patch in ['2d', '3d']
+        self.linear_patch = linear_patch
+        if self.linear_patch == '3d':
+            self.conv2 = nn.Conv3d(in_channels=3, out_channels=width, kernel_size=(3, patch_size, patch_size),
+                                   stride=(1, patch_size, patch_size), padding=(1, 0, 0), bias=False)
+
+    def forward(self, x: torch.Tensor, video_frame=-1):
+        # logger.info("x.shape:{}".format(x.shape))
+        if self.linear_patch == '3d':
+            assert video_frame != -1
+            x_3d = x.reshape(-1, video_frame, x.shape[-3], x.shape[-2], x.shape[-1])
+            x_3d = x_3d.permute(0, 2, 1, 3, 4)
+            x_3d = self.conv2(x_3d)     # shape = [*, width, frame, grid, grid]
+            x_3d = x_3d.permute(0, 2, 1, 3, 4)      # shape = [*, frame, width, grid, grid]
+            x = x_3d.reshape(-1, x_3d.shape[-3], x_3d.shape[-2], x_3d.shape[-1]).contiguous() # shape = [*, width, grid, grid]
+        else:
+            x = self.conv1(x)  # shape = [*, width, grid, grid]
+        # logger.info("x conv1.shape:{}".format(x.shape))
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+        x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        # logger.info("x ln_pre.shape:{}".format(x.shape))
+        x = self.transformer(x, video_frame=video_frame)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+        # logger.info("x transformer.shape:{}".format(x.shape))
+
+        # Move the three lines below to `encode_image` for entire hidden sequence
+        # x = self.ln_post(x[:, 0, :])
+        # if self.proj is not None:
+        #     x = x @ self.proj
+
+        return x
+
+
+class CLIP(nn.Module):
+    def __init__(self,
+                 embed_dim: int,
+                 # vision
+                 image_resolution: int,
+                 vision_layers: Union[Tuple[int, int, int, int], int],
+                 vision_width: int,
+                 vision_patch_size: int,
+                 # text
+                 context_length: int,
+                 vocab_size: int,
+                 transformer_width: int,
+                 transformer_heads: int,
+                 transformer_layers: int,
+                 # vision linear of patch
+                 linear_patch: str = '2d',
+                 ):
+        super().__init__()
+
+        self.context_length = context_length
+
+        if isinstance(vision_layers, (tuple, list)):
+            vision_heads = vision_width * 32 // 64
+            self.vit = False
+            self.visual = ModifiedResNet(
+                layers=vision_layers,
+                output_dim=embed_dim,
+                heads=vision_heads,
+                input_resolution=image_resolution,
+                width=vision_width
+            )
+        else:
+            self.vit = True
+            vision_heads = vision_width // 64
+            self.visual = VisualTransformer(
+                input_resolution=image_resolution,
+                patch_size=vision_patch_size,
+                width=vision_width,
+                layers=vision_layers,
+                heads=vision_heads,
+                output_dim=embed_dim,
+                linear_patch=linear_patch
+            )
+
+        self.transformer = Transformer(
+            width=transformer_width,
+            layers=transformer_layers,
+            heads=transformer_heads,
+            attn_mask=self.build_attention_mask
+        )
+
+        self.vocab_size = vocab_size
+        self.token_embedding = nn.Embedding(vocab_size, transformer_width)
+        self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))
+        self.ln_final = LayerNorm(transformer_width)
+
+        self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
+        self.logit_scale = nn.Parameter(torch.ones([]))
+
+        self.initialize_parameters()
+
+    def initialize_parameters(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+
+        if isinstance(self.visual, ModifiedResNet):
+            if self.visual.attnpool is not None:
+                std = self.visual.attnpool.c_proj.in_features ** -0.5
+                nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
+
+            for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:
+                for name, param in resnet_block.named_parameters():
+                    if name.endswith("bn3.weight"):
+                        nn.init.zeros_(param)
+
+        proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        attn_std = self.transformer.width ** -0.5
+        fc_std = (2 * self.transformer.width) ** -0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
+
+    @staticmethod
+    def get_config(pretrained_clip_name="ViT-B/32"):
+        # logger.info("pretrained_clip_name:{}".format(pretrained_clip_name))
+        model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ViT-B-32.pt")
+        if pretrained_clip_name == "ViT-B/32" and os.path.exists(model_path):
+            pass
+        else:
+            if pretrained_clip_name in _MODELS:
+                model_path = _download(_MODELS[pretrained_clip_name])
+            elif os.path.isfile(pretrained_clip_name):
+                model_path = pretrained_clip_name
+            else:
+                raise RuntimeError(f"Model {pretrained_clip_name} not found; available models = {available_models()}")
+
+        try:
+            # loading JIT archive
+            model = torch.jit.load(model_path, map_location="cpu").eval()
+            state_dict = model.state_dict()
+        except RuntimeError:
+            state_dict = torch.load(model_path, map_location="cpu")
+
+        return state_dict
+
+    def build_attention_mask(self, context_length):
+        # lazily create causal attention mask, with full attention between the vision tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.zeros(context_length, context_length)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    @property
+    def dtype(self):
+        return self.visual.conv1.weight.dtype
+
+    def encode_image(self, image, return_hidden=False, video_frame=-1):
+        if self.vit:
+            # logger.info("image.shape:{}".format(image.shape))
+            hidden = self.visual(image.type(self.dtype), video_frame=video_frame)
+            # logger.info("hidden1.shape:{}".format(hidden.shape))
+            hidden = self.visual.ln_post(hidden) @ self.visual.proj
+            # logger.info("hidden2.shape:{}".format(hidden.shape))
+            x = hidden[:, 0, :]
+            # x = hidden
+        else:
+            hidden = self.visual(image.type(self.dtype))
+            x = hidden
+        if return_hidden:
+            return x, hidden
+
+        return x
+
+    def encode_text(self, text, return_hidden=False):
+        x = self.token_embedding(text).type(self.dtype)  # [batch_size, n_ctx, d_model]
+
+        pos_emd = self.positional_embedding[:x.size(1), :].type(self.dtype)
+        x = x + pos_emd
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        hidden = self.ln_final(x).type(self.dtype) @ self.text_projection
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = hidden[torch.arange(hidden.shape[0]), text.argmax(dim=-1)]
+
+        if return_hidden:
+            return x, hidden
+
+        return x
+
+    def forward(self, image, text):
+        image_features = self.encode_image(image)
+        text_features = self.encode_text(text)
+
+        # normalized features
+        image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+        text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+
+        # cosine similarity as logits
+        logit_scale = self.logit_scale.exp()
+        logits_per_image = logit_scale * image_features @ text_features.t()
+        logits_per_text = logit_scale * text_features @ image_features.t()
+
+        # shape = [global_batch_size, global_batch_size]
+        return logits_per_image, logits_per_text
+
+
+def convert_weights(model: nn.Module):
+    """Convert applicable model parameters to fp16"""
+
+    def _convert_weights_to_fp16(l):
+        if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear)):
+            l.weight.data = l.weight.data.half()
+            if l.bias is not None:
+                l.bias.data = l.bias.data.half()
+
+        if isinstance(l, nn.MultiheadAttention):
+            for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
+                tensor = getattr(l, attr)
+                if tensor is not None:
+                    tensor.data = tensor.data.half()
+
+        for name in ["text_projection", "proj"]:
+            if hasattr(l, name):
+                attr = getattr(l, name)
+                if attr is not None:
+                    attr.data = attr.data.half()
+
+    model.apply(_convert_weights_to_fp16)
+
+
+def build_model(state_dict: dict, local_rank):
+    vit = "visual.proj" in state_dict
+
+    if vit:
+        vision_width = state_dict["visual.conv1.weight"].shape[0]
+        vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
+        vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
+        grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
+        image_resolution = vision_patch_size * grid_size
+    else:
+        counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
+        vision_layers = tuple(counts)
+        vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
+        output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
+        vision_patch_size = None
+        assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
+        image_resolution = output_width * 32
+
+    embed_dim = state_dict["text_projection"].shape[1]
+    context_length = state_dict["positional_embedding"].shape[0]
+    vocab_size = state_dict["token_embedding.weight"].shape[0]
+    transformer_width = state_dict["ln_final.weight"].shape[0]
+    transformer_heads = transformer_width // 64
+    transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith(f"transformer.resblocks")))
+    if local_rank == 0:
+        logger.info("\t embed_dim: {}".format(embed_dim))
+        logger.info("\t image_resolution: {}".format(image_resolution))
+        logger.info("\t vision_layers: {}".format(vision_layers))
+        logger.info("\t vision_width: {}".format(vision_width))
+        logger.info("\t vision_patch_size: {}".format(vision_patch_size))
+        logger.info("\t context_length: {}".format(context_length))
+        logger.info("\t not used vocab_size: {}".format(vocab_size))
+        logger.info("\t transformer_width: {}".format(transformer_width))
+        logger.info("\t transformer_heads: {}".format(transformer_heads))
+        logger.info("\t transformer_layers: {}".format(transformer_layers))
+
+    model = CLIP(
+        embed_dim,
+        image_resolution, vision_layers, vision_width, vision_patch_size,
+        context_length, vocab_size, transformer_width, transformer_heads, transformer_layers
+    ).float()
+
+    for key in ["input_resolution", "context_length", "vocab_size"]:
+        if key in state_dict:
+            del state_dict[key]
+
+    # convert_weights(model) # cpu mode should comment this line
+    model.load_state_dict(state_dict)
+    # return model.eval()
+    return model

modules/module_cross.py

@@ -0,0 +1,356 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import copy
+import json
+import math
+import logging
+import tarfile
+import tempfile
+import shutil
+import sys
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+from .file_utils import cached_path
+from .until_config import PretrainedConfig
+from .until_module import PreTrainedModel, LayerNorm, ACT2FN
+from collections import OrderedDict
+from modules.module_clip import build_model, CLIP, convert_weights
+from transformers import AutoConfig, AutoModel, RobertaModel, RobertaConfig
+
+
+logger = logging.getLogger(__name__)
+
+PRETRAINED_MODEL_ARCHIVE_MAP = {}
+CONFIG_NAME = 'cross_config.json'
+WEIGHTS_NAME = 'cross_pytorch_model.bin'
+
+
+def gelu(x):
+    """Implementation of the gelu activation function.
+        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
+        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+        Also see https://arxiv.org/abs/1606.08415
+    """
+    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
+
+
+def swish(x):
+    return x * torch.sigmoid(x)
+
+ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish}
+
+class CrossConfig(PretrainedConfig):
+    """Configuration class to store the configuration of a `CrossModel`.
+    """
+    pretrained_model_archive_map = PRETRAINED_MODEL_ARCHIVE_MAP
+    config_name = CONFIG_NAME
+    weights_name = WEIGHTS_NAME
+    def __init__(self,
+                 vocab_size_or_config_json_file,
+                 hidden_size=768,
+                 num_hidden_layers=12,
+                 num_attention_heads=12,
+                 intermediate_size=3072,
+                 hidden_act="gelu",
+                 hidden_dropout_prob=0.1,
+                 attention_probs_dropout_prob=0.1,
+                 max_position_embeddings=512,
+                 type_vocab_size=2,
+                 initializer_range=0.02):
+        """Constructs CrossConfig.
+
+        Args:
+            vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CrossModel`.
+            hidden_size: Size of the encoder layers and the pooler layer.
+            num_hidden_layers: Number of hidden layers in the Transformer encoder.
+            num_attention_heads: Number of attention heads for each attention layer in
+                the Transformer encoder.
+            intermediate_size: The size of the "intermediate" (i.e., feed-forward)
+                layer in the Transformer encoder.
+            hidden_act: The non-linear activation function (function or string) in the
+                encoder and pooler. If string, "gelu", "relu" and "swish" are supported.
+            hidden_dropout_prob: The dropout probabilitiy for all fully connected
+                layers in the embeddings, encoder, and pooler.
+            attention_probs_dropout_prob: The dropout ratio for the attention
+                probabilities.
+            max_position_embeddings: The maximum sequence length that this model might
+                ever be used with. Typically set this to something large just in case
+                (e.g., 512 or 1024 or 2048).
+            type_vocab_size: The vocabulary size of the `token_type_ids` passed into
+                `CrossModel`.
+            initializer_range: The sttdev of the truncated_normal_initializer for
+                initializing all weight matrices.
+        """
+        if isinstance(vocab_size_or_config_json_file, str):
+            with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader:
+                json_config = json.loads(reader.read())
+            for key, value in json_config.items():
+                self.__dict__[key] = value
+        elif isinstance(vocab_size_or_config_json_file, int):
+            self.vocab_size = vocab_size_or_config_json_file
+            self.hidden_size = hidden_size
+            self.num_hidden_layers = num_hidden_layers
+            self.num_attention_heads = num_attention_heads
+            self.hidden_act = hidden_act
+            self.intermediate_size = intermediate_size
+            self.hidden_dropout_prob = hidden_dropout_prob
+            self.attention_probs_dropout_prob = attention_probs_dropout_prob
+            self.max_position_embeddings = max_position_embeddings
+            self.type_vocab_size = type_vocab_size
+            self.initializer_range = initializer_range
+        else:
+            raise ValueError("First argument must be either a vocabulary size (int)"
+                             "or the path to a pretrained model config file (str)")
+
+class QuickGELU(nn.Module):
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, n_head: int):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, d_model * 4)),
+            ("gelu", QuickGELU()),
+            ("c_proj", nn.Linear(d_model * 4, d_model))
+        ]))
+        self.ln_2 = LayerNorm(d_model)
+        self.n_head = n_head
+
+    def attention(self, x: torch.Tensor, attn_mask: torch.Tensor):
+        attn_mask_ = attn_mask.repeat(self.n_head, 1, 1)
+        return self.attn(x, x, x, need_weights=False, attn_mask=attn_mask_)[0]
+
+    def forward(self, para_tuple: tuple):
+        # x: torch.Tensor, attn_mask: torch.Tensor
+        # print(para_tuple)
+        x, attn_mask = para_tuple
+        x = x + self.attention(self.ln_1(x), attn_mask)
+        x = x + self.mlp(self.ln_2(x))
+        return (x, attn_mask)
+
+class Transformer(nn.Module):
+    def __init__(self, width: int, layers: int, heads: int):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads) for _ in range(layers)])
+
+    def forward(self, x: torch.Tensor, attn_mask: torch.Tensor):
+        # logger.info("x.shpae:{},attn_mask:{}".format(x.shape, attn_mask.shape))
+        return self.resblocks((x, attn_mask))[0]
+
+
+class VisualEncoder(nn.Module):
+    def __init__(self, task_config, cross_config):
+        super().__init__()
+        pretrained_clip_name = cross_config.pretrained_clip_name
+        if task_config.local_rank == 0:
+            logger.info("pretrained_clip_name:{}".format(pretrained_clip_name))
+        clip_state_dict = CLIP.get_config(pretrained_clip_name=pretrained_clip_name)
+        clip = build_model(clip_state_dict, local_rank=task_config.local_rank)
+        self.use_temp = task_config.use_temp
+        self.is_vit = copy.deepcopy(clip.vit)
+        self.visual = copy.deepcopy(clip.visual)
+
+        if self.use_temp:
+            self.temporal_transformer = Transformer(width=cross_config.temporal_hidden_size,
+                                              layers=cross_config.temporal_hidden_layers,
+                                              heads=cross_config.temporal_attention_heads)
+            self.frame_position_embeddings = nn.Embedding(cross_config.max_position_embeddings,
+                                                      cross_config.temporal_hidden_size)
+
+            # use clip.transformer to initial temporal_transformer
+            # for param_1, param_2 in zip(self.temporal_transformer.parameters(), clip.transformer.parameters()):
+            #     param_1.data.copy_(param_2.data)  # initialize
+            # if task_config.local_rank == 0:
+            #     logger.info("clip.positional_embedding:{}".format(clip.positional_embedding))
+            # self.frame_position_embeddings.weight = copy.deepcopy(clip.positional_embedding)
+
+    def forward(self, video, video_frames):
+        # encode frames
+        bs, frames, channel, h, w = video.shape
+        # [bs*frame, 3, 224, 224]
+        video = video.view(bs * frames, channel, h, w)
+        # logger.info("video_b.shape:{}, dtype:{}".format(video_b.shape, video_b.dtype))
+        # logger.info("video_frame[{}]:{}".format(b, video_frame))
+        visual_hidden = self.encode_image(video, video_frame=frames)
+        # [bs, frame, hidden_size]
+        # logger.info("visual_hidden.shape:{}".format(visual_hidden.shape))
+        visual_hidden = visual_hidden.view(bs, frames, visual_hidden.size(-1))
+        # logger.info("visual_hidden1.shape:{}".format(visual_hidden.shape))
+        # get temporal information
+        visual_hidden_original = visual_hidden
+        frame_output = visual_hidden_original
+        if self.use_temp:
+            seq_length = visual_hidden.size(1)
+            position_ids = torch.arange(seq_length, dtype=torch.long, device=visual_hidden.device)
+            # logger.info("position_ids.shape:{}".format(position_ids.shape))
+            frame_position_embeddings = self.frame_position_embeddings(position_ids)
+            # logger.info("frame_position_embeddings.shape:{}".format(frame_position_embeddings.shape))
+            visual_hidden = visual_hidden + frame_position_embeddings
+
+            video_mask = torch.ones([bs, frames], device=visual_hidden.device)
+            extended_video_mask = (1.0 - video_mask.unsqueeze(1)) * -1000000.0
+            extended_video_mask = extended_video_mask.expand(-1, video_mask.size(1), -1)
+            visual_hidden = visual_hidden.permute(1, 0, 2)  # NLD -> LND
+            visual_hidden = self.temporal_transformer(visual_hidden, extended_video_mask)
+            visual_hidden = visual_hidden.permute(1, 0, 2)  # LND -> NLD
+            visual_hidden = visual_hidden + visual_hidden_original
+
+        # logger.info("visual_hidden.shape:{}".format(visual_hidden.shape))
+        visual_output = visual_hidden / visual_hidden.norm(dim=-1, keepdim=True)
+        # [bs, frames,512] -> [bs, 512]
+        visual_output = torch.mean(visual_output, dim=1)
+        # logger.info("visual_hidden mean.shape:{}".format(visual_hidden.shape))
+
+        # logger.info("visual encoder visual_output.shape:{}".format(visual_output.shape))
+        return visual_output, frame_output
+
+    @property
+    def dtype(self):
+        return self.visual.conv1.weight.dtype
+
+    def encode_image(self, image, return_hidden=False, video_frame=-1):
+        if self.is_vit:
+            # logger.info("image.shape:{}".format(image.shape))
+            # hidden = self.visual(image, video_frame=video_frame)
+            hidden = self.visual(image.type(self.dtype), video_frame=video_frame)
+            # logger.info("hidden1.shape:{}".format(hidden.shape))
+            hidden = self.visual.ln_post(hidden) @ self.visual.proj
+            # logger.info("hidden2.shape:{}".format(hidden.shape))
+            x = hidden[:, 0, :]
+            # x = hidden
+        else:
+            hidden = self.visual(image)
+            x = hidden
+        if return_hidden:
+            return x.float(), hidden.float()
+        return x.float()
+
+
+class TextEncoder(nn.Module):
+    def __init__(self, task_config, cross_config):
+        super().__init__()
+        self.language = task_config.language
+        pretrained_clip_name = cross_config.pretrained_clip_name
+        if task_config.local_rank == 0:
+            logger.info("pretrained_clip_name:{}".format(pretrained_clip_name))
+        clip_state_dict = CLIP.get_config(pretrained_clip_name=pretrained_clip_name)
+        clip = build_model(clip_state_dict, local_rank=task_config.local_rank)
+        self.logit_scale = copy.deepcopy(clip_state_dict["logit_scale"])
+        if self.language == "english":
+            self.token_embedding = copy.deepcopy(clip.token_embedding)
+            self.positional_embedding = copy.deepcopy(clip.positional_embedding)
+            self.transformer = copy.deepcopy(clip.transformer)
+            self.ln_final = copy.deepcopy(clip.ln_final)
+            self.text_projection = copy.deepcopy(clip.text_projection)
+            self.dtype = clip.visual.conv1.weight.dtype
+        elif self.language == "chinese":
+            pretrained = task_config.pretrained_text
+            t_config = AutoConfig.from_pretrained(pretrained)
+            if task_config.rank == 0:
+                logger.info("name:{},chinesebert_config:{}".format(pretrained, t_config))
+            self.chinese_encoder = AutoModel.from_pretrained(pretrained)
+            # logger.info("random Roberta")
+            # self.chinese_encoder = RobertaModel(RobertaConfig())
+            self.text_proj = nn.Linear(cross_config.chinese_hidden_size, cross_config.temporal_hidden_size)
+        else:
+            raise NotImplementedError("wrong language")
+
+    def forward(self, input_ids, attention_mask, return_hidden=False):
+        bs_pair = input_ids.size(0)
+        if self.language == "english":
+            text_output, hidden = self.encode_text(input_ids, return_hidden=True)
+        else:
+            temp_output = self.chinese_encoder(input_ids, attention_mask=attention_mask)
+            # logger.info("hidden:{},text_output:{}".format(temp_output[0].shape, temp_output[1].shape))
+            hidden = self.text_proj(temp_output[0])
+            text_output = self.text_proj(temp_output[1])
+
+        text_output = text_output.view(bs_pair, text_output.size(-1))
+        hidden = hidden.view(bs_pair, -1, hidden.size(-1))
+        if return_hidden:
+            return hidden
+        else:
+            return text_output
+
+    def encode_text(self, text, return_hidden=False):
+        x = self.token_embedding(text).type(self.dtype)  # [batch_size, n_ctx, d_model]
+
+        pos_emd = self.positional_embedding[:x.size(1), :].type(self.dtype)
+        x = x + pos_emd
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        hidden = self.ln_final(x).type(self.dtype) @ self.text_projection
+
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = hidden[torch.arange(hidden.shape[0]), text.argmax(dim=-1)]
+
+        if return_hidden:
+            return x.float(), hidden.float()
+
+        return x.float()
+
+
+class BertLMPredictionHead(nn.Module):
+    def __init__(self, config):
+        super(BertLMPredictionHead, self).__init__()
+        self.transform = BertPredictionHeadTransform(config)
+
+        # The output weights are the same as the input embeddings, but there is
+        # an output-only bias for each token.
+        self.decoder = nn.Linear(config.hidden_size, config.vocab_size,bias=False,)
+        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
+        self.decoder.bias = self.bias
+
+    def forward(self, hidden_states):
+        hidden_states = self.transform(hidden_states)
+        hidden_states = self.decoder(hidden_states)
+        return hidden_states
+
+
+class BertPredictionHeadTransform(nn.Module):
+    def __init__(self, config):
+        super(BertPredictionHeadTransform, self).__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        if isinstance(config.hidden_act, str) or (
+            sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)
+        ):
+            self.transform_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.transform_act_fn = config.hidden_act
+        self.LayerNorm = BertLayerNorm(config.hidden_size, eps=1e-12)
+
+    def forward(self, hidden_states):
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.transform_act_fn(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states)
+        return hidden_states
+
+
+class BertLayerNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-12):
+        """Construct a layernorm module in the TF style (epsilon inside the square root).
+        """
+        super(BertLayerNorm, self).__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.bias = nn.Parameter(torch.zeros(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, x):
+        u = x.mean(-1, keepdim=True)
+        s = (x - u).pow(2).mean(-1, keepdim=True)
+        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
+        return self.weight * x + self.bias

modules/optimization.py

@@ -0,0 +1,168 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch optimization for BERT model."""
+
+import math
+import torch
+from torch.optim import Optimizer
+from torch.optim.optimizer import required
+from torch.nn.utils import clip_grad_norm_
+import logging
+
+logger = logging.getLogger(__name__)
+
+def warmup_cosine(x, warmup=0.002):
+    if x < warmup:
+        return x/warmup
+    return 0.5 * (1.0 + math.cos(math.pi * x))
+
+def warmup_constant(x, warmup=0.002):
+    """ Linearly increases learning rate over `warmup`*`t_total` (as provided to BertAdam) training steps.
+        Learning rate is 1. afterwards. """
+    if x < warmup:
+        return x/warmup
+    return 1.0
+
+def warmup_linear(x, warmup=0.002):
+    """ Specifies a triangular learning rate schedule where peak is reached at `warmup`*`t_total`-th (as provided to BertAdam) training step.
+        After `t_total`-th training step, learning rate is zero. """
+    if x < warmup:
+        return x/warmup
+    return max((x-1.)/(warmup-1.), 0)
+
+SCHEDULES = {
+    'warmup_cosine':   warmup_cosine,
+    'warmup_constant': warmup_constant,
+    'warmup_linear':   warmup_linear,
+}
+
+
+class BertAdam(Optimizer):
+    """Implements BERT version of Adam algorithm with weight decay fix.
+    Params:
+        lr: learning rate
+        warmup: portion of t_total for the warmup, -1  means no warmup. Default: -1
+        t_total: total number of training steps for the learning
+            rate schedule, -1  means constant learning rate. Default: -1
+        schedule: schedule to use for the warmup (see above). Default: 'warmup_linear'
+        b1: Adams b1. Default: 0.9
+        b2: Adams b2. Default: 0.999
+        e: Adams epsilon. Default: 1e-6
+        weight_decay: Weight decay. Default: 0.01
+        max_grad_norm: Maximum norm for the gradients (-1 means no clipping). Default: 1.0
+    """
+    def __init__(self, params, lr=required, warmup=-1, t_total=-1, schedule='warmup_linear',
+                 b1=0.9, b2=0.999, e=1e-6, weight_decay=0.01,
+                 max_grad_norm=1.0):
+        if lr is not required and lr < 0.0:
+            raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr))
+        if schedule not in SCHEDULES:
+            raise ValueError("Invalid schedule parameter: {}".format(schedule))
+        if not 0.0 <= warmup < 1.0 and not warmup == -1:
+            raise ValueError("Invalid warmup: {} - should be in [0.0, 1.0[ or -1".format(warmup))
+        if not 0.0 <= b1 < 1.0:
+            raise ValueError("Invalid b1 parameter: {} - should be in [0.0, 1.0[".format(b1))
+        if not 0.0 <= b2 < 1.0:
+            raise ValueError("Invalid b2 parameter: {} - should be in [0.0, 1.0[".format(b2))
+        if not e >= 0.0:
+            raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(e))
+        defaults = dict(lr=lr, schedule=schedule, warmup=warmup, t_total=t_total,
+                        b1=b1, b2=b2, e=e, weight_decay=weight_decay,
+                        max_grad_norm=max_grad_norm)
+        super(BertAdam, self).__init__(params, defaults)
+
+    def get_lr(self):
+        lr = []
+        for group in self.param_groups:
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+                state = self.state[p]
+                if len(state) == 0:
+                    return [0]
+                if group['t_total'] != -1:
+                    schedule_fct = SCHEDULES[group['schedule']]
+                    lr_scheduled = group['lr'] * schedule_fct(state['step']/group['t_total'], group['warmup'])
+                else:
+                    lr_scheduled = group['lr']
+                lr.append(lr_scheduled)
+        return lr
+
+    def step(self, closure=None):
+        """Performs a single optimization step.
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            loss = closure()
+
+        for group in self.param_groups:
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+                grad = p.grad.data
+                if grad.is_sparse:
+                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+
+                state = self.state[p]
+
+                # State initialization
+                if len(state) == 0:
+                    state['step'] = 0
+                    # Exponential moving average of gradient values
+                    state['next_m'] = torch.zeros_like(p.data)
+                    # Exponential moving average of squared gradient values
+                    state['next_v'] = torch.zeros_like(p.data)
+
+                next_m, next_v = state['next_m'], state['next_v']
+                beta1, beta2 = group['b1'], group['b2']
+
+                # Add grad clipping
+                if group['max_grad_norm'] > 0:
+                    clip_grad_norm_(p, group['max_grad_norm'])
+
+                # Decay the first and second moment running average coefficient
+                # In-place operations to update the averages at the same time
+                # next_m.mul_(beta1).add_(1 - beta1, grad) --> pytorch 1.7
+                next_m.mul_(beta1).add_(grad, alpha=1 - beta1)
+                # next_v.mul_(beta2).addcmul_(1 - beta2, grad, grad) --> pytorch 1.7
+                next_v.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
+                update = next_m / (next_v.sqrt() + group['e'])
+
+                # Just adding the square of the weights to the loss function is *not*
+                # the correct way of using L2 regularization/weight decay with Adam,
+                # since that will interact with the m and v parameters in strange ways.
+                #
+                # Instead we want to decay the weights in a manner that doesn't interact
+                # with the m/v parameters. This is equivalent to adding the square
+                # of the weights to the loss with plain (non-momentum) SGD.
+                if group['weight_decay'] > 0.0:
+                    update += group['weight_decay'] * p.data
+
+                if group['t_total'] != -1:
+                    schedule_fct = SCHEDULES[group['schedule']]
+                    progress = state['step']/group['t_total']
+                    lr_scheduled = group['lr'] * schedule_fct(progress, group['warmup'])
+                else:
+                    lr_scheduled = group['lr']
+
+                update_with_lr = lr_scheduled * update
+                p.data.add_(-update_with_lr)
+
+                state['step'] += 1
+
+        return loss

modules/tokenization_clip.py

@@ -0,0 +1,152 @@
+import gzip
+import html
+import os
+from functools import lru_cache
+
+import ftfy
+import regex as re
+import logging
+logger = logging.getLogger(__name__)
+
+
+@lru_cache()
+def default_bpe():
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+class SimpleTokenizer(object):
+    def __init__(self, bpe_path: str = default_bpe()):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
+        merges = merges[1:49152-256-2+1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v+'</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        vocab.extend(['<|startoftext|>', '<|endoftext|>'])
+        vocab[49394] = '[MASK]'
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>', '[MASK]': '[MASK]'}
+        self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
+
+        self.vocab = self.encoder
+        self.vocab_size = len(vocab)
+        self.pad_token_id = self.encoder['<|endoftext|>']
+        self.cls_token_id = self.encoder["<|startoftext|>"]
+        self.mask_token_id = self.encoder['[MASK]']
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token+'</w>'
+
+        while True:
+            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word)-1 and word[i+1] == second:
+                    new_word.append(first+second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+        return text
+
+    def tokenize(self, text):
+        tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            tokens.extend(bpe_token for bpe_token in self.bpe(token).split(' '))
+        return tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return [self.encoder[bpe_token] for bpe_token in tokens]

modules/until_config.py

@@ -0,0 +1,126 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch BERT model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import copy
+import json
+import logging
+import tarfile
+import tempfile
+import shutil
+import torch
+from .file_utils import cached_path
+
+logger = logging.getLogger(__name__)
+
+class PretrainedConfig(object):
+
+    pretrained_model_archive_map = {}
+    config_name = ""
+    weights_name = ""
+
+    @classmethod
+    def get_config(cls, pretrained_model_name, cache_dir, type_vocab_size, state_dict, task_config=None):
+        archive_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), pretrained_model_name)
+        if os.path.exists(archive_file) is False:
+            if pretrained_model_name in cls.pretrained_model_archive_map:
+                archive_file = cls.pretrained_model_archive_map[pretrained_model_name]
+            else:
+                archive_file = pretrained_model_name
+
+        # redirect to the cache, if necessary
+        try:
+            resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir)
+        except FileNotFoundError:
+            if task_config is None or task_config.local_rank == 0:
+                logger.error(
+                    "Model name '{}' was not found in model name list. "
+                    "We assumed '{}' was a path or url but couldn't find any file "
+                    "associated to this path or url.".format(
+                        pretrained_model_name,
+                        archive_file))
+            return None
+        if resolved_archive_file == archive_file:
+            if task_config is None or task_config.local_rank == 0:
+                logger.info("loading archive file {}".format(archive_file))
+        else:
+            if task_config is None or task_config.local_rank == 0:
+                logger.info("loading archive file {} from cache at {}".format(
+                    archive_file, resolved_archive_file))
+        tempdir = None
+        if os.path.isdir(resolved_archive_file):
+            serialization_dir = resolved_archive_file
+        else:
+            # Extract archive to temp dir
+            tempdir = tempfile.mkdtemp()
+            if task_config is None or task_config.local_rank == 0:
+                logger.info("extracting archive file {} to temp dir {}".format(
+                    resolved_archive_file, tempdir))
+            with tarfile.open(resolved_archive_file, 'r:gz') as archive:
+                archive.extractall(tempdir)
+            serialization_dir = tempdir
+        # Load config
+        config_file = os.path.join(serialization_dir, cls.config_name)
+        config = cls.from_json_file(config_file)
+        config.type_vocab_size = type_vocab_size
+        if task_config is None or task_config.local_rank == 0:
+            logger.info("Model config {}".format(config))
+
+        if state_dict is None:
+            weights_path = os.path.join(serialization_dir, cls.weights_name)
+            if os.path.exists(weights_path):
+                state_dict = torch.load(weights_path, map_location='cpu')
+            else:
+                if task_config is None or task_config.local_rank == 0:
+                    logger.info("Weight doesn't exsits. {}".format(weights_path))
+
+        if tempdir:
+            # Clean up temp dir
+            shutil.rmtree(tempdir)
+
+        return config, state_dict
+
+    @classmethod
+    def from_dict(cls, json_object):
+        """Constructs a `BertConfig` from a Python dictionary of parameters."""
+        config = cls(vocab_size_or_config_json_file=-1)
+        for key, value in json_object.items():
+            config.__dict__[key] = value
+        return config
+
+    @classmethod
+    def from_json_file(cls, json_file):
+        """Constructs a `BertConfig` from a json file of parameters."""
+        with open(json_file, "r", encoding='utf-8') as reader:
+            text = reader.read()
+        return cls.from_dict(json.loads(text))
+
+    def __repr__(self):
+        return str(self.to_json_string())
+
+    def to_dict(self):
+        """Serializes this instance to a Python dictionary."""
+        output = copy.deepcopy(self.__dict__)
+        return output
+
+    def to_json_string(self):
+        """Serializes this instance to a JSON string."""
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"

modules/until_module.py

@@ -0,0 +1,295 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch BERT model."""
+
+import logging
+import numpy as np
+import torch
+from torch import nn
+import torch.nn.functional as F
+import math
+from modules.until_config import PretrainedConfig
+
+logger = logging.getLogger(__name__)
+
+
+def gelu(x):
+    """Implementation of the gelu activation function.
+        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
+        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+    """
+    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
+
+def swish(x):
+    return x * torch.sigmoid(x)
+
+def get_dual_matrix(sim_matrix):
+    if torch.is_tensor(sim_matrix):
+        pass
+    else:
+        sim_matrix = torch.tensor(sim_matrix)
+    temp = 1
+    # sim_matrix = sim_matrix * F.softmax(sim_matrix / temp, dim=0) * len(sim_matrix)
+    alpha = F.softmax(sim_matrix / temp, dim=0)
+    beta = F.softmax(sim_matrix / temp, dim=1)
+    sim_matrix = sim_matrix * alpha * beta
+    return sim_matrix
+
+
+ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish}
+
+class LayerNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-12):
+        """Construct a layernorm module in the TF style (epsilon inside the square root).
+        """
+        super(LayerNorm, self).__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.bias = nn.Parameter(torch.zeros(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, x):
+        u = x.mean(-1, keepdim=True)
+        s = (x - u).pow(2).mean(-1, keepdim=True)
+        x = (x - u) / torch.sqrt(s + self.variance_epsilon)
+        return self.weight * x + self.bias
+
+class PreTrainedModel(nn.Module):
+    """ An abstract class to handle weights initialization and
+        a simple interface for dowloading and loading pretrained models.
+    """
+    def __init__(self, config, *inputs, **kwargs):
+        super(PreTrainedModel, self).__init__()
+        if not isinstance(config, PretrainedConfig):
+            raise ValueError(
+                "Parameter config in `{}(config)` should be an instance of class `PretrainedConfig`. "
+                "To create a model from a Google pretrained model use "
+                "`model = {}.from_pretrained(PRETRAINED_MODEL_NAME)`".format(
+                    self.__class__.__name__, self.__class__.__name__
+                ))
+        self.config = config
+
+    def init_weights(self, module):
+        """ Initialize the weights.
+        """
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+        elif isinstance(module, LayerNorm):
+            if 'beta' in dir(module) and 'gamma' in dir(module):
+                module.beta.data.zero_()
+                module.gamma.data.fill_(1.0)
+            else:
+                module.bias.data.zero_()
+                module.weight.data.fill_(1.0)
+        if isinstance(module, nn.Linear) and module.bias is not None:
+            module.bias.data.zero_()
+
+    def resize_token_embeddings(self, new_num_tokens=None):
+        raise NotImplementedError
+
+    @classmethod
+    def init_preweight(cls, model, state_dict, prefix=None, task_config=None):
+        old_keys = []
+        new_keys = []
+        for key in state_dict.keys():
+            new_key = None
+            if 'gamma' in key:
+                new_key = key.replace('gamma', 'weight')
+            if 'beta' in key:
+                new_key = key.replace('beta', 'bias')
+            if new_key:
+                old_keys.append(key)
+                new_keys.append(new_key)
+        for old_key, new_key in zip(old_keys, new_keys):
+            state_dict[new_key] = state_dict.pop(old_key)
+
+        if prefix is not None:
+            old_keys = []
+            new_keys = []
+            for key in state_dict.keys():
+                old_keys.append(key)
+                new_keys.append(prefix + key)
+            for old_key, new_key in zip(old_keys, new_keys):
+                state_dict[new_key] = state_dict.pop(old_key)
+
+        missing_keys = []
+        unexpected_keys = []
+        error_msgs = []
+        # copy state_dict so _load_from_state_dict can modify it
+        metadata = getattr(state_dict, '_metadata', None)
+        state_dict = state_dict.copy()
+        if metadata is not None:
+            state_dict._metadata = metadata
+
+        def load(module, prefix=''):
+            local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
+            module._load_from_state_dict(
+                state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs)
+            for name, child in module._modules.items():
+                if child is not None:
+                    load(child, prefix + name + '.')
+
+        load(model, prefix='')
+
+        if prefix is None and (task_config is None or task_config.local_rank == 0):
+            logger.info("-" * 20)
+            if len(missing_keys) > 0:
+                logger.info("Weights of {} not initialized from pretrained model: {}"
+                            .format(model.__class__.__name__, "\n   " + "\n   ".join(missing_keys)))
+            if len(unexpected_keys) > 0:
+                logger.info("Weights from pretrained model not used in {}: {}"
+                            .format(model.__class__.__name__, "\n   " + "\n   ".join(unexpected_keys)))
+            if len(error_msgs) > 0:
+                logger.error("Weights from pretrained model cause errors in {}: {}"
+                             .format(model.__class__.__name__, "\n   " + "\n   ".join(error_msgs)))
+
+        return model
+
+    @property
+    def dtype(self):
+        """
+        :obj:`torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype).
+        """
+        try:
+            return next(self.parameters()).dtype
+        except StopIteration:
+            # For nn.DataParallel compatibility in PyTorch 1.5
+            def find_tensor_attributes(module: nn.Module):
+                tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)]
+                return tuples
+
+            gen = self._named_members(get_members_fn=find_tensor_attributes)
+            first_tuple = next(gen)
+            return first_tuple[1].dtype
+
+    @classmethod
+    def from_pretrained(cls, config, state_dict=None,  *inputs, **kwargs):
+        """
+        Instantiate a PreTrainedModel from a pre-trained model file or a pytorch state dict.
+        Download and cache the pre-trained model file if needed.
+        """
+        # Instantiate model.
+        model = cls(config, *inputs, **kwargs)
+        if state_dict is None:
+            return model
+        model = cls.init_preweight(model, state_dict)
+
+        return model
+
+##################################
+###### LOSS FUNCTION #############
+##################################
+class CrossEn(nn.Module):
+    def __init__(self,):
+        super(CrossEn, self).__init__()
+
+    def forward(self, sim_matrix):
+        logpt = F.log_softmax(sim_matrix, dim=-1)
+        logpt = torch.diag(logpt)
+        nce_loss = -logpt
+        sim_loss = nce_loss.mean()
+        return sim_loss
+
+class Dual_CrossEn(nn.Module):
+    def __init__(self,):
+        super(Dual_CrossEn, self).__init__()
+
+    def forward(self, sim_matrix):
+        sim_matrix = get_dual_matrix(sim_matrix)
+        logpt = F.log_softmax(sim_matrix, dim=-1)
+        logpt = torch.diag(logpt)
+        nce_loss = -logpt
+        sim_loss = nce_loss.mean()
+        return sim_loss
+
+class MILNCELoss(nn.Module):
+    def __init__(self, batch_size=1, n_pair=1,):
+        super(MILNCELoss, self).__init__()
+        self.batch_size = batch_size
+        self.n_pair = n_pair
+        torch_v = float(".".join(torch.__version__.split(".")[:2]))
+        self.bool_dtype = torch.bool if torch_v >= 1.3 else torch.uint8
+
+    def forward(self, sim_matrix):
+        mm_mask = np.eye(self.batch_size)
+        mm_mask = np.kron(mm_mask, np.ones((self.n_pair, self.n_pair)))
+        mm_mask = torch.tensor(mm_mask).float().to(sim_matrix.device)
+
+        from_text_matrix = sim_matrix + mm_mask * -1e12
+        from_video_matrix = sim_matrix.transpose(1, 0)
+
+        new_sim_matrix = torch.cat([from_video_matrix, from_text_matrix], dim=-1)
+        logpt = F.log_softmax(new_sim_matrix, dim=-1)
+
+        mm_mask_logpt = torch.cat([mm_mask, torch.zeros_like(mm_mask)], dim=-1)
+        masked_logpt = logpt + (torch.ones_like(mm_mask_logpt) - mm_mask_logpt) * -1e12
+
+        new_logpt = -torch.logsumexp(masked_logpt, dim=-1)
+
+        logpt_choice = torch.zeros_like(new_logpt)
+        mark_ind = torch.arange(self.batch_size).to(sim_matrix.device) * self.n_pair + (self.n_pair//2)
+        logpt_choice[mark_ind] = 1
+        sim_loss = new_logpt.masked_select(logpt_choice.to(dtype=self.bool_dtype)).mean()
+        return sim_loss
+
+class MaxMarginRankingLoss(nn.Module):
+    def __init__(self,
+                 margin=1.0,
+                 negative_weighting=False,
+                 batch_size=1,
+                 n_pair=1,
+                 hard_negative_rate=0.5,
+        ):
+        super(MaxMarginRankingLoss, self).__init__()
+        self.margin = margin
+        self.n_pair = n_pair
+        self.batch_size = batch_size
+        easy_negative_rate = 1 - hard_negative_rate
+        self.easy_negative_rate = easy_negative_rate
+        self.negative_weighting = negative_weighting
+        if n_pair > 1 and batch_size > 1:
+            alpha = easy_negative_rate / ((batch_size - 1) * (1 - easy_negative_rate))
+            mm_mask = (1 - alpha) * np.eye(self.batch_size) + alpha
+            mm_mask = np.kron(mm_mask, np.ones((n_pair, n_pair)))
+            mm_mask = torch.tensor(mm_mask) * (batch_size * (1 - easy_negative_rate))
+            self.mm_mask = mm_mask.float()
+
+    def forward(self, x):
+        d = torch.diag(x)
+        max_margin = F.relu(self.margin + x - d.view(-1, 1)) + \
+                     F.relu(self.margin + x - d.view(1, -1))
+        if self.negative_weighting and self.n_pair > 1 and self.batch_size > 1:
+            max_margin = max_margin * self.mm_mask.to(max_margin.device)
+        return max_margin.mean()
+
+class AllGather(torch.autograd.Function):
+    """An autograd function that performs allgather on a tensor."""
+
+    @staticmethod
+    def forward(ctx, tensor, args):
+        output = [torch.empty_like(tensor) for _ in range(args.world_size)]
+        torch.distributed.all_gather(output, tensor)
+        ctx.rank = args.rank
+        ctx.batch_size = tensor.shape[0]
+        return torch.cat(output, dim=0)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        return (
+            grad_output[ctx.batch_size * ctx.rank : ctx.batch_size * (ctx.rank + 1)],
+            None,
+        )

search.py

@@ -0,0 +1,299 @@
+import numpy as np
+# import gradio
+import torch
+from transformers import BertTokenizer
+import argparse
+import gradio as gr
+import time
+
+from modules.tokenization_clip import SimpleTokenizer as ClipTokenizer
+from modules.modeling import BirdModel
+
+show_num = 10
+max_words = 32
+video_path_zh = "features/Chinese_batch_visual_output_list.npy"
+frame_path_zh = "features/Chinese_batch_frame_output_list.npy"
+video_fea_zh = np.load(video_path_zh)
+video_fea_zh = torch.from_numpy(video_fea_zh)
+frame_fea_zh = np.load(frame_path_zh)
+frame_fea_zh = torch.from_numpy(frame_fea_zh)
+
+video_path_en = "features/English_batch_visual_output_list.npy"
+frame_path_en = "features/English_batch_frame_output_list.npy"
+video_fea_en = np.load(video_path_en)
+video_fea_en = torch.from_numpy(video_fea_en)
+frame_fea_en = np.load(frame_path_en)
+frame_fea_en = torch.from_numpy(frame_fea_en)
+
+test_path = "test_list.txt"
+# video_dir = "test1500_400_400/"
+video_dir = "test1500/"
+
+with open(test_path, 'r', encoding='utf8') as f_list:
+    lines = f_list.readlines()
+    video_ids = [itm.strip() + ".mp4" for itm in lines]
+
+
+def get_videoname(idx):
+    videoname = []
+    videopath = []
+    for i in idx:
+        videoname.append(video_ids[i])
+        path = video_dir + video_ids[i]
+        videopath.append(path)
+    return videoname, videopath
+
+
+def get_text(caption, tokenizer):
+    # tokenize word
+    words = tokenizer.tokenize(caption)
+
+    # add cls token
+    words = ["<|startoftext|>"] + words
+    total_length_with_CLS = max_words - 1
+    if len(words) > total_length_with_CLS:
+        words = words[:total_length_with_CLS]
+
+    # add end token
+    words = words + ["<|endoftext|>"]
+
+    # convert token to id according to the vocab
+    input_ids = tokenizer.convert_tokens_to_ids(words)
+
+    # add zeros for feature of the same length
+    input_mask = [1] * len(input_ids)
+    while len(input_ids) < max_words:
+        input_ids.append(0)
+        input_mask.append(0)
+
+    # ensure the length of feature to be equal with max words
+    assert len(input_ids) == max_words
+    assert len(input_mask) == max_words
+    pairs_text = np.array(input_ids).reshape(-1, max_words)
+    pairs_text = torch.from_numpy(pairs_text)
+    pairs_mask = np.array(input_mask).reshape(-1, max_words)
+    pairs_mask = torch.from_numpy(pairs_mask)
+
+    return pairs_text, pairs_mask
+
+
+def get_args(description='Retrieval Task'):
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument("--do_pretrain", action='store_true', help="Whether to run training.")
+    parser.add_argument("--do_train", action='store_true', help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true', help="Whether to run eval on the dev set.")
+    parser.add_argument("--do_params", action='store_true', help="text the params of the model.")
+    parser.add_argument("--use_frame_fea", action='store_true', help="whether use frame feature matching text")
+    parser.add_argument('--task', type=str, default="retrieval", choices=["retrieval_VT", "retrieval"],
+                        help="choose downstream task.")
+    parser.add_argument('--dataset', type=str, default="bird", choices=["bird", "msrvtt", "vatex", "msvd"],
+                        help="choose dataset.")
+    parser.add_argument('--num_thread_reader', type=int, default=1, help='')
+    parser.add_argument('--lr', type=float, default=0.0001, help='initial learning rate')
+    parser.add_argument('--text_lr', type=float, default=0.00001, help='text encoder learning rate')
+    parser.add_argument('--epochs', type=int, default=20, help='upper epoch limit')
+    parser.add_argument('--batch_size', type=int, default=256, help='batch size')
+    parser.add_argument('--batch_size_val', type=int, default=3500, help='batch size eval')
+    parser.add_argument('--lr_decay', type=float, default=0.9, help='Learning rate exp epoch decay')
+    parser.add_argument('--weight_decay', type=float, default=0.2, help='Learning rate exp epoch decay')
+    parser.add_argument('--n_display', type=int, default=100, help='Information display frequence')
+    parser.add_argument('--seed', type=int, default=42, help='random seed')
+    parser.add_argument('--max_words', type=int, default=32, help='')
+    parser.add_argument('--max_frames', type=int, default=12, help='')
+    parser.add_argument('--top_frames', type=int, default=3, help='')
+    parser.add_argument('--frame_sample', type=str, default="uniform", choices=["uniform", "random", "uniform_random"],
+                        help='frame sample strategy')
+    parser.add_argument('--frame_sample_len', type=str, default="fix", choices=["dynamic", "fix"],
+                        help='use dynamic frame length of fix frame length')
+    parser.add_argument('--language', type=str, default="chinese", choices=["chinese", "english"],
+                        help='language for text encoder')
+    parser.add_argument('--use_temp', action='store_true', help='whether to use temporal transformer')
+
+    parser.add_argument("--logdir", default=None, type=str, required=False, help="log dir for tensorboardX writer")
+    parser.add_argument("--cross_model", default="cross-base", type=str, required=False, help="Cross module")
+    parser.add_argument("--pretrained_text", default="hfl/chinese-roberta-wwm-ext", type=str, required=False, help="pretrained_text")
+    parser.add_argument("--init_model", default=None, type=str, required=False, help="Initial model.")
+    parser.add_argument("--warmup_proportion", default=0.1, type=float,
+                        help="Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%% of training.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument('--n_gpu', type=int, default=1, help="Changed in the execute process.")
+
+    parser.add_argument("--cache_dir", default="", type=str,
+                        help="Where do you want to store the pre-trained models downloaded from s3")
+
+    parser.add_argument('--enable_amp', action='store_true', help="whether to use pytorch amp")
+
+    parser.add_argument("--world_size", default=0, type=int, help="distribted training")
+    parser.add_argument("--local_rank", default=0, type=int, help="distribted training")
+    parser.add_argument("--rank", default=0, type=int, help="distribted training")
+    parser.add_argument('--coef_lr', type=float, default=1., help='coefficient for bert branch.')
+
+    args = parser.parse_args()
+
+    # Check paramenters
+    args.do_eval = True
+    args.use_frame_fea = True
+    args.use_temp = True
+
+    return args
+
+
+def init_model(language):
+    time1 = time.time()
+    args = get_args()
+    args.language = language
+    if language == "chinese":
+        model_path = "models/Chinese_vatex.bin"
+        tokenizer = BertTokenizer.from_pretrained("hfl/chinese-roberta-wwm-ext")
+    elif language == "english":
+        model_path = "models/English_vatex.bin"
+        tokenizer = ClipTokenizer()
+    else:
+        raise Exception("language should be Chinese or English!")
+    model_state_dict = torch.load(model_path, map_location='cpu')
+    cross_model = "cross-base"
+    model = BirdModel.from_pretrained(cross_model, state_dict=model_state_dict, task_config=args)
+    device = torch.device("cpu")
+    # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model.to(device)
+    model.eval()
+    print("language={}".format(language))
+    print("init model time: {}".format(time.time() - time1))
+    print("device:{}".format(device))
+    return model, tokenizer
+
+
+model_zh, tokenizer_zh = init_model(language="chinese")
+model_en, tokenizer_en = init_model(language="english")
+
+
+def t2v_search_zh(text):
+    with torch.no_grad():
+        time1 = time.time()
+        text_ids, text_mask = get_text(text, tokenizer_zh)
+        print("get_text time: {}".format(time.time() - time1))
+        time1 = time.time()
+        text_fea_zh = model_zh.text_encoder(text_ids, text_mask)
+        print("text_encoder time: {}".format(time.time() - time1))
+        # print("text_fea.shape:{}".format(text_fea.shape))
+        # print("video_fea.shape:{}".format(video_fea.shape))
+        # print("frame_fea.shape:{}".format(frame_fea.shape))
+        time1 = time.time()
+        sim_video = model_zh.loose_similarity(text_fea_zh, video_fea_zh)
+        # print("sim_video.shape:{}".format(sim_video.shape))
+        sim_frame = model_zh.loose_similarity(text_fea_zh, frame_fea_zh)
+        # print("sim_frame.shape:{}".format(sim_frame.shape))
+        sim_frame = torch.topk(sim_frame, k=model_zh.top_frames, dim=1)[0]
+        sim_frame = torch.mean(sim_frame, dim=1)
+        sim = sim_video + sim_frame
+        value, index = sim.topk(show_num, dim=0, largest=True, sorted=True)
+        # value, index = sim_video.topk(show_num, dim=0, largest=True, sorted=True)
+        print("calculate_similarity time: {}".format(time.time() - time1))
+        print("value:{}".format(value))
+        print("index:{}".format(index))
+        videoname, videopath = get_videoname(index)
+        print("videoname:{}".format(videoname))
+        print("videopath:{}".format(videopath))
+        return videopath
+
+
+def t2v_search_en(text):
+    with torch.no_grad():
+        time1 = time.time()
+        text_ids, text_mask = get_text(text, tokenizer_en)
+        print("get_text time: {}".format(time.time() - time1))
+        time1 = time.time()
+        text_fea_en = model_en.text_encoder(text_ids, text_mask)
+        print("text_encoder time: {}".format(time.time() - time1))
+        # print("text_fea.shape:{}".format(text_fea.shape))
+        # print("video_fea.shape:{}".format(video_fea.shape))
+        # print("frame_fea.shape:{}".format(frame_fea.shape))
+        time1 = time.time()
+        sim_video = model_en.loose_similarity(text_fea_en, video_fea_en)
+        # print("sim_video.shape:{}".format(sim_video.shape))
+        sim_frame = model_en.loose_similarity(text_fea_en, frame_fea_en)
+        # print("sim_frame.shape:{}".format(sim_frame.shape))
+        sim_frame = torch.topk(sim_frame, k=model_en.top_frames, dim=1)[0]
+        sim_frame = torch.mean(sim_frame, dim=1)
+        sim = sim_video + sim_frame
+        value, index = sim.topk(show_num, dim=0, largest=True, sorted=True)
+        # value, index = sim_video.topk(show_num, dim=0, largest=True, sorted=True)
+        print("calculate_similarity time: {}".format(time.time() - time1))
+        print("value:{}".format(value))
+        print("index:{}".format(index))
+        videoname, videopath = get_videoname(index)
+        print("videoname:{}".format(videoname))
+        print("videopath:{}".format(videopath))
+        return videopath
+
+
+def hello_world(name):
+    return "hello world, my name is " + name + "!"
+
+
+def search_demo():
+    with gr.Blocks() as demo:
+        gr.Markdown("# <center>HMMC中英文本-视频检索</center>")
+        demo.title = "HMMC中英文本-视频检索"
+        with gr.Tab("中文"):
+            with gr.Column(variant="panel"):
+                with gr.Row(variant="compact"):
+                    input_text = gr.Textbox(
+                        label="输入文本",
+                        show_label=False,
+                        max_lines=1,
+                        placeholder="请输入检索文本...",
+                    ).style(
+                        container=False,
+                    )
+                    btn = gr.Button("搜索").style(full_width=False)
+
+                with gr.Column(variant="panel", scale=2):
+                    with gr.Row(variant="compact"):
+                        videos_top = [gr.Video(
+                            format="mp4", label="视频 "+str(i+1),
+                        ).style(height=300, width=300) for i in range(3)]
+                with gr.Column(variant="panel", scale=1):
+                    with gr.Row(variant="compact"):
+                        videos_rest = [gr.Video(
+                            format="mp4", label="视频 "+str(i),
+                        ).style(height=150, width=150) for i in range(3, show_num)]
+
+            searched_videos = videos_top + videos_rest
+            btn.click(t2v_search_zh, inputs=input_text, outputs=searched_videos)
+
+        with gr.Tab("English"):
+            with gr.Column(variant="panel"):
+                with gr.Row(variant="compact"):
+                    input_text = gr.Textbox(
+                        label="input text",
+                        show_label=False,
+                        max_lines=1,
+                        placeholder="Please input text to search...",
+                    ).style(
+                        container=False,
+                    )
+                    btn = gr.Button("Search").style(full_width=False)
+
+                with gr.Column(variant="panel", scale=2):
+                    with gr.Row(variant="compact"):
+                        videos_top = [gr.Video(
+                            format="mp4", label="video " + str(i+1),
+                        ).style(height=300, width=300) for i in range(3)]
+                with gr.Column(variant="panel", scale=1):
+                    with gr.Row(variant="compact"):
+                        videos_rest = [gr.Video(
+                            format="mp4", label="video " + str(i),
+                        ).style(height=150, width=150) for i in range(3, show_num)]
+
+            searched_videos = videos_top + videos_rest
+            btn.click(t2v_search_en, inputs=input_text, outputs=searched_videos)
+
+    demo.launch(share=True)
+
+
+if __name__ == '__main__':
+    search_demo()
+    # text = "两个男人正在随着音乐跳舞，他们正在努力做着macarena舞蹈的动作。"
+    # t2v_search(text)

test_list.txt

@@ -0,0 +1,1500 @@
+sf2PPeXQrUA_000000_000010
+rIJPGdtdaHg_000006_000016
+6FjgdUricEs_000022_000032
+-PCVHF3Kp3M_000169_000179
+cb3RvnukQVs_000159_000169
+RbpDta9EICM_000082_000092
+QDUZ4L9gpP8_000317_000327
+5BNZDNxZfX0_000004_000014
+B8uJabL49FI_000074_000084
+nAD5PuUHVDM_000027_000037
+YLqQSpRH09o_000002_000012
+CKUaeC_FWoA_000019_000029
+2E6ZLAK593E_000002_000012
+CiTHS0W9vkg_000047_000057
+OxiWoQOzZaM_000376_000386
+qbTYNlpD5nM_000078_000088
+sSX1KaKyrag_000007_000017
+BFYBAg9fSL4_000388_000398
+2zqplqQwlL8_000447_000457
+1y_uKjQmjWg_000020_000030
+dinFGaxi9Rc_000023_000033
+5F4jp64eh1w_000000_000010
+ihOTHLgvm8s_000031_000041
+7zzNK8TtWls_000003_000013
+u4sITQgf03I_000009_000019
+sEw4ZBSbTzQ_000000_000010
+MrzLrekK1Cc_000000_000010
+yIus7dY2BKU_000094_000104
+QVygizGIC3M_000004_000014
+DLxAkF7lYHE_000184_000194
+aL6zh0oCzZE_000039_000049
+0B7puDmojro_000061_000071
+i7SAVqA-3Bk_000032_000042
+3IuP3YYwzzA_000000_000010
+Vli3ay79kJk_000000_000010
+OnIqmqa9mqc_001907_001917
+sFrRHqQ9AIw_000022_000032
+NnHX9Auke60_000022_000032
+ln9NeisDnFU_000116_000126
+P2abrZ-cbqk_000000_000010
+9gCXYhcc8K0_000104_000114
+v1onbz5rSDo_000000_000010
+YG8JiyFKQ9M_000094_000104
+khURP8O3k_k_000241_000251
+Eks62nEKfH8_000066_000076
+nxDAt9SkPoA_000123_000133
+IrbMK1MKgQA_000006_000016
+iq4ysEnDQBc_000071_000081
+c_72NRiKphQ_000178_000188
+2bCUoQLw7OE_000000_000010
+uGBeU98HV0Y_000156_000166
+tSQqcJqGplA_000011_000021
+Hnex85IkEPg_000000_000010
+ca0muRN-gjM_000007_000017
+JwDePTiGRcY_000029_000039
+q-QOU4kjUCQ_000001_000011
+OfDDeDByNKE_000489_000499
+uh50grtCQSA_000026_000036
+amvcZogOvvI_000007_000017
+jiKZmCswPeE_000001_000011
+XBlVVyhfQ2Y_000052_000062
+rbaoOjMKbio_000019_000029
+aqRD8TK9K1w_000005_000015
+JpLrWNjyaQU_000027_000037
+s6bOJtUE0FU_000007_000017
+bpw7kP8IEJU_000008_000018
+wG1wBA9lyJE_000004_000014
+0gZqPa1qo4s_000013_000023
+WONC-tU62RA_000009_000019
+pzf5Gqy1JxQ_000162_000172
+Rs9T2YtS46Q_000100_000110
+3exdpRbcMmA_000034_000044
+pNvKrzbJEhg_000010_000020
+ErRHPNJo9f4_000003_000013
+gbYcrYeqCHM_000250_000260
+YIkysT5HGrM_000235_000245
+WEtK-Ndd9XE_000140_000150
+Ca7inKrelzI_000064_000074
+Lvg6fb0DXSY_000035_000045
+2BNkQFQGaNI_000572_000582
+-Tc2NGLTTD0_000121_000131
+GzlcYW2YF8o_000011_000021
+WeL8Ie7hCTo_000003_000013
+X4052giOzGA_000017_000027
+8wID3JI7m78_000744_000754
+OCLMRUbRb6Q_000109_000119
+d5FWcjUn1xU_000064_000074
+Eicloge6jFM_000012_000022
+fq0kIJhthbA_000076_000086
+Ai0RaLVZW_g_000004_000014
+XqkIJrR1loM_000013_000023
+Ie1GRyaFpQc_000045_000055
+bmbtEThyum0_000000_000010
+hm9NCa_E3ig_000243_000253
+Pg4VtQT5dAk_000004_000014
+nc1ptYDp3zI_000033_000043
+GNUDkrXspaQ_000069_000079
+DOa84IoN8N8_000014_000024
+K420tiUw2q8_000000_000010
+2m8H3ViIs0U_000000_000010
+CZ1oWiDS_5c_000000_000010
+PCicM6i59_I_000118_000128
+HCs5Vj6Zy0I_000004_000014
+m5D-kLzwNkg_000027_000037
+k6icUmA7cM4_000071_000081
+zcqnH8LO1Dg_000016_000026
+67Izc6v8_zc_000010_000020
+TdC_JQ8SfVg_000004_000014
+Ag9mJ4tEIEE_000273_000283
+Z5ywBikWPqU_000045_000055
+jMo78oeASw0_000002_000012
+45HnjQIVvT4_000009_000019
+yDa27vSKGYY_000036_000046
+3AsOzMAM91E_000004_000014
+4Bdkmj2hFm8_000034_000044
+2kM2NrMhnis_000156_000166
+KqDAM8_S0DU_000048_000058
+2VRuFjC8HDM_000004_000014
+sldlahvl3Tc_000008_000018
+4N-OoJZD_Qg_000102_000112
+Rzz2jV9q9gU_000013_000023
+VFp8GkLwi50_000095_000105
+nPIdoH_CJYE_000126_000136
+lV4r_8tHGAY_000034_000044
+DeRCbSPU8NM_000085_000095
+duhAEqT-Ttw_000017_000027
+-b6gqj_mZOs_000009_000019
+anzQwjeBbYs_000009_000019
+RGNxTr2vZ8o_000170_000180
+R7PXIX_YWJg_000131_000141
+nQJAlymtQt0_000034_000044
+FHxv5n5CrsA_000000_000010
+kGWKUPfsBuY_000088_000098
+QfNv-NxsWfc_000062_000072
+isW9E-Zp90A_000002_000012
+zlgbDWzA5-w_000122_000132
+Qxx2JJE9Ev4_000000_000010
+kDNG-8C2Jrs_000000_000010
+-_axH6kxbic_000003_000013
+1Jz8JZFuVLk_000014_000024
+tIcwx3MH_kw_000017_000027
+Zjr3hHz4Q2M_000150_000160
+p7r-ajQtBZs_000031_000041
+jGH_eRWyGVQ_000004_000014
+OSfIV2aQd_I_000014_000024
+6yHTZHY8_3Y_000026_000036
+cfBtVPEwqIo_000005_000015
+ZnKbREYJ1C0_000002_000012
+Z03ZQ_h59Pk_000015_000025
+cBuoOA5jIRI_000039_000049
+ecNLf37mews_000012_000022
+sy9T57J6qEg_000002_000012
+clZ-eZWW-HE_000105_000115
+yujJLRHVm-A_000005_000015
+58QjPoalXlg_000002_000012
+FnbrvKMwJFc_000025_000035
+fE0yN1CBMvA_000160_000170
+H1rf96-Jm0o_000023_000033
+Be3L2e7HN54_000336_000346
+mAV6Pn-3a5U_000006_000016
+epfCEGpA8OY_000000_000010
+9JonoONyXN0_000025_000035
+mj4JGloJS4o_000031_000041
+thn033gNKOA_000021_000031
+2fx306CeqEI_000025_000035
+G88xmYArd8o_000001_000011
+sovJ_-BsO6s_000018_000028
+9G1LGoB-8as_000001_000011
+QFmHnhLqf6c_000006_000016
+WY9TXZ2ODgw_000150_000160
+6oywr9jgp1M_000084_000094
+vj7eeYGfozs_000112_000122
+36JI7dIZSRw_000071_000081
+R4l6hWmfKqM_000006_000016
+9Zn5zr7J7qs_000041_000051
+5EheWAKuZnY_000003_000013
+TYTWS2PZLQg_000032_000042
+j-v9WpQtvT4_000001_000011
+891pot4xKPk_000191_000201
+PQSZLVqO7rU_000110_000120
+q_zCR9ITIrE_000006_000016
+QLNszf39dbM_000253_000263
+_UoN-y6SzjI_000004_000014
+AUJdvbM0dLo_000058_000068
+YTdjyKCu53M_000032_000042
+oX90Q352-D0_000019_000029
+7qVkwNb0dtM_000002_000012
+l17LtEO6z3o_000033_000043
+5RIe5niLskU_000004_000014
+E8EhsfkqT0c_000092_000102
+9bx30W8aOKM_000001_000011
+CVmh-a1APJc_000006_000016
+a_1mjN2I7RI_001337_001347
+BlHNGhui6LA_000021_000031
+fl56tjPytbY_000094_000104
+Y_dxesnBzg0_000020_000030
+7COkxV_YfOU_000000_000010
+b9TfeDnfemw_000047_000057
+D1-cCXclgZo_000010_000020
+Ya7JaHCbtQw_000065_000075
+S7NEpkqr0Xc_000027_000037
+3D3IWV72ZGI_000041_000051
+M2DXKEp_u9U_000023_000033
+bHdGTlzuwRQ_000002_000012
+ArN3eVUygAg_000058_000068
+vbQ2x8_9giw_000095_000105
+p107RA-3woA_000137_000147
+35_IiB3_nVw_000017_000027
+ShgajdY_h9I_000000_000010
+8LHvdxSF9Ek_000004_000014
+dQOTyHhTLbY_000066_000076
+DIMBtwbtnYQ_000182_000192
+F6ZLuarySKo_000136_000146
+VxDOiYNIfnA_000041_000051
+V3Wp5siliwc_000090_000100
+NRC5oMoNHn0_000003_000013
+_lDzLPASjfg_000007_000017
+hB2DZcypK0Y_000187_000197
+iNdJfXbW4UY_000326_000336
+LJbIxxpTctA_000350_000360
+CweM_4o1JOM_000004_000014
+WMAR-K-68jI_000005_000015
+i3ZGzZ65rvg_000008_000018
+SUhKIoo8Y7U_000011_000021
+AhMpS-dz2pc_000002_000012
+CdpUfCli4ys_000049_000059
+m6RF5r3I8dA_000033_000043
+vhls-0FOAqE_000042_000052
+OF-OZUe4GJc_000018_000028
+DhnHzooyFo4_000042_000052
+5D4HjS92zSQ_000126_000136
+vgia9zM-Tao_000033_000043
+SwdKVfwtSQw_000000_000010
+v5UfN6dUVcs_000019_000029
+tDmfTtpDF6Q_000333_000343
+3nTMc2nK_W8_000000_000010
+JPFdgQvqwjI_000328_000338
+qwLeYfEOrnU_000000_000010
+VHFuW1yFqXY_000120_000130
+h6_FiX5vO_U_000009_000019
+kh6KufiQW5A_000097_000107
+c25IqRBGKlA_000004_000014
+Ip0yJAkp7Pw_000119_000129
+t-YDm1WRdEY_000143_000153
+hZ34EcXYuIU_000002_000012
+WFG38qMr2Fk_000021_000031
+nTK7fjMfbSA_000002_000012
+GCtPDRrBBc0_000000_000010
+nmefMgzgISw_000020_000030
+0GWr1EV6ZuY_000106_000116
+syI2EiICWOM_000001_000011
+EGQxtiReVnE_000014_000024
+tsnbD70rddU_000024_000034
+IZmQ1ndVmyc_000004_000014
+fzuw0Wp6dF8_000003_000013
+5AJUbDC0YwY_000102_000112
+YMvyLtafd6Q_000115_000125
+L9xSjSgDamc_000000_000010
+mwKmET7GFNU_000022_000032
+b1qE4GNZ9OA_000144_000154
+QTOaEVeoCRE_000177_000187
+AtmUxjr6ZFI_000004_000014
+uoOEgOBo7Uw_000123_000133
+jbyVJdDlo2Q_000059_000069
+P4LS1ilfloQ_000000_000010
+OtyiGXrVnZc_000157_000167
+aC5txX6gJLI_000003_000013
+2v3otYP4bZc_000196_000206
+ady69p6ykqE_000001_000011
+7YJZ6Q2ZMFU_000060_000070
+_G9ka7ghM1c_000049_000059
+FTxa5j8T03k_000009_000019
+y1we1RI5qmg_000000_000010
+1nihsi5qNJA_000074_000084
+n7wyjDPtFtg_000003_000013
+At8LPKnA9Zg_000134_000144
+5eh2MbjEyXs_000003_000013
+ueNk8_69WOE_000002_000012
+NfAvGwYhhSw_000025_000035
+vGRkF47_b-8_000164_000174
+GqnNJywvDy4_000053_000063
+JGluzZgbe2E_000048_000058
+0nQBmFswEOw_000062_000072
+h6Hf1d7hJ8M_000010_000020
+A0yyN0Z68-A_000053_000063
+aMTce4pTh00_000047_000057
+FcbL8yPAzZo_000155_000165
+VNUjTJNGUa8_000029_000039
+WsEfaI-2azs_000024_000034
+iUANfGZ5dCY_000014_000024
+IHcaQ-P7ytg_000048_000058
+U5wQ8PwwXFg_000009_000019
+VHKS2AZEgro_000139_000149
+cU1qVk7HXfE_000304_000314
+Dqg_m36iVuM_000081_000091
+QkG03wZh7ew_000034_000044
+lwCnqd0E4HI_000003_000013
+eHtYCCJwKbs_000003_000013
+AcWGNfJnBOE_000375_000385
+WEQbqLqRvOY_000021_000031
+nrF8FOuvACg_000127_000137
+8oDKuNb4olE_000042_000052
+AxQPpRGxm-Q_000054_000064
+QgzJW2EWJK8_000002_000012
+_6LbEATZ-jM_000045_000055
+JfIz0f8WCNM_000010_000020
+sqOojwuFaFU_000002_000012
+RFX6UWeTTtU_000007_000017
+F29GqCDFic8_000711_000721
+aw55asaJJ34_000010_000020
+cHycKylBtlY_000022_000032
+Q8FlLIyZF50_000004_000014
+fcWWf5cAnWQ_000023_000033
+GqCUEAigBWw_000012_000022
+VKFX8D6bwo4_000001_000011
+NjMJsEHTnOE_000048_000058
+mHDdn4z-cjI_000076_000086
+gf37sAjEfRc_000013_000023
+nPKOfBLkLpw_000461_000471
+L3lh_TzunmI_000003_000013
+5alRoEFZJCs_000002_000012
+nT6bAb_4HgY_000003_000013
+gmPNqbpI04w_000044_000054
+_WPR0Auk7ZM_000024_000034
+t61eKij6cNo_000019_000029
+kRdxCmOsY2Y_000003_000013
+2axIb7lejhw_000167_000177
+KKm1bM51CZs_000070_000080
+iwIttka43nE_000037_000047
+BWxQXzXynhQ_000387_000397
+eTUGCFPG9N4_000243_000253
+OH0FJZSwIeg_000103_000113
+i4go33PLX6A_000117_000127
+yJW-DDaMZ-M_000024_000034
+plVYb0kiH-I_000252_000262
+Jiwq0zk18P0_000082_000092
+cMwJhIEqROM_000000_000010
+MPWJCZpIZoM_000029_000039
+XxY0Z47n7pg_000080_000090
+9nEHFb1mGuI_000058_000068
+P7TDOO5paL8_000137_000147
+s1VemC8UQ9k_000090_000100
+U3VrnCjpEmI_000164_000174
+RxO7IEU7_I8_000391_000401
+sLRwdRnwCWs_000051_000061
+RNA_wIhYPp8_000047_000057
+luSl6aGT1kw_000010_000020
+37k7DrOPzbw_000002_000012
+0y9jAiuCu78_000073_000083
+SA080tXbV6I_000000_000010
+Vt6hfbMUzWc_000044_000054
+MjBWljbRcxg_000007_000017
+Vn0FNH2XTx4_000002_000012
+OjowI-z7-oo_000006_000016
+VhrYiENr0Fk_000440_000450
+20gCvLBfyzQ_000069_000079
+y2CfrrXbLQI_000517_000527
+M29qNwCV8Sk_000069_000079
+shktJpYmOOk_000011_000021
+hRQK0ToE90w_000375_000385
+BcF2As-ebhw_000061_000071
+t4BHlEbafvc_000040_000050
+paxANprdjk0_000013_000023
+sMtcdADtkNU_000004_000014
+NHn-t_VarHg_000000_000010
+_iyEqlFFrxU_000047_000057
+udEsPNoiJR4_000012_000022
+T6bx9H6gL2I_000000_000010
+7AY86S8Vg6g_000009_000019
+VihXKPrgTrE_000069_000079
+pTgIYOnWrw0_000047_000057
+88oXj0zKdp8_000000_000010
+1Ez_zoJDukw_000014_000024
+wC-mLP684J4_000028_000038
+OBbboTWbUQo_000090_000100
+-IGwsxNxUpo_000002_000012
+_3DfEgSKpjM_000035_000045
+n64kp3OZY6Q_000262_000272
+2EkpqUeIUGA_000069_000079
+h-ekL4jR214_000000_000010
+5s0Hr7dZRXU_000044_000054
+z1DcfYXY8J0_000134_000144
+3jP9HNJddR8_000003_000013
+c-7aCvrVkoM_000002_000012
+o7irCXceNjg_000046_000056
+wPv6ODnrLAE_000044_000054
+kamRtGTha5Q_000057_000067
+4kgJJ1FjTrw_000091_000101
+ZVDR2od1gn8_000037_000047
+e9WO9WHuP8s_000018_000028
+5AqRrxvRC70_000025_000035
+1sazJwkVbNY_000010_000020
+UrV44zz_gnE_000056_000066
+glIq9KmXoYw_000004_000014
+fV2RqP9ZlSc_000197_000207
+9RWgHcPZ07k_000039_000049
+-MV_iN5WXuw_000002_000012
+j5GuVIzDgrE_000047_000057
+Z1WLfpOiqpI_000002_000012
+fGuGmBLkWKg_000034_000044
+6eCU1LOTyCw_000024_000034
+PXOluGNqZcc_000120_000130
+9yPQHK-GXUs_000056_000066
+b7-jU2adf_Q_000010_000020
+9NWpyCuMjBM_000021_000031
+-6CM7VV8-_w_000262_000272
+-kmzssWKO-o_000029_000039
+LAH48I8WTbU_000008_000018
+wAVx8xJAJxc_000007_000017
+yhZCJEr4x1w_000050_000060
+a8HehnYjquw_000107_000117
+oIosEbECUo8_000077_000087
+06fOUg4SADg_000003_000013
+t3lMq5lZJ04_000000_000010
+E1nEWt5VZzY_000061_000071
+Z5BkKxbGuXc_000000_000010
+4IwQFgz7Ni8_000000_000010
+NcmA-Oyvx1A_000078_000088
+BqdjAlbg9XE_000021_000031
+rEJaqonTVVE_000000_000010
+fB1mByWn1Xs_000023_000033
+q2XxNNIPl_A_000039_000049
+K0xgh53GRGg_000007_000017
+xc17jJnxGk0_000012_000022
+TEj_A_BC-aU_000184_000194
+-HKeIdvg7U0_000011_000021
+lUeIoyXODec_000001_000011
+HvnLVSTwOUE_000239_000249
+GkkWbdydkPk_000014_000024
+lBkWzOXr3Y4_000012_000022
+UKGWaUaRXi4_000095_000105
+Vpz3Xxj8ndU_000012_000022
+lj4GXOHT-88_000028_000038
+j59F32qbHEg_000088_000098
+P0coxgcUXQ4_000164_000174
+5sg_PjZ2kvo_000011_000021
+J2MNJqVHPX8_000001_000011
+08jcCNdrp4E_000044_000054
+bB0p-_ie8sg_000423_000433
+9wXgbdeeS1w_000000_000010
+RQUxuTHi2P4_000020_000030
+5bPlcd0w7uA_000001_000011
+kXJ8xHHxTn8_000035_000045
+I2YlCISB8TY_000036_000046
+MHQSH4QlQoo_000277_000287
+47oVUabZiic_000000_000010
+5GP0aPugZZA_000000_000010
+mDXWM3kEvfE_000065_000075
+CZOroZ61VSg_000000_000010
+hkoAdY1ZzDU_000004_000014
+KE9NpeyzfmA_000257_000267
+bL3YnakZWHo_000006_000016
+FsrD9In_oBM_000002_000012
+UPujViBOznY_000062_000072
+6dCaO9SMg60_000508_000518
+n97--O-1Gao_000055_000065
+MVeOJmSOV6w_000102_000112
+BPAZjVFei4M_000000_000010
+2qo35QV3w-s_000017_000027
+K1EIjRaLAS8_000034_000044
+WQqCGlffDvQ_000081_000091
+sC-ecZhBWgU_000000_000010
+AOVNu8IC7PE_000059_000069
+7sv__wEPiLA_000002_000012
+PYZbDPuPEmg_000000_000010
+XlNCltb6q2g_000010_000020
+TIIMHjK9GU4_000041_000051
+v-TJuL61DbI_000056_000066
+c54-0dOxrac_000016_000026
+IfX6cllnCSA_000074_000084
+E2_RYbnGF5U_000087_000097
+abcm6SdJ43o_000000_000010
+dC2Ih_JFoOM_000000_000010
+mGu58sIxqj4_000023_000033
+9rudQQiIWac_000022_000032
+lCVFKqQpTIg_000000_000010
+0k8ZyywA8Ac_000004_000014
+tGiBJQ5RhZQ_000077_000087
+--oJV4vFNeI_000003_000013
+_PZtfoG-H1M_000166_000176
+a0OxjWs1ljY_000017_000027
+J5pgXKJL4wg_000012_000022
+1dmiavULOAg_000003_000013
+N1DRDrpALco_000108_000118
+lVwFn9m8Q_Q_000053_000063
+D8MzzIhvBdI_000005_000015
+lVoyp938QBs_000040_000050
+D3EeiT6hZI4_000104_000114
+74SXYejRo-Y_000000_000010
+brJ2L7pbeA0_000108_000118
+wR3cCcmP0SQ_000028_000038
+0AaSiIVdKXU_000027_000037
+ZSiSZX5UNqY_000015_000025
+kSmmIhj24cc_000004_000014
+iBKnfBpxUII_000055_000065
+M4jq4TZm8R4_000044_000054
+xeZr-gqjBYY_000072_000082
+sKgJFd0bMgg_000033_000043
+ecnHGeTB4VY_000008_000018
+CZfe0PkjvA4_000025_000035
+ZupvPUGTifY_000000_000010
+hVrvXFrYU6I_000004_000014
+2mgdkontHv4_000041_000051
+wgi2kzp1js8_000074_000084
+iGn7dk9hxN8_000004_000014
+RYz48Rc385M_000151_000161
+TG7dCEc9PHA_000028_000038
+BgxgcBNT1eA_000191_000201
+67nxVB56psY_000057_000067
+OpDii8AnOcQ_000004_000014
+7SEew824P2A_000164_000174
+1HDx1DeSq6A_000136_000146
+PHEyn4-rJSA_000011_000021
+bI6t7z9IYVQ_000006_000016
+-dLVSg5JvxY_000022_000032
+r917E_NCmgo_000027_000037
+DmjsEgVPK34_000073_000083
+9fygsDgFWJE_000001_000011
+t9sQHL2ypHs_000097_000107
+eWQWYDoLVXs_000195_000205
+Rgvug46euIE_000114_000124
+xr4wTn-GDT8_000085_000095
+sBJk5li0O5o_000003_000013
+BdWCltVcdxk_000015_000025
+q6dyWFq3-LE_000000_000010
+IDgQ4Oy-hrg_000053_000063
+lSaYiQRMtjs_000018_000028
+gfHCbDqMmm4_000047_000057
+NcuHH57heWE_000008_000018
+ZAH-BX96pmA_000002_000012
+66NEJHDxIDk_000068_000078
+Wojf4uAaFV8_000057_000067
+DUmFFNmfgAg_000013_000023
+MSzj5Rl1K5c_000020_000030
+sWCaswVL01Q_000101_000111
+l5RIYpoxCS0_000020_000030
+bkU7KncK3e0_000077_000087
+ffvtOV7En_c_000003_000013
+uqibGSe0vNw_000051_000061
+mL4jKEZDAFc_000015_000025
+n6lUXDwL4Y0_000022_000032
+IiATZMBx1eY_000032_000042
+9iytLJcY9gk_000005_000015
+9jgih4uYH2E_000000_000010
+4x0kKZY_IhE_000057_000067
+H4yFkDFjPCQ_000027_000037
+mSvYTWkVBJA_000008_000018
+77bGWKIfsvk_000061_000071
+ym2ehgmrqlg_000000_000010
+76qMNUzfX7w_000097_000107
+6WtcpH6IkIc_000058_000068
+P_L4Tuvc3l0_000084_000094
+_YWpv2_K8Pk_000054_000064
+Mf7NHE3AL0k_000004_000014
+WOtjzZC0JPI_000001_000011
+yF34Pg30MiI_000308_000318
+PWWB3RDEVHc_000038_000048
+-gsRLvAO4mw_000022_000032
+A7DMjBZyvDY_000090_000100
+wN4GxQ2566M_000003_000013
+HeEsL9yTiKg_000033_000043
+5Jf_z8s8ctU_000012_000022
+LcQ3cShXEWE_000011_000021
+EiRZjAB5JOY_000002_000012
+WEcPPLZ3-JA_000114_000124
+3IfSTN-Eptg_000022_000032
+KCSaZGVBHkU_000147_000157
+LWQ_cBRP7yA_000034_000044
+URrS4GtGQmE_000167_000177
+6ZojdU1fPOo_000001_000011
+y3OSgyt346E_000000_000010
+jE4OreT7B9I_000377_000387
+tXjOdvOPSc4_000005_000015
+y5y1k6eJXyA_000021_000031
+Jo79VXKw1uo_000000_000010
+pCWdCVKE2qo_000000_000010
+Kr3sO1eqVEs_000069_000079
+i-YJVFCh0zg_000028_000038
+AwsTvv5mfeE_000009_000019
+PmJW9OWTibQ_000230_000240
+Xw-WT0yUimg_000334_000344
+Uws0FYxwS90_000158_000168
+MU0HXzaN7qo_000040_000050
+NJu5gpi6W50_000325_000335
+YAmbVR1n56c_000144_000154
+tOgVE8ModME_000003_000013
+3g0zSuxMIkQ_000051_000061
+kCk-RVcoR7g_000059_000069
+B5b2rBinfCw_000024_000034
+qVoiEgXxOTU_000001_000011
+dBAT_3nPgX8_000076_000086
+etShLTcatJs_000121_000131
+xFO0lEzQkQI_000036_000046
+5irp8ASgdhQ_000029_000039
+2L8B__meOLE_000020_000030
+Zrhf5rNcLcA_000055_000065
+xUK01b0W4_I_000013_000023
+HzO-QXUsHb4_000083_000093
+ObYmv1FIGVU_001263_001273
+-Si5SfAGWek_000000_000010
+OU5dJpNHATk_000148_000158
+tXBjXHtckDY_000032_000042
+TUOIarQEnRM_000009_000019
+tec79sCOXyA_000012_000022
+IFXyeVwqIoo_000000_000010
+nJVM_LYlZK8_000015_000025
+yFSBC-iXnqA_000051_000061
+144kxqKEkNE_000000_000010
+qdKbgwpgh04_000032_000042
+h5jF0HR-J1g_000003_000013
+MX1hcxfiltU_000032_000042
+FCZQSGXsvw4_000091_000101
+x6IAyph6dMc_000032_000042
+WQZ1rwEzGEk_000000_000010
+LaaPuezxneg_000078_000088
+nj3KLfg5Kbg_000003_000013
+-dHjiNfYz4E_000015_000025
+Nn7ypmEaad0_000014_000024
+-l309dqYR-8_000113_000123
+l6jMv1FfGhM_000001_000011
+O1PXECSNwr4_000034_000044
+FWuFkDZUY5w_000093_000103
+oCkPMmVif2k_000051_000061
+3uLjjmc3Azw_000059_000069
+q2-YgfjojXw_000018_000028
+-myPMnDFa4Q_000003_000013
+Pj8051a9yj4_000000_000010
+LLGQJhkSyCY_000441_000451
+7WOty3qVdts_000000_000010
+q8aCC3LSlgE_000027_000037
+rjHCBP--jMo_000195_000205
+Hgo7xkutPno_000059_000069
+xnyOA58A07Q_000127_000137
+zmDrZGRmHXM_000018_000028
+n5Fr76ZUlrM_000004_000014
+c11YrmfQ14Q_000092_000102
+ZgBm7w1CRU4_000000_000010
+2HnBaGajUi8_000083_000093
+vnSld6eV0M4_000000_000010
+BYLMB5s0M8Q_000168_000178
+XaCg7OXyglk_000034_000044
+6j9BYhLmFvg_000000_000010
+77aDh42ddw8_000209_000219
+x7EIJEzNGlQ_000004_000014
+NFytp8srWCg_000000_000010
+PMpVRRs6NzI_000016_000026
+DDwSxyD5A1A_000102_000112
+Im7rGy0xCaM_000009_000019
+L9BJsQyrJbo_000089_000099
+VCdsrK2qNeA_000134_000144
+yfKm2hz9psg_000000_000010
+eWgy8eG3stA_000164_000174
+6FqS59XxFNY_000003_000013
+-VatMZ0ztMk_000000_000010
+q3cRM4TVVRI_000000_000010
+VXrzA3M1Erc_000068_000078
+LCdm6pFrmtw_000015_000025
+c68Z9cjgVsk_000267_000277
+3dHsAg2A2ms_000008_000018
+01UETgu-H60_000035_000045
+h3wDKrgNzC8_000016_000026
+1ah8avfU5YE_000053_000063
+uhziJEu3Xmo_000026_000036
+j0-SQa7igwI_000188_000198
+rFHugHoOTcc_000034_000044
+hSGR05xCpL0_000077_000087
+5sBSi4RPXKc_000113_000123
+tAffztlKpiI_000035_000045
+8EBYQucNo1Q_000057_000067
+xwKuZ811HRw_000558_000568
+aD-2SrdN-Is_000184_000194
+4wcWJqEyTSE_000064_000074
+ZXvgHRz92h4_000056_000066
+xtMTgyNduLc_000180_000190
+ogA3nkMl0Q4_000001_000011
+bnSYqx_JBaI_000001_000011
+COnYleGIBw0_000025_000035
+cHJJtEVn1os_000019_000029
+W8sGcM-SRIw_000000_000010
+1onZf1b-az4_000000_000010
+4eGnWcA-lo4_000026_000036
+URLxWfi96Os_000029_000039
+mkoRSlkknhM_000070_000080
+9lrW9DpfkHk_000044_000054
+qAY_lryy0j4_000013_000023
+q8Tklyq_rbA_000116_000126
+OmoC3X888rg_000001_000011
+wnfbsZt7vRs_000009_000019
+KgE1nhFYH4k_000022_000032
+mM8A7RHUtlI_000064_000074
+XbKnoNtqSEw_000000_000010
+ZQj3kEu6q1I_000383_000393
+plIJXZF0eqk_000228_000238
+1rsLh2kG96A_000106_000116
+f8Pp5JNDlnI_000287_000297
+82_AFzsFnhs_000052_000062
+PbAstXAC1f0_000041_000051
+yCMBMuxp7hE_000063_000073
+5UMVOoEoMEY_001542_001552
+SBHh3bkii70_000014_000024
+8Wfh9zIzLfk_000045_000055
+958745Brjw8_000049_000059
+hd7dxyjovCY_000031_000041
+B8TZJ94CcIA_000000_000010
+4ebFdrC9Fec_000020_000030
+JYdMmbtYKDQ_000004_000014
+hREynPJ5mHE_000000_000010
+Y3sWpSIxS1I_000075_000085
+g688nwyO2lY_000028_000038
+K-Nokzt_Zqc_000284_000294
+Ry_mhLjzBDw_000004_000014
+5WFUemuKN5Y_000005_000015
+5qqV2ZnH_4I_000030_000040
+wvho1p1E_nc_000010_000020
+kf5-mcJ32Ss_000038_000048
+mDzcJdO9Wfg_000000_000010
+SvuscI1XA-I_000055_000065
+LgZt6EpY4G4_000015_000025
+jnnpnL0pxuY_000000_000010
+qQWOAoWM60M_000163_000173
+0-3yl2_Myso_000003_000013
+Rz5umsLIZTA_000056_000066
+Y9WjlhlcmJs_000003_000013
+lQcirD6UngU_000012_000022
+NJbB5BA7hwk_000003_000013
+hJ2SPDIZH3U_000059_000069
+Vqzf8Gkq7OA_000129_000139
+D1ddrQYAlhw_000001_000011
+x6wRCc5J5eY_000066_000076
+U0bYM2bPTKM_000006_000016
+PcdvVIsO2eY_000061_000071
+FinRO8rSgAY_000000_000010
+wH0GUySIQ_s_000004_000014
+t_E8gEnLtdY_000079_000089
+OPjonSK9blk_000018_000028
+xxqkCP1oTis_000223_000233
+E3FG1SdjzVk_000021_000031
+6kkPlWASbbA_000368_000378
+bIiDiDUl2RU_000009_000019
+QqBS6yJuBog_000000_000010
+QyMTEHd-VCc_000004_000014
+b8vAhQSYN-g_000000_000010
+w4zEea_xufs_000605_000615
+EZX06L1OsQU_000016_000026
+fh64vbutTrI_000141_000151
+syqGFVCbDWQ_000003_000013
+x2fEWmcJXjg_000008_000018
+COrYGNIyI7Y_000230_000240
+Yqd0U012_Hw_000368_000378
+nflG2ce8VY8_000294_000304
+9dpvCH9fcIc_000343_000353
+N0jrPWhNhwU_000004_000014
+HCvhz5HkpF4_000078_000088
+ZjzhQBpqqSg_000073_000083
+pYKexDBJAYA_000004_000014
+5rZzxIgLlPU_000000_000010
+GpI8DSsvwXI_000044_000054
+fBcKhmdWCvE_000012_000022
+aVWB66MJ-uA_000010_000020
+11DNgvKLRHM_000078_000088
+flBrhLZdr98_000000_000010
+nFqTpZAo3qY_000027_000037
+tv5sUDmBmqU_000023_000033
+EPTwkPkjpwU_000087_000097
+hq7oXt2kveE_000000_000010
+LdWlQFqqits_000171_000181
+1zSg9oighVw_000000_000010
+uNLTuNK8skM_000054_000064
+v7yobVKtMi4_000052_000062
+N6z7pUTrTt0_000000_000010
+6ScXEmM8JO4_000145_000155
+UPoWxo4Fogw_000002_000012
+AYdb1XhQXGo_000023_000033
+Ky49iSd6zDs_000014_000024
+CvSIs_R_BEk_000082_000092
+de1rSoht9t4_000002_000012
+y7hn2Ya16ug_000178_000188
+Y7CZDrJ6Jv4_000000_000010
+nwDKOJHMHhs_000023_000033
+0Q_QyVrbQKI_000015_000025
+P7CgvqGhidw_000073_000083
+Yio9nO2UuuA_000052_000062
+8VZCvv0h3DI_000032_000042
+ucI2PxYGTmI_000000_000010
+CiRq6h7RDBE_000004_000014
+Tdz2yMXJcZI_000003_000013
+ZoGtMtjjPMQ_000087_000097
+DcYqRCy3LsI_000046_000056
+WcxNna60adM_000062_000072
+gw2PjMz-TJI_000028_000038
+jiZgpSw9cKQ_000035_000045
+ONFppy-AC0M_000082_000092
+sKUrb7gV9XE_000003_000013
+YkYoZyGVzCg_000000_000010
+EU5g-Pi0a24_000153_000163
+-5wvw64KXqQ_000000_000010
+X8Tzn9cdCiA_000043_000053
+GRWpczd858Q_000203_000213
+Xqso_EmafEg_000003_000013
+u5SW9SFXBTQ_000032_000042
+JBGTIJbfrPo_000077_000087
+8hZdfjj3cU4_000000_000010
+7M3utA5j6vU_000002_000012
+IDh_IR5L3yk_000000_000010
+vI8Vp2-gfiU_000014_000024
+hceOcG3ZgGk_000073_000083
+JeNGL6M9Bok_000013_000023
+dqh0qThZUjE_000054_000064
+ng8f7_uedZ4_000038_000048
+IeDin6oB-IY_000003_000013
+AE66r-8pfLE_000176_000186
+p2s5fet6Xu0_000068_000078
+x6T517yfG54_000038_000048
+mZGCStZvP18_000027_000037
+oVeHcRapPao_000429_000439
+N8pzeulGgPI_000231_000241
+2v75xvVshtQ_000000_000010
+e1DfuOIVBho_000000_000010
+ofhjWgPguFI_000002_000012
+G91ngyXp6S0_000010_000020
+m0R5rkIi6VM_000077_000087
+rc3aO3TyAzg_000000_000010
+Iz5nOXvBzWM_000009_000019
+IWuwX0hg6sw_000051_000061
+E8_pIIAr90c_000000_000010
+cTx8W_JL_D8_000020_000030
+eMhRkRpOEZw_000197_000207
+LDun0fTcT4w_000087_000097
+ysIr-YTnCN8_000198_000208
+XKxKNnrLloM_000000_000010
+bTT-Lk4Hc88_000208_000218
+W-hlYP6vj3M_000044_000054
+8mdhjvJ9jE0_000063_000073
+pBy55jurdRw_000093_000103
+W9LR5Z9C33E_000104_000114
+3-UHHYrkjGM_000001_000011
+_lR3PdrCkVU_000000_000010
+2bA0vJGml6Q_000005_000015
+ZlWcMe2ZJCA_000011_000021
+F_WeSA5gy2Q_000041_000051
+4-9_szZr2xo_000015_000025
+d1xqBCytAok_000008_000018
+R-icd8119D4_000026_000036
+878gvnBSZlg_000367_000377
+Vx2RMyFUCrU_000037_000047
+Mrlqkm6wzW0_000023_000033
+8IqHQwVqg8g_000076_000086
+YbLmMm66uUk_000192_000202
+_2zXAtOKXm0_000004_000014
+igA7TJcaxqo_000056_000066
+TeCuiDh9CxY_000075_000085
+u40B4vhkPF0_000081_000091
+wKhO5-BMphQ_000000_000010
+gCGT2L5jJoo_000085_000095
+FIO-_zM7Rac_000054_000064
+f_KuNcFykFc_000092_000102
+8yoMHlEeZXU_000019_000029
+QxtqFjvbeKM_000321_000331
+wFuL-cskQn4_000020_000030
+m8bbW5gbYVw_000178_000188
+XKJbWSZLI6E_000027_000037
+HuVxgC3XBco_000040_000050
+wnQOc9Q5oC0_000002_000012
+5ZBwM_KsOpo_000064_000074
+Dbzm5Gl6gWA_000017_000027
+p7Yt39yhPvY_000075_000085
+MsG96pPLUI8_000012_000022
+i6bLvWT5n0A_000000_000010
+kpcrsFCJMf4_000067_000077
+PzD2BkZye2U_000013_000023
+CQ1_ovjb-Ho_000258_000268
+KwCBcXCqOy8_000074_000084
+E2QZ_ErlrJ4_000022_000032
+UBjwOghFbvY_000012_000022
+xPAhqvmiaWI_000100_000110
+GFDeZ1M-j2E_000001_000011
+m19Citzz748_000232_000242
+DYcbS0QgOs8_000003_000013
+vk506HLk254_000000_000010
+enEATg4tI6I_000116_000126
+ZYH4ja41BQs_000003_000013
+ZmvFrZKCxt0_000106_000116
+2DfK16gO5mc_000019_000029
+AHXpkJSLqg4_000460_000470
+dCVZadCffts_000012_000022
+klbIHNGPYR4_000024_000034
+socL3CLqXuA_000007_000017
+VpUVzgDEj9U_000005_000015
+Xr3Tle4trX8_000000_000010
+leWjhZKaK7Q_000350_000360
+QCWm8MqlUO0_000005_000015
+W6CY4iqLdSY_000000_000010
+Adspplz782U_000015_000025
+KbqWQKStExk_000003_000013
+NFeh8RMi7H8_000043_000053
+7mP4mouloV0_000011_000021
+Tka7TzOO3EE_000086_000096
+9XkicLU0ZRQ_000000_000010
+jRvQ6MBC29I_000018_000028
+VnDRckszp7c_000006_000016
+Q80s-wLw6kA_000014_000024
+fIXf9Iv_w9M_000004_000014
+QC5Zscj3reE_000016_000026
+JJOkXFxcw34_000167_000177
+NOhVk13afn4_000008_000018
+8nceWpmAE1c_000004_000014
+s6uinCc6itc_000007_000017
+lRp-y4ezD3A_000067_000077
+F5JTPUwBDw4_000405_000415
+h0rWRIF1Si4_000023_000033
+jjcVjm790-s_000574_000584
+j4POnrecrIc_000113_000123
+gs45ojguZXY_000000_000010
+14TXmFXcnBY_000028_000038
+zgV_kYzVyJY_000003_000013
+w8-DVGaU1Cg_000005_000015
+RI1-69UIurQ_000035_000045
+4hgh-ruFH0c_000113_000123
+qWr7plHuEtE_000024_000034
+TzfkS-SPp30_000013_000023
+MrI8XlHC4p0_000000_000010
+HM4tiCR6BGA_000003_000013
+_gdT1bgS-68_000131_000141
+lllqzohXPD8_000001_000011
+CZykUx8OAbU_000067_000077
+05DTHPMVt_I_000005_000015
+ZFX7oYdu_hI_000009_000019
+lgPslaxBQt0_000000_000010
+vhoawk6MPwM_000442_000452
+QoyIT1L54Ks_000004_000014
+ZGi_1YXNkvw_000006_000016
+9vvv9vfDiuo_000147_000157
+0w7jPdjFhfg_000336_000346
+nKhSp29wB8c_000035_000045
+us_pXO_vwPc_000031_000041
+RhRtKsipSzM_000002_000012
+VlTbwUyIm-k_000072_000082
+5P6Q57A9OUk_000284_000294
+3g6vYQ3Qp8U_000003_000013
+qS2-xjUoyxk_000013_000023
+Z2nrv3qiSZQ_000152_000162
+4eeA-gswYjc_000019_000029
+JBmRfj_N6EA_000000_000010
+9E6_4PgjC50_000010_000020
+MB38Nepx5Gs_000006_000016
+5v_7XLNUrc0_000008_000018
+9rs1Yvxh8H4_000030_000040
+kjCtZfFL8Wk_000128_000138
+q3Yrx7-5W9I_000016_000026
+M4WxnBb8dww_000000_000010
+eJ_SjjpBYE4_000139_000149
+F6tyMvJJvUE_000185_000195
+Z_cdeyDDTHs_000141_000151
+6oxymlT4EU8_000000_000010
+D7R6eFwMd4Y_000041_000051
+7namAi4s1Kc_000021_000031
+aB2vxMeQfG8_000052_000062
+LmP6Y6rEptg_000021_000031
+fRR-rFGoVX8_000048_000058
+VEKFBUXZgBY_000011_000021
+Qol81g7sOz4_000018_000028
+x-uWz2-2WkI_000056_000066
+-6yaNVdS-2E_000000_000010
+0elHrwxt_bI_000004_000014
+meoWZ1SQ0Xo_000003_000013
+Y7fRiJkjnno_000000_000010
+ZJD0IvXAmvo_000001_000011
+O293sucZTDc_000009_000019
+U9QFI8-Q2sA_000024_000034
+IkxOgDBCdug_000044_000054
+EkT5y-2ie94_000238_000248
+i1Zf3AUlaL8_000083_000093
+2cLWIXDi3ks_000077_000087
+djydGQkk1Ck_000000_000010
+dOc89368npY_000214_000224
+4A-hQHF5wFc_000047_000057
+vYUfJD176IU_000122_000132
+Td1A-OQM3fU_000002_000012
+swDObGd9wd4_000025_000035
+zFSdiCg9Rjs_000079_000089
+pKEgGe8lWnA_000007_000017
+9npPGoZ_g1M_000029_000039
+jEeV0LPXPxI_000015_000025
+NDfVG35vYmw_000005_000015
+1DAtchstb_4_000004_000014
+H5KcYdDzf9c_000176_000186
+2FgAQrOe4lg_000208_000218
+u94z2C0J5Fo_000000_000010
+cxYfDw3qt1M_000195_000205
+SkqvHnlsnVE_000023_000033
+WXt6Xqc9st0_000011_000021
+6ODMKujAWhw_000158_000168
+wPeWVPrOl8A_000004_000014
+8SZHEgS1d50_000020_000030
+DoOiOKwy1gk_000050_000060
+ym1Eu_YVj4s_000036_000046
+O0ymrFvrgYY_000130_000140
+NVDgaixYPgA_000031_000041
+4fRGzKO1KLU_000000_000010
+YLJaiKWGQys_000139_000149
+M5aBsf_IGvc_000067_000077
+6n4j0MoLDww_000017_000027
+F7yII1xbF-k_000465_000475
+QqlTEhzK_Gw_000015_000025
+1SzM6XlB81s_000000_000010
+SeCwevKut3k_000183_000193
+SQtvychI6Bk_000014_000024
+flCiy8akWyw_000051_000061
+BYoIZZLnizg_000008_000018
+UAHoOHpQWlA_000053_000063
+zr7y21HASOs_000021_000031
+LtKnLVqr8i8_000217_000227
+T05qWt3oxxc_000050_000060
+H8rMDUDnqTE_000032_000042
+HAezZ-g5ntY_000123_000133
+nG8A-M9tvtA_000100_000110
+bYq9JMbV4Vs_000036_000046
+ENorOsdty00_000347_000357
+-d7gxstQux4_000067_000077
+i4K7rWrsgyI_000038_000048
+4lPNXqiS6d8_000045_000055
+wG8u7evqvhM_000014_000024
+o7oG4LKLrdo_000390_000400
+mF5qk0efJbo_000000_000010
+8f0U0oGyugQ_000007_000017
+NMd_TLbm9d8_000156_000166
+NAiHI-77OQk_000098_000108
+0Je9lZCyNgk_000016_000026
+m0mo6zbBkGc_000001_000011
+K-r_Tj7uFGw_000007_000017
+OopGmHDZI2M_000090_000100
+66ZIRfYTeXg_000149_000159
+m1XBfBhMSK0_000076_000086
+yUvBOFF9OPw_000000_000010
+4nJ07xfDip0_000003_000013
+a4ROubm0cIc_000000_000010
+MgZTNN8jQbo_000013_000023
+W8Fbjslydzs_000039_000049
+b3f4C0ngOwE_000000_000010
+_xE_0yhuehs_000163_000173
+V8-rdFVhU6U_000096_000106
+4OPAmSlJ31o_000015_000025
+2nnGmrCfXJI_000000_000010
+ghvlnZcOSJs_000000_000010
+10UDBRZ5wnM_000000_000010
+ag8KLqzMZnI_000092_000102
+36bFQ0AdkTc_000000_000010
+hd408sIsIjI_000083_000093
+eu5seBCWFcw_000014_000024
+Nb9tS5UreXk_000154_000164
+JLaFmdOjW-I_000023_000033
+swTrjCn5iPg_000000_000010
+pbneLOUYW68_000018_000028
+wo36qZoOOLU_000002_000012
+-pHrb1FLXfY_000011_000021
+CSMEQvz-OUc_000001_000011
+A8sjbCSzUow_000038_000048
+w7ZRJ7JNJrQ_000009_000019
+mhfVGeD6XxU_000051_000061
+t-COcCPV-T4_000020_000030
+i1A0qQHDBHk_000002_000012
+2N_bg_UqJkA_000064_000074
+9acHiJcAyz8_000039_000049
+uAwY_6l7680_000000_000010
+Vk0PYLTTg3w_000004_000014
+B4EcC9bJv8U_000176_000186
+tRDLInPlu50_000071_000081
+9cha2WxlvSM_000005_000015
+7h3uoKOxYHg_000065_000075
+--uGS0Y4D6k_000009_000019
+i-MlHUUx_j0_000000_000010
+xO81JlPwed0_000100_000110
+19hKnhFMLUs_000008_000018
+wXJShRNmthQ_000018_000028
+z9s8IrLfcFE_000017_000027
+ww-qxWwnN1Y_000011_000021
+29Dj0ISRaMs_000107_000117
+p3_wp0Cq6Lo_000406_000416
+0S3itzH8NBU_000000_000010
+l3BXqg2lP2k_000197_000207
+NhGhlKiC-lk_000049_000059
+KKFHlgwXljM_000011_000021
+-fvtIUVgGtc_000223_000233
+IrrO4-aFvwg_000001_000011
+J8bZhQNBjYY_000015_000025
+oUfAz5k4EQ0_000373_000383
+-CHTxmgMUes_000010_000020
+bUHZAtuW-zU_000049_000059
+QtrShBu6bqA_000102_000112
+mEVYwZxXz68_000006_000016
+epTrdwLVzKg_000075_000085
+nEinWzcCaNc_000404_000414
+9wO0YRODVTE_000043_000053
+DHJB2Kx3fcM_000015_000025
+lcUJjVgqXp4_000000_000010
+DIYiKOjA0v0_000001_000011
+ziRHCx9gvFk_000007_000017
+0hVtX45KhS8_000075_000085
+cEmW-d9dda8_000011_000021
+2fojVBo1tv0_000043_000053
+xaddtCnpkTI_000001_000011
+n1DUmdez1hA_000354_000364
+y7ZyI7aXTGs_000009_000019
+lGTMPfMfXLY_000221_000231
+DOVzrocR2fM_000043_000053
+o8BzuNJOoGk_000033_000043
+-FOfyuyabWQ_000016_000026
+3-kjbGxqSeE_000077_000087
+i-qeC0YDfDM_000080_000090
+KVc_8dAigSs_000024_000034
+8qReQWU-Ffg_000040_000050
+iR35vmsJ994_000063_000073
+C_QP4vOVTrE_000164_000174
+rPI-N-nGhQw_000053_000063
+x0xXPnl_SNE_000021_000031
+mQwjoRqpQro_000103_000113
+WFBA0RxdDqw_000027_000037
+PAJ3uH0tVc4_000025_000035
+dGvMfhUqnz8_000003_000013
+fzeIUdxSoIQ_000022_000032
+ztAfXKZ0ovM_000141_000151
+6QroafCsHfc_000013_000023
+vgrr-N5x7kE_000035_000045
+-2VKVjgNuE0_000000_000010
+hy7xI8-xSOE_000031_000041
+KHGbByOzErk_000207_000217
+6_jT33BqtK8_000038_000048
+WazX6nI7kes_000002_000012
+xn3RnUcBKAQ_000009_000019
+gEWC75dITlc_000003_000013
+BWUegTwLnpU_000007_000017
+WtnQKvOuukE_000081_000091
+k9-EFe0dAFo_000007_000017
+Q_F8YU0vWOA_000052_000062
+vfB6RccLMbI_000118_000128
+xwQmw_alBbU_000002_000012
+_ZIq66VAlJY_000064_000074
+o6NRDIu4qrI_000115_000125
+svVhMowxtT0_000000_000010
+ssn_saANNaI_000336_000346
+810dEynPvOo_000002_000012
+jD1_v1R2raw_000000_000010
+PmOCR_IwCsY_000004_000014
+6LOV6-dkNZE_000251_000261
+A9TcWJnbcWs_000001_000011
+l6CrXrj01GA_000102_000112
+slw7UYOTO28_000023_000033
+6Q4-x1VOous_000043_000053
+-D6-UmndVJk_000001_000011
+pwCguMjX_3A_000002_000012
+L0Mfvzh4peo_000062_000072
+E2croigfy4Y_000017_000027
+3qBLumz-KJo_000279_000289
+5x-3iU4Mjq8_000005_000015
+8Y3K5w3b2CU_000000_000010
+M-B6nIEF3KM_000111_000121
+BMdrFU5SGNI_000028_000038
+n753IrV58Gk_000013_000023
+JlVJ2vA9vmU_000022_000032
+9I8me3WxbzA_000004_000014
+C5yTd8hS6AY_000039_000049
+CXgv232iW3c_000002_000012
+DLr9B2Bi0M0_000008_000018
+xQCi12r6Lfg_000003_000013
+LgHB-PEWODk_000002_000012
+-6tk1dzeCmA_000144_000154
+Q5YXwZAe4i0_000140_000150
+wUVxf6GUTHM_000019_000029
+AMPSyUu1r0U_000199_000209
+hEi8ty_0-JU_000132_000142
+2wIcUAH9yaM_000024_000034
+yDjvtHR_VT0_000015_000025
+B57NAi7KZ78_000008_000018
+DxjGNZXHIDo_000193_000203
+9XGSi2nIY9E_000040_000050
+sh3K9P5UXL8_000006_000016
+JUbuepvei2k_000013_000023
+558iWbgz8RQ_000052_000062
+KX6X-0TVEsI_000015_000025
+VReq0APgHnc_000008_000018
+BrQi6a26vXU_000016_000026
+Y9_AJlP0iUY_000373_000383
+3PKi-hiB7YA_000113_000123
+LvD57lc6WnU_000016_000026
+PcQ3hA5iOPM_000073_000083
+FKjRMG8tnU0_000061_000071
+Kcz7K-1UVag_000000_000010
+nD6aGWeuJbk_000000_000010
+uQk0V2_KliQ_000815_000825
+-8wsYw78yTc_000007_000017
+n15ydcfmZCQ_000090_000100
+yszYp04CCwE_000012_000022
+q21qB87cnos_000111_000121
+TpaHKH6UkDs_000304_000314
+Ms2iNKIsaXU_000050_000060
+e9mZDLGioxU_000104_000114
+l3uAuCYd6HM_000000_000010
+lGGYYuk5994_000119_000129
+wzyuVMClxt0_000040_000050
+xrWfU7q3eXM_000010_000020
+WHgCLj6fgKM_000042_000052
+UArdunmwEdA_000049_000059
+PnNDSAt3kz8_000001_000011
+TR-JsNOeXOE_000255_000265
+yimS7fmRQU8_000002_000012
+tx4wUC8oRJ0_000000_000010
+eN3Ikew2bis_000069_000079
+-MUTGKtPyX0_000063_000073
+Cc0wpiKhLw0_000024_000034
+VNXpHy5Tb_U_000064_000074
+lP45_ktmS1w_000016_000026
+olX9OOSDY1A_000129_000139
+r5NqGSZsN0E_000009_000019
+I44HVnyEKVA_000016_000026
+77Yafjbsrn0_000032_000042
+SLHqXKWlQx4_000030_000040
+L89FX34lmsI_000053_000063
+Xyt9EJ5QLQc_000008_000018
+xrOdt9rrdcM_000003_000013
+75XC6l5Mung_000000_000010
+XoGV-AYJjvo_000042_000052
+6NAwC1njhSs_000000_000010
+qvHzrD45wIM_000181_000191
+uQRBFQp_i_U_000143_000153
+iFcUzlgCHcA_000000_000010
+Z3r5apR3C6Y_000011_000021
+Th5WqxdXDb0_000007_000017
+0OfYCZ3Kiow_000084_000094
+wbqtzX2Mmc4_000033_000043
+TaOqyX1yYDM_000177_000187
+10VlOsVeZb8_000000_000010
+UM-rrG3AN14_000000_000010
+TZ0l4CSYI0Y_000419_000429
+pQW2gNEHG6U_000035_000045
+31Il_bktDaY_000000_000010
+UKf3QMXSOSs_000000_000010
+0OjaGxvpEzo_000026_000036
+Lgtqpb-9rpY_000001_000011
+Td5C8t3IjYk_000000_000010
+Tpv49Chfjks_000077_000087
+gvTxw3L7qQI_000019_000029
+Ug5jVJ-9YS4_000002_000012
+behtgkZntMg_000030_000040
+LkjbmuyqaL8_000083_000093
+YFVR5y_EtH8_000049_000059
+Yv0w8FvXQ-w_000000_000010
+VgqbAvvnbdc_000040_000050
+tPL_uVgo9so_000137_000147
+McNooX_wwF0_000072_000082
+TEKN8M2clbk_000348_000358
+lpEa5Qd5utg_000100_000110
+tEosgxP3KSI_000385_000395
+l-Sq7PUMnW4_000019_000029
+lbSUj_faLsA_000010_000020
+0mvWamNDkB4_000007_000017
+f_lSThX3i_A_000025_000035
+aNN4j-njBes_000009_000019
+R-BfixOEL68_000011_000021
+x32aT1LN_AM_000255_000265
+iBFE2AI4B38_000516_000526
+B3riQBJYQ4w_000442_000452
+OqjsXki0Br0_000091_000101
+CQzUU7-cVck_000006_000016
+RudqvdVN77k_000001_000011
+qN7njQDP_bk_000015_000025
+d38G9PMkHiE_000012_000022
+SHpIilEOwpg_000014_000024
+eptncA3-gjY_000012_000022
+79SK43KrK-4_000016_000026
+jWbhG981nUI_000095_000105
+WFsMi09soZs_000002_000012
+yCt9X4RGgrA_000091_000101
+EJLELzXZ2iY_000125_000135
+l8QnGBveplU_000025_000035
+5IPMlxae7TI_000042_000052
+zsR-xVwn0go_000003_000013
+kzXmdd6Z8U4_000004_000014
+-PxRByksLVk_000738_000748
+E6cnyjaCD3U_000195_000205
+lrbO88eK9is_000005_000015
+d1VB1vA-UsI_000003_000013
+vv8LAxmG-2M_000032_000042
+b1VygfGIpYg_000163_000173
+rpqKHNZd2u4_000004_000014
+1vsk3D51Otc_000000_000010
+vALzTXrF5xk_000022_000032
+whax73x_rh4_000001_000011
+DLEEYdFodkc_000003_000013
+TNkot064mgY_000007_000017
+WQ0Y8NFuaVs_000010_000020
+yeG8vIrsBzw_000016_000026
+O2QjdVG53O4_000068_000078
+TvURKUx1eSQ_000177_000187
+idS_distWcQ_000066_000076
+53Ia-BtpM5A_000002_000012
+ay4oamtb_Bo_000009_000019
+3l8tUCqavPM_000371_000381
+Kpnd15O9aYE_000013_000023
+ymjpBYQkadk_000012_000022
+kmwYoFp51Ww_000153_000163
+heNARpg9oAg_000069_000079
+ziyNnkqbuYA_000002_000012
+YMO199qZ6SA_000182_000192
+TxqD1yP-Sa4_000000_000010
+Xkxjo2RtO6k_000036_000046
+VNulfciEozA_000020_000030
+OPkDRqy3XoM_000008_000018
+TMmdYBamtP0_000038_000048
+iusGKMd54jw_000099_000109
+F6M8pFiD19E_000269_000279
+d1x2oWTC06Y_000140_000150
+99_8K7ds7u8_000015_000025
+uxPQEfYNFPk_000014_000024
+oBWsovXUbbg_000008_000018
+36E29x22tnQ_000018_000028
+tNaFZgjsnck_000046_000056
+uLzU42YMPLw_000005_000015
+_HUQYqzmppk_000000_000010
+d5TYy52mMHk_000006_000016
+VCOx5pHG3Ho_000267_000277
+8qxh1QyYO5s_000026_000036
+BuZdSvfZvaM_000492_000502
+u4_AHcoJWX8_000003_000013
+nba6dbLPKPY_000000_000010
+891_hiMUeFI_000026_000036
+5dRnwv9ZiG4_000041_000051
+WU5KZsG_mQk_000483_000493
+tvl_Jr4ai3M_000193_000203
+HpnuCO7skxc_000107_000117
+fSFSMuhmb7E_000001_000011
+cXsQwIVDsKY_000004_000014
+VzLuzfd15xY_000019_000029
+jnTuUuETpSs_000002_000012
+VLbQ2mlzL_Q_000037_000047
+K2TDPZ3_N-g_000074_000084
+E1v-Q_eyAng_000003_000013
+1OP0uCl3Awc_000000_000010
+CieWVgeB-MM_000000_000010
+8Z4mPltOS5A_000024_000034
+_mffV4nn_vY_000018_000028
+tAn1JD4xFbc_000007_000017
+FlsunhtRy44_000145_000155
+-jToAVyxs-g_000046_000056
+Pxj9gmt4PnM_000000_000010
+EtZA_L8iH8Y_000279_000289
+C2C4a3VMJNw_000000_000010
+M8bk7ZWgQP0_000011_000021
+8Za73dZkPSw_000013_000023
+DtLbnBXrDgs_000001_000011
+TbleNWZsI4g_000024_000034
+pmH650EmzaI_000004_000014
+uPVfS9Ojg_Q_000112_000122
+51RYJAVpZQY_000000_000010
+UPeTYuxFLr0_000464_000474
+_vMrhrBmDhc_000042_000052
+ipGBf8JypGA_000278_000288
+LhzSXvr_Yec_000007_000017
+x0J0c8qcL1A_000000_000010
+nsGgsWfY4S8_000001_000011
+G-L0hmjWScI_000000_000010
+j6zxxFvT-F0_000015_000025
+--gEBElruic_000067_000077
+r9WM1QZcBTM_000031_000041
+baENtPQwXcQ_000048_000058
+0ybnpf_KSwg_000089_000099
+GWOHr1RViOo_000441_000451
+OFqqZU0wIP0_000000_000010
+eanhmmKIolc_000044_000054
+7U7-3kO4pik_000012_000022
+k4JhKErAIcU_000000_000010
+XB0pZG4b2Ic_000424_000434
+VPV_NoRxn9Y_000018_000028
+jN1u2FANVX8_000122_000132
+C8pn8Fa30N4_000088_000098
+Uc3dOM6B1z0_000003_000013
+rm8ir9fmwPs_000089_000099
+DMNkPedEFtc_000108_000118
+H20kn210Ge4_000030_000040
+hMoO-1XY7oI_000000_000010
+tdHELvi9z7U_000006_000016
+gSxbTg_EGz4_000046_000056
+SE5_JeN7g50_000109_000119
+d87bGTzOOWk_000071_000081
+vTSO26j_g3E_000006_000016
+rJBwUrElbkA_000005_000015
+1xS5GcVGBVk_000013_000023
+0DtaFHAH_bY_000000_000010
+xeoH1CgrGNo_000000_000010
+tFtVI4BP_Jg_000000_000010
+8Zody4_Ouos_000000_000010
+08qD_jd4qLY_000066_000076
+PCQVd8rmprg_000020_000030
+7UACcUm3y1g_000006_000016
+d6fdMPKYVK0_000285_000295
+bAar9lhU3-U_000031_000041
+A9_HJgc-Fy4_000012_000022
+XuQ16JZ_OX4_000001_000011
+E14H_dK-fTY_000042_000052
+H8WWknTZUmA_000016_000026
+h6L6_CGk284_000007_000017
+eA1h7zbg9lM_000004_000014
+po5wMUhcWug_000050_000060
+OMK0OJ4f_TI_000000_000010
+hbPpPCe6ou8_000038_000048
+bxxK9LLquRs_000016_000026
+EFiV66yJqpM_000205_000215
+9Ih3vgHzc-o_000000_000010
+RanVrk-mUsc_000142_000152
+JCVQcNTQS2M_000036_000046
+19zOVnkSOeM_000000_000010
+LSLzeVlyYAU_000029_000039
+1uZK-W4S6Is_000000_000010
+VkJHlcbwmIs_000129_000139
+7l20ds7nvA8_002870_002880
+lsuyMZWr_Nw_000042_000052
+vcX_JOjjHGE_000003_000013
+PgyV1sylw4o_000007_000017
+cr7f5GBtVTI_000069_000079
+tEhoqyS5vVc_000234_000244
+2DNwbwWhYgA_000033_000043
+77IRHMWRjwM_000010_000020
+DKqPSRSDPBk_000074_000084
+ch8PWd0CUwc_000415_000425
+cExTd4JuSrQ_000040_000050
+d6_XQqZIO98_000087_000097
+Nkwq2Jp2WaA_000110_000120
+YLHPbG4pe-g_000038_000048
+b-rcZkrdtl8_000043_000053
+L91CNH1RgvI_000002_000012
+Y1NEWHZywDM_000008_000018
+LaGwOQvYY0s_000179_000189
+iYH2ZfbMy-I_000142_000152
+bbu7lxDJ3_A_000001_000011
+iEBq_VR_JSE_003800_003810
+ill9sZOMQtE_000000_000010
+HncTHpgDDoc_000105_000115
+REhRcS9YrzM_000019_000029
+NbZITwhH46E_000079_000089
+3Kaledle0Jw_000040_000050
+ysXgEIDm3nY_000045_000055
+e47kpEV6afo_000000_000010
+2bRXcLakTZQ_000002_000012
+OCJCwHZU31A_000009_000019
+M2JvulLtG40_000012_000022
+By1lQRot9eo_000035_000045
+S9nCwjMbbDk_000068_000078
+jablL-kz594_000010_000020
+5Odj7n4_P-I_000015_000025
+zg_P56oZJEc_000000_000010
+rJokQsBf1as_000077_000087
+-abgu7xMKe8_000151_000161
+G9zN5TTuGO4_000179_000189
+4nRvHwt1Qkw_000002_000012
+o08RX_saVBk_000025_000035
+3jx36HklC_Y_000067_000077
+9c7mU3hyLLw_000000_000010
+eoGt3l3BZ7w_000163_000173
+-uayIf2Y92U_000036_000046
+1iX9rmvzOHI_000039_000049
+4rf7LdbwgTg_000086_000096
+wHQ5wF3oJoo_000057_000067
+0FuVWCWL1n4_000018_000028
+X3JvaX1DYd0_000036_000046
+hkLWjpndOoQ_000018_000028
+VHVZ22Sv_fY_000013_000023
+IH7SR20aCuU_000089_000099
+tSFY2jfmdt8_000028_000038
+uxl6HFwuWEo_000358_000368
+apjEvOG8-Zo_000039_000049
+Iv0XyMqrnw8_000115_000125
+wObc9HsS3Ls_000038_000048
+vv1cVGVcelc_000007_000017
+vuPuOn_Dkm0_000124_000134
+U4lBS3FWvC0_000111_000121
+giUxmMn8Nfs_000129_000139
+BdmLB7IdGrk_000004_000014
+6FiNye9H70M_000000_000010
+R5NmJB0lGmI_000010_000020
+j-AUhHY-9nE_000001_000011
+n2zPcPtF78o_000009_000019
+MaaKJZ6AvY8_000056_000066
+6KbcDK8XE38_000035_000045
+eqK9gnEV8Y8_000156_000166
+w0axq1_jyCs_000008_000018
+UQP8kz5kAKg_000021_000031
+E6cLsQqwcbQ_000040_000050
+5JJV_r6zLlM_000028_000038
+P0khQxBYXFE_000040_000050
+GJmfSgkvH7o_000019_000029
+S2Mkf6NTrM4_000022_000032
+68UzWy9hgVE_000075_000085
+-0cIpwdV-DU_000722_000732
+VGMx_RsSBn4_000085_000095
+4vv-m4xCVbc_000196_000206
+QgFNh6Bx0Xs_000047_000057
+HyThLeZVMy8_000004_000014
+aJxa7KNArB4_000096_000106
+cgmM6SyvIhA_000049_000059
+FEJFDsPci1o_000135_000145
+9epLyjhQonM_000401_000411

util.py

@@ -0,0 +1,75 @@
+import torch
+import torch.nn as nn
+import threading
+from torch._utils import ExceptionWrapper
+import logging
+import torch.nn.functional as F
+
+def get_a_var(obj):
+    if isinstance(obj, torch.Tensor):
+        return obj
+
+    if isinstance(obj, list) or isinstance(obj, tuple):
+        for result in map(get_a_var, obj):
+            if isinstance(result, torch.Tensor):
+                return result
+    if isinstance(obj, dict):
+        for result in map(get_a_var, obj.items()):
+            if isinstance(result, torch.Tensor):
+                return result
+    return None
+
+
+def parallel_apply(fct, model, inputs, device_ids):
+    modules = nn.parallel.replicate(model, device_ids)
+    assert len(modules) == len(inputs)
+    lock = threading.Lock()
+    results = {}
+    grad_enabled = torch.is_grad_enabled()
+
+    def _worker(i, module, input):
+        torch.set_grad_enabled(grad_enabled)
+        device = get_a_var(input).get_device()
+        try:
+            with torch.cuda.device(device):
+                # this also avoids accidental slicing of `input` if it is a Tensor
+                if not isinstance(input, (list, tuple)):
+                    input = (input,)
+                output = fct(module, *input)
+            with lock:
+                results[i] = output
+        except Exception:
+            with lock:
+                results[i] = ExceptionWrapper(where="in replica {} on device {}".format(i, device))
+
+    if len(modules) > 1:
+        threads = [threading.Thread(target=_worker, args=(i, module, input))
+                   for i, (module, input) in enumerate(zip(modules, inputs))]
+
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+    else:
+        _worker(0, modules[0], inputs[0])
+
+    outputs = []
+    for i in range(len(inputs)):
+        output = results[i]
+        if isinstance(output, ExceptionWrapper):
+            output.reraise()
+        outputs.append(output)
+    return outputs
+
+def get_logger(filename=None):
+    logger = logging.getLogger('logger')
+    logger.setLevel(logging.DEBUG)
+    logging.basicConfig(format='%(asctime)s - %(levelname)s -   %(message)s',
+                    datefmt='%m/%d/%Y %H:%M:%S',
+                    level=logging.INFO)
+    if filename is not None:
+        handler = logging.FileHandler(filename)
+        handler.setLevel(logging.DEBUG)
+        handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s: %(message)s'))
+        logging.getLogger().addHandler(handler)
+    return logger