在# 将 4输入分开，构建新的相同模态结合的2输入，2分支
import math
import logging
from functools import partial
from collections import OrderedDict
from copy import deepcopy

import torch
import torch.nn as nn
import torch.nn.functional as F

from timm.models.layers import to_2tuple

from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise
from .utils import combine_tokens, recover_tokens
from .vit import VisionTransformer
from ..layers.attn_blocks import CEBlock

_logger = logging.getLogger(__name__)
                            

class VisionTransformerCE_S(VisionTransformer):
    """ Vision Transformer with candidate elimination (CE) module

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929

    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
        - https://arxiv.org/abs/2012.12877
    """

    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init='',
                 ce_loc=None, ce_keep_ratio=None):
        super().__init__()
        if isinstance(img_size, tuple):
            self.img_size = img_size
        else:
            self.img_size = to_2tuple(img_size)
        self.patch_size = patch_size
        self.in_chans = in_chans

        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU

        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches

        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)
        self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        blocks = []
        ce_index = 0
        self.ce_loc = ce_loc
        for i in range(depth):
            ce_keep_ratio_i = 1.0
            if ce_loc is not None and i in ce_loc:
                ce_keep_ratio_i = ce_keep_ratio[ce_index]
                ce_index += 1

            blocks.append(
                CEBlock(
                    dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                    attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                    keep_ratio_search=ce_keep_ratio_i)
            )

        self.blocks = nn.Sequential(*blocks)
        self.norm = norm_layer(embed_dim)

        self.init_weights(weight_init)
    
    
    def forward_features(self, z, x, z1,x1,
                         mask_z=None, mask_x=None,
                         ce_template_mask=None, 
                         ce_keep_rate=None,
                         return_last_attn=False
                         ):
        # 分支1 处理流程
        B = x.shape[0]
        x = self.patch_embed(x)
        z = self.patch_embed(z)
        x += self.pos_embed_x
        z += self.pos_embed_z

        if mask_z is not None and mask_x is not None:
            mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_z = mask_z.flatten(1).unsqueeze(-1)

            mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_x = mask_x.flatten(1).unsqueeze(-1)

            mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
            mask_x = mask_x.squeeze(-1)

        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed

        if self.add_sep_seg:
            x += self.search_segment_pos_embed
            z += self.template_segment_pos_embed

        x_combined = torch.cat([z, x], dim=1)    # student  64+256

        if self.add_cls_token:
            x_combined = torch.cat([cls_tokens, x_combined], dim=1)
        x_combined = self.pos_drop(x_combined)
        
        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]

        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
        global_index_t = global_index_t.repeat(B, 1)

        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
        global_index_s = global_index_s.repeat(B, 1)

        removed_indexes_s = []
        for i, blk in enumerate(self.blocks):
            x_combined, global_index_t, global_index_s, removed_index_s, s_attn = blk(x_combined, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
            
            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s.append(removed_index_s)
        
        x_combined = self.norm(x_combined)
        #######  新增分支2处理流程
        # B = x.shape[0]
        x1 = self.pos_embed_event(x1)
        z1 = self.pos_embed_event(z1)
        x1 += self.pos_embed_x
        z1 += self.pos_embed_z

        # 假设mask处理相同
        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed

        if self.add_sep_seg:
            x1 += self.search_segment_pos_embed
            z1 += self.template_segment_pos_embed

        x1_combined = torch.cat([z1, x1], dim=1)    # student  64+256
        if self.add_cls_token:
            x1_combined = torch.cat([cls_tokens, x1_combined], dim=1)
        x1_combined = self.pos_drop(x1_combined)
        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x1.device)
        global_index_t = global_index_t.repeat(B, 1)

        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x1.device)
        global_index_s = global_index_s.repeat(B, 1)

        removed_indexes_s1 = []
        for i, blk in enumerate(self.blocks):
            x1_combined, global_index_t, global_index_s, removed_index_s, s1_attn = blk(x1_combined, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
            
            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s1.append(removed_index_s)

        x1_combined = self.norm(x1_combined)

        return x_combined, x1_combined,s_attn,s1_attn

################## 双分支结构 完成 

# 扩展 forward函数，以实现双分支结构
    def forward(self,z,x,z1,x1,
                ce_template_mask=None, 
                ce_keep_rate=None,
                return_last_attn=False):
        
        x_combined, x1_combined,s_attn,s1_attn = self.forward_features(z, x,z1,x1,
                                                                       ce_template_mask, ce_keep_rate,
                                                                       return_last_attn)
        return x_combined, x1_combined,s_attn,s1_attn


def _create_vision_transformer(pretrained=False, **kwargs):
    model = VisionTransformerCE_S(**kwargs)
    
    if pretrained:
        if 'npz' in pretrained:
            model.load_pretrained(pretrained, prefix='')
        else:
            checkpoint = torch.load(pretrained, map_location="cpu")
            missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
            print('Load pretrained model from: ' + pretrained)

    return model


def vit_base_patch16_224_ce_s(pretrained=False, **kwargs):
    """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    """
    model_kwargs = dict(
        patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
    return model


def vit_large_patch16_224_ce_s(pretrained=False, **kwargs):
    """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    """
    model_kwargs = dict(
        patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
    return model中提取出相同的操作，解读以下整个流程