分析：import math
import logging
from functools import partial
from collections import OrderedDict
from copy import deepcopy

import torch
import torch.nn as nn
import torch.nn.functional as F

from timm.models.layers import to_2tuple

from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise
from .utils import combine_tokens, recover_tokens
from .vit import VisionTransformer
from ..layers.attn_blocks import CEBlock
from .new_counter_guide import Counter_Guide
from .ad_counter_guide import Counter_Guide_Enhanced
from .crossReplace import CrossReplaceTransformerLayer, CrossReplaceTransformer
_logger = logging.getLogger(__name__)
                            

class VisionTransformerCE(VisionTransformer):
    """ Vision Transformer with candidate elimination (CE) module

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929

    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
        - https://arxiv.org/abs/2012.12877
    """
    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init='',
                 ce_loc=None, ce_keep_ratio=None):
        super().__init__()
        if isinstance(img_size, tuple):
            self.img_size = img_size
        else:
            self.img_size = to_2tuple(img_size)
        self.patch_size = patch_size
        self.in_chans = in_chans

        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU

        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches

        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)
        self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        blocks = []
        ce_index = 0
        self.ce_loc = ce_loc
        for i in range(depth):
            ce_keep_ratio_i = 1.0
            if ce_loc is not None and i in ce_loc:
                ce_keep_ratio_i = ce_keep_ratio[ce_index]
                ce_index += 1

            blocks.append(
                CEBlock(
                    dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                    attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                    keep_ratio_search=ce_keep_ratio_i)
            )

        self.blocks = nn.Sequential(*blocks)
        self.norm = norm_layer(embed_dim)

        self.init_weights(weight_init)
    
        # 添加交互模块counter_guide
        self.counter_guide = Counter_Guide_Enhanced(768, 768)
        ############# 添加 cross-replace 模块
        self.base_layer = CrossReplaceTransformerLayer(d_model=embed_dim, nhead=num_heads, theta=(0.025, 0.031))
        num_layers = 2
        replace_start = 1  # 开始替换的层
        replace_end = 2    # 结束替换的层
        self.cross_replace_transformer = CrossReplaceTransformer(self.base_layer, num_layers, replace_start, replace_end)

    
    def forward_features(self, z, x, event_z, event_x,
                         mask_z=None, mask_x=None,
                         ce_template_mask=None, ce_keep_rate=None,
                         return_last_attn=False
                         ):
        # 分支1 处理流程
        B, H, W = x.shape[0], x.shape[2], x.shape[3]

        x = self.patch_embed(x)
        z = self.patch_embed(z)
        z += self.pos_embed_z
        x += self.pos_embed_x        

        if mask_z is not None and mask_x is not None:
            mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_z = mask_z.flatten(1).unsqueeze(-1)

            mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_x = mask_x.flatten(1).unsqueeze(-1)

            mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
            mask_x = mask_x.squeeze(-1)

        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed
        if self.add_sep_seg:
            x += self.search_segment_pos_embed
            z += self.template_segment_pos_embed
        x = combine_tokens(z, x, mode=self.cat_mode)
        if self.add_cls_token:
            x = torch.cat([cls_tokens, x], dim=1)
        x = self.pos_drop(x)

        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
        global_index_t = global_index_t.repeat(B, 1)
        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
        global_index_s = global_index_s.repeat(B, 1)
        removed_indexes_s = []
        # # 分支2 处理流程
        event_x = self.pos_embed_event(event_x)
        event_z = self.pos_embed_event(event_z)
        event_x += self.pos_embed_x
        event_z += self.pos_embed_z

        event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)

        if self.add_cls_token:
            event_x = torch.cat([cls_tokens, event_x], dim=1)

        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]

        global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
        global_index_t1 = global_index_t1.repeat(B, 1)
        global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
        global_index_s1 = global_index_s1.repeat(B, 1)

        removed_indexes_s1 = []

        for i, blk in enumerate(self.blocks):
            # 第一个分支处理
            x, global_index_t, global_index_s, removed_index_s, attn = \
                blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
            # 第二个分支处理
            event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \
                blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)


            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s.append(removed_index_s)
                removed_indexes_s1.append(removed_index_s1)

            # 在第1层增加counter_guide模块,验证早期融合效果
            if i == 0 :
                # 新增原始特征（在counter_guide之前的特征，便于后续的loss计算）
                A_original_x = x.clone() # 保存原始x特征 （RGB）
                A_original_event_x = event_x.clone() # 保存原始event特征 （event）

                # 引入counter_guide进行特征增强
                enhanced_x, enhanced_event_x = self.counter_guide(x, event_x)

                # 保存经过counter_guide增强的特征
                A_enhanced_x = enhanced_x  # 保存增强后的特征
                A_enhanced_event_x = enhanced_event_x  # 保存增强后的特征
                # 将增强后的特征与原特征相加
                x = x + enhanced_x
                event_x = event_x + enhanced_event_x
                
            ########### 增加 cross-replace 模块
            if i == 1:
                src1 = x.transpose(0, 1)
                src2 = event_x.transpose(0, 1)
                x, event_x = self.cross_replace_transformer(src1, src2, replace=True)
                x = x.transpose(0, 1)
                event_x = event_x.transpose(0, 1)
        # 应用LayerNorm归一化处理
        x = self.norm(x) 
        event_x = self.norm(event_x)

        x_cat = torch.cat([event_x,x], dim=1)
        x = x_cat
        aux_dict = {
            "attn": attn,
            'attn1': attn1,
            "removed_indexes_s": removed_indexes_s,  # used for visualization
            'removed_indexes_s1': removed_indexes_s1,
            # 在aux_dict中返回原始特征和增强特征
            'A_original_x': A_original_x,
            'A_original_event_x': A_original_event_x,
            'A_enhanced_x': A_enhanced_x,
            'A_enhanced_event_x': A_enhanced_event_x,
        }
        
        return x, aux_dict 

    def forward(self, z, x, event_z, event_x,
                ce_template_mask=None, ce_keep_rate=None,
                tnc_keep_rate=None,
                return_last_attn=False):

        x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,)

        return x, aux_dict


def _create_vision_transformer(pretrained=False, **kwargs):
    model = VisionTransformerCE(**kwargs)
    
    if pretrained:
        if 'npz' in pretrained:
            model.load_pretrained(pretrained, prefix='')
        else:
            checkpoint = torch.load(pretrained, map_location="cpu")
            missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
            print('Load pretrained model from: ' + pretrained)

    return model


def vit_base_patch16_224_ce(pretrained=False, **kwargs):
    """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    """
    model_kwargs = dict(
        patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
    return model


def vit_large_patch16_224_ce(pretrained=False, **kwargs):
    """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    """
    model_kwargs = dict(
        patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
    return model，关注不同模态之间的交互，其中class Cross_Attention(nn.Module):
    def __init__(self, input_channels, output_channels):
        super(Cross_Attention, self).__init__()
        self.query = nn.Linear(input_channels, output_channels)
        self.key = nn.Linear(input_channels, output_channels)
        self.value = nn.Linear(input_channels, output_channels)
        self.scale = output_channels ** -0.5  # 缩放因子，防止softmax的梯度消失/爆炸
    
    def forward(self, assistant, present):
        # 生成查询、键和值
        q = self.query(assistant)
        k = self.key(present)
        v = self.value(present)
        
        # 计算注意力分数
        attention_scores = torch.bmm(q, k.transpose(1, 2)) * self.scale
        attention = F.softmax(attention_scores, dim=-1)
        
        # 应用注意力机制
        attended_features = torch.bmm(attention, v)
        return attended_features

class Multi_Context(nn.Module):
    def __init__(self, input_channels, output_channels):  # 修正了def init为def init
        super(Multi_Context, self).__init__()
        self.linear1 = nn.Linear(input_channels, output_channels)
        self.linear2 = nn.Linear(input_channels, output_channels)
        self.linear3 = nn.Linear(input_channels, output_channels)
        self.linear_final = nn.Linear(output_channels * 3, output_channels)

    def forward(self, x):
        x1 = F.relu(self.linear1(x))
        x2 = F.relu(self.linear2(x))
        x3 = F.relu(self.linear3(x))
        x = torch.cat([x1, x2, x3], dim=-1)  # Concatenate along the feature dimension
        x = self.linear_final(x)
        return x

class EnhancedMultiContext(nn.Module):
    def __init__(self, input_channels, output_channels):
        super(EnhancedMultiContext, self).__init__()
        self.linear1 = nn.Linear(input_channels, output_channels)
        self.linear2 = nn.Linear(input_channels, output_channels)
        self.linear3 = nn.Linear(input_channels, output_channels)
        self.relu1 = nn.ReLU()
        self.leaky_relu2 = nn.LeakyReLU()
        self.swish3 = nn.SiLU()  # Alternatively, you can use nn.SiLU() for Swish activation
        self.linear_final = nn.Linear(output_channels * 3, output_channels)

    def forward(self, x):
        x1 = self.relu1(self.linear1(x))
        x2 = self.leaky_relu2(self.linear2(x))
        x3 = self.swish3(self.linear3(x))
        x = torch.cat([x1, x2, x3], dim=-1)  
        x = self.linear_final(x)  
        return x

class Adaptive_Weight(nn.Module):
    def __init__(self, input_channels):  # 修正了def init为def init
        super(Adaptive_Weight, self).__init__()
        self.fc1 = nn.Linear(input_channels, input_channels // 4)
        self.fc2 = nn.Linear(input_channels // 4, input_channels)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x_avg = torch.mean(x, dim=1)  # Global Average Pooling along the ‘sequence length’ dimension
        weight = F.relu(self.fc1(x_avg))
        weight = self.fc2(weight)
        weight = self.sigmoid(weight).unsqueeze(1)
        out = x * weight
        return out
    
class Multi_Adaptive_Weight(nn.Module):
    def __init__(self, input_channels):  
        super(Multi_Adaptive_Weight, self).__init__()
        self.fc1 = nn.Linear(input_channels, input_channels // 4)
        self.fc2 = nn.Linear(input_channels // 4, input_channels)
        self.sigmoid = nn.Sigmoid()
        self.global_avg_pool = nn.AdaptiveAvgPool1d(1)
        self.global_max_pool = nn.AdaptiveMaxPool1d(1)

    def forward(self, x):
        x_avg = self.global_avg_pool(x.transpose(1, 2)).squeeze(-1)  
        x_max = self.global_max_pool(x.transpose(1, 2)).squeeze(-1)  
        weight = F.relu(self.fc1(x_avg + x_max))
        weight = self.fc2(weight)
        weight = self.sigmoid(weight).unsqueeze(1)
        out = x * weight
        return out

class Counter_Guide_Enhanced(nn.Module):
    def __init__(self, input_channels, output_channels):
        super(Counter_Guide_Enhanced, self).__init__()
        # self.mc = Multi_Context(input_channels, output_channels)
        self.mc = EnhancedMultiContext(input_channels, output_channels)
        self.cross_attention = Cross_Attention(output_channels, output_channels)
        # 结合自适应
        self.adaptive_weight = Multi_Adaptive_Weight(output_channels)
    
    def forward(self, x, event_x):
        # 首先使用Multi-Context增强事件特征
        enhanced_event_x = self.mc(event_x)

        # 使用Cross-Attention进行特征交互,(x和event_x互相增强)
        enhanced_x = self.cross_attention(x, event_x)
        enhanced_event_x_via_x = self.cross_attention(event_x, enhanced_x)
        
        # 进一步通过adaptive weight调整增强的特征权重
        adaptive_enhanced_x = self.adaptive_weight(enhanced_x)
        adaptive_enhanced_event_x = self.adaptive_weight(enhanced_event_x_via_x)
        
        return adaptive_enhanced_x, adaptive_enhanced_event_x 和 class CrossReplaceTransformerLayer(nn.Module):
    def __init__(self, d_model, nhead, theta, skip_connection=False, use_quantile=False, dim_feedforward=2048, dropout=0.1, activation=F.relu, layer_norm_eps=1e-5,
                 norm_first=False, dtype=None):
        #factory_kwargs = {'device': device, 'dtype': dtype}
        super(CrossReplaceTransformerLayer, self).__init__()
        self.multihead_attn = nn.MultiheadAttention(
            d_model, nhead, dropout=dropout )

        # feedforward model
        self.linear1 = nn.Linear(d_model, dim_feedforward )
        self.dropout = nn.Dropout(dropout)
        self.linear2 = nn.Linear(dim_feedforward, d_model )

        self.norm_first = norm_first
        self.norm1 = nn.LayerNorm(
            d_model, eps=layer_norm_eps )
        self.norm2 = nn.LayerNorm(
            d_model, eps=layer_norm_eps )

        self.dropout1 = nn.Dropout(dropout)
        self.dropout2 = nn.Dropout(dropout)

        self.activatiion = activation

        self.theta = theta  # select the attention value
        self.skip_connection = skip_connection
        self.use_quantile = use_quantile

        self.mlp = nn.Sequential(
            nn.Linear(d_model, d_model),
            nn.ReLU(),
            nn.Linear(d_model, d_model)
        )


    # self-attention block
    def _sa_block(self, x, attn_mask, key_padding_mask):
        x, attn_weight = self.multihead_attn(
            x, x, x, attn_mask=attn_mask, key_padding_mask=key_padding_mask, need_weights=True)
        return self.dropout1(x), attn_weight

    # feedforward block

    def _ff_block(self, x):
        x = self.linear2(self.dropout2(self.activatiion(self.linear1(x))))
        return self.dropout2(x)

    # cross-replace block
    # notice: batch_first must be true

    def _cr_block(self, x1, x2, attn_weight1, attn_weight2):
                
        cls_weight1 = attn_weight1[:, 0, :].transpose(0, 1)  # 交换维度
        cls_weight2 = attn_weight2[:, 0, :].transpose(0, 1)  # 交换维度
        #print('cls_weight1', cls_weight1.shape) #cls_weight1 torch.Size([2, 32])
        #print(cls_weight1)
        #print('cls_weight2', cls_weight2.shape) #cls_weight2 torch.Size([1, 9129])
        #print(cls_weight2)
        #print (x1.shape,x2.shape)
        x1_mean = torch.mean(x1, dim=-2)
        x2_mean = torch.mean(x2, dim=-2)
        # print('self.theta[0]',self.theta[0])
        # print('self.theta[1]',self.theta[1])
        for i in range(cls_weight1.shape[0]):
            if self.use_quantile:
                theta1 = np.quantile(
                    cls_weight1[i][1:].detach().cpu().numpy(), self.theta[0])
                theta2 = np.quantile(
                    cls_weight2[i][1:].detach().cpu().numpy(), self.theta[1])
            else:
                theta1, theta2 = self.theta

            for j in range(1, cls_weight1.shape[1]):
                if cls_weight1[i][j] < theta1:
                    x1[i][j] = x2_mean[i] + \
                        x1[i][j] if self.skip_connection else x2_mean[i]
                elif cls_weight1[i][j] < theta2:
                    x1[i][j] = self.mlp(x2_mean[i]).squeeze() + x1[i][j]

                if cls_weight2[i][j] < theta1:
                    x2[i][j] = x1_mean[i] + \
                        x2[i][j] if self.skip_connection else x1_mean[i]
                elif cls_weight2[i][j] < theta2:
                    x2[i][j] = self.mlp(x1_mean[i]).squeeze() + x2[i][j]

        return x1, x2

    def forward(self, src1, src2, replace=False, src1_mask=None, src1_key_padding_mask=None, src2_mask=None, src2_key_padding_mask=None):
        x1 = src1
        x2 = src2
        #print (x1.shape)#32,6,768
        if self.norm_first:
            res1, attn_weight1 = self._sa_block(
                self.norm1(x1), src1_mask, src1_key_padding_mask)
            res2, attn_weight2 = self._sa_block(
                self.norm1(x2), src2_mask, src2_key_padding_mask)
            x1 = x1 + res1
            x2 = x2 + res2

            if replace:
                
                x1, x2 = self._cr_block(x1, x2, attn_weight1, attn_weight2)

            x1 = x1 + self._ff_block(self.norm2(x1))
            x2 = x2 + self._ff_block(self.norm2(x2))
        else:
            
            res1, attn_weight1 = self._sa_block(
                x1, src1_mask, src1_key_padding_mask)
            res2, attn_weight2 = self._sa_block(
                x2, src2_mask, src2_key_padding_mask)
            
            
            x1 = self.norm1(x1 + res1)
            x2 = self.norm1(x2 + res2)
            # np.set_printoptions(threshold=np.inf)
            # print(attn_weight1.detach().cpu().numpy())
            # print(attn_weight1[0].shape)
            #print (x1.shape)#32,6,768
            if replace:
                
                x1, x2 = self._cr_block(x1, x2, attn_weight1, attn_weight2)

            x1 = self.norm2(x1 + self._ff_block(x1))
            x2 = self.norm2(x2 + self._ff_block(x2))

        return x1, x2


def _get_clones(module, N):
    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])


class CrossReplaceTransformer(nn.Module):
    def __init__(self, encoder_layer, num_layers, replace_start, replace_end, norm=None):
        super(CrossReplaceTransformer, self).__init__()
        self.layers = _get_clones(encoder_layer, num_layers)
        self.num_layers = num_layers
        self.norm = norm
        self.replace_start = replace_start
        self.replace_end = replace_end

    def forward(self, src1, src2, mask1=None, src1_key_padding_mask=None, mask2=None, src2_key_padding_mask=None):
        output1 = src1
        output2 = src2
        # print('output1', output1.shape)
        # print(output1)
        # print('output2', output2.shape)
        # print(output2)
        
        for i, mod in enumerate(self.layers):
            if i >= self.replace_start and i <= self.replace_end:
                replace = True
            else:
                replace = False
            output1, output2 = mod(output1, output2, replace=replace, src1_mask=mask1,
                                   src1_key_padding_mask=src1_key_padding_mask, src2_mask=mask2, src2_key_padding_mask=src2_key_padding_mask)
            # print('output1', output1.shape)
            # print(output1)
            # print('output2', output2.shape)
            # print(output2)

        if self.norm is not None:
            output1 = self.norm(output1)
            output2 = self.norm(output2)

        return output1, output2。