在# 将 4输入分开，构建新的相同模态结合的2输入，2分支
import math
import logging
from functools import partial
from collections import OrderedDict
from copy import deepcopy

import torch
import torch.nn as nn
import torch.nn.functional as F

from timm.models.layers import to_2tuple

from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise
from .utils import combine_tokens, recover_tokens
from .vit import VisionTransformer
from ..layers.attn_blocks import CEBlock
from .new_counter_guide import Counter_Guide
from .ad_counter_guide import Counter_Guide_Enhanced
from ..mamba_fetrack.models_mamba import VisionMamba, vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2
# 集成CrossMamba模块进行不同模态特征交互
from ..mamba_fetrack.mamba_cross import CrossMamba
_logger = logging.getLogger(__name__)
                            

class VisionTransformerCE(VisionTransformer):
    """ Vision Transformer with candidate elimination (CE) module

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929

    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
        - https://arxiv.org/abs/2012.12877
    """
    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init='',
                 ce_loc=None, ce_keep_ratio=None):
        super().__init__()
        if isinstance(img_size, tuple):
            self.img_size = img_size
        else:
            self.img_size = to_2tuple(img_size)
        self.patch_size = patch_size
        self.in_chans = in_chans

        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU

        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches

        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)
        self.pos_embed_event = PatchEmbed_event(in_chans=32, embed_dim=768, kernel_size=4, stride=4)
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        blocks = []
        ce_index = 0
        self.ce_loc = ce_loc
        for i in range(depth):
            ce_keep_ratio_i = 1.0
            if ce_loc is not None and i in ce_loc:
                ce_keep_ratio_i = ce_keep_ratio[ce_index]
                ce_index += 1

            blocks.append(
                CEBlock(
                    dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                    attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                    keep_ratio_search=ce_keep_ratio_i)
            )

        self.blocks = nn.Sequential(*blocks)
        self.norm = norm_layer(embed_dim)

        self.init_weights(weight_init)
    
        # 添加交互模块counter_guide
        self.counter_guide = Counter_Guide_Enhanced(768, 768)
        # 初始化CrossMamba模块
        self.cross_mamba = CrossMamba(dim=768)
        #########################################
        
    def forward_features(self, z, x, event_z, event_x,
                         mask_z=None, mask_x=None,
                         ce_template_mask=None, ce_keep_rate=None,
                         return_last_attn=False
                         ):
        # 分支1 处理流程
        B, H, W = x.shape[0], x.shape[2], x.shape[3]

        x = self.patch_embed(x)
        z = self.patch_embed(z)
        z += self.pos_embed_z
        x += self.pos_embed_x        

        if mask_z is not None and mask_x is not None:
            mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_z = mask_z.flatten(1).unsqueeze(-1)

            mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_x = mask_x.flatten(1).unsqueeze(-1)

            mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
            mask_x = mask_x.squeeze(-1)

        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed
        if self.add_sep_seg:
            x += self.search_segment_pos_embed
            z += self.template_segment_pos_embed
        x = combine_tokens(z, x, mode=self.cat_mode)
        if self.add_cls_token:
            x = torch.cat([cls_tokens, x], dim=1)
        x = self.pos_drop(x)

        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
        global_index_t = global_index_t.repeat(B, 1)
        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
        global_index_s = global_index_s.repeat(B, 1)
        removed_indexes_s = []
        # # 分支2 处理流程
        event_x = self.pos_embed_event(event_x)
        event_z = self.pos_embed_event(event_z)
        event_x += self.pos_embed_x
        event_z += self.pos_embed_z

        event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)

        if self.add_cls_token:
            event_x = torch.cat([cls_tokens, event_x], dim=1)

        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]

        global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
        global_index_t1 = global_index_t1.repeat(B, 1)
        global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
        global_index_s1 = global_index_s1.repeat(B, 1)

        removed_indexes_s1 = []

        for i, blk in enumerate(self.blocks):
            # 第一个分支处理
            x, global_index_t, global_index_s, removed_index_s, attn = \
                blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)
            # 第二个分支处理
            event_x, global_index_t1, global_index_s1, removed_index_s1, attn1 = \
                blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)


            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s.append(removed_index_s)
                removed_indexes_s1.append(removed_index_s1)

            # 在第1层增加counter_guide模块,验证早期融合效果
            if i == 0 :
                # 引入counter_guide进行特征增强
                enhanced_x, enhanced_event_x = self.counter_guide(x, event_x)
                # 将增强后的特征与原特征相加
                x = x + enhanced_x
                event_x = event_x + enhanced_event_x
        ##############
        lens_x_new = global_index_s.shape[1]
        lens_z_new = global_index_t.shape[1]
        z = x[:, :lens_z_new]
        x = x[:, lens_z_new:]
        lens_eventx_new = global_index_s1.shape[1]
        lens_eventz_new = global_index_t1.shape[1]
        event_z = event_x[:, :lens_eventz_new]
        event_x = event_x[:, lens_eventz_new:]
        ## 恢复原始的 token 顺序 #################################
        if removed_indexes_s and removed_indexes_s[0] is not None:
            removed_indexes_cat = torch.cat(removed_indexes_s, dim=1)
            pruned_lens_x = lens_x - lens_x_new
            pad_x = torch.zeros([B, pruned_lens_x, x.shape[2]], device=x.device)
            x = torch.cat([x, pad_x], dim=1)
            index_all = torch.cat([global_index_s, removed_indexes_cat], dim=1)
            # 恢复原始顺序
            C = x.shape[-1]
            x = torch.zeros_like(x).scatter(dim=1, index=index_all.unsqueeze(-1).expand(B, -1, C).to(torch.int64), src=x)
            x = recover_tokens(x, lens_z_new, lens_x, mode=self.cat_mode)

        if removed_indexes_s1 and removed_indexes_s1[0] is not None:
            removed_indexes_cat1 = torch.cat(removed_indexes_s1, dim=1)
            pruned_lens_x1 = lens_x - lens_eventx_new
            pad_x1 = torch.zeros([B, pruned_lens_x1, event_x.shape[2]], device=event_x.device)
            event_x = torch.cat([event_x, pad_x1], dim=1)
            index_all1 = torch.cat([global_index_s1, removed_indexes_cat1], dim=1)
            # 恢复原始顺序
            C1 = event_x.shape[-1]
            event_x = torch.zeros_like(event_x).scatter(dim=1, index=index_all1.unsqueeze(-1).expand(B, -1, C1).to(torch.int64), src=event_x)
            event_x = recover_tokens(event_x, lens_eventz_new, lens_x, mode=self.cat_mode)
        ################################################################
        
        ## 引入Crossmamba进行事件和图像特征融合,不采用concatenate
        x = self.cross_mamba(x, event_x)
        # 应用LayerNorm归一化处理
        x = self.norm(x) 

        aux_dict = {
            "attn": attn,
            'attn1': attn1,
            "removed_indexes_s": removed_indexes_s,  # used for visualization
            'removed_indexes_s1': removed_indexes_s1,
        }
        
        return x, aux_dict 

    def forward(self, z, x, event_z, event_x,
                ce_template_mask=None, ce_keep_rate=None,
                tnc_keep_rate=None,
                return_last_attn=False):

        x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,)

        return x, aux_dict


def _create_vision_transformer(pretrained=False, **kwargs):
    model = VisionTransformerCE(**kwargs)
    
    if pretrained:
        if 'npz' in pretrained:
            model.load_pretrained(pretrained, prefix='')
        else:
            checkpoint = torch.load(pretrained, map_location="cpu")
            missing_keys, unexpected_keys = model.load_state_dict(checkpoint["model"], strict=False)
            print('Load pretrained model from: ' + pretrained)

    return model


def vit_base_patch16_224_ce(pretrained=False, **kwargs):
    """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    """
    model_kwargs = dict(
        patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
    return model


def vit_large_patch16_224_ce(pretrained=False, **kwargs):
    """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    """
    model_kwargs = dict(
        patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
    return model中，调用cross_mamba有问题，其中报错：Traceback (most recent call last):
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/train/trainers/base_trainer.py", line 85, in train
    self.train_epoch()
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/train/trainers/ltr_trainer.py", line 133, in train_epoch
    self.cycle_dataset(loader)
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/train/trainers/ltr_trainer.py", line 86, in cycle_dataset
    loss, stats = self.actor(data)
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/train/actors/ceutrack.py", line 285, in __call__
    out_dict = self.forward_pass(data)
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/train/actors/ceutrack.py", line 328, in forward_pass
    out_dict = self.net(template=template_list,
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1519, in forward
    else self._run_ddp_forward(*inputs, **kwargs)
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1355, in _run_ddp_forward
    return self.module(*inputs, **kwargs)  # type: ignore[index]
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/models/ceutrack/ceutrack.py", line 57, in forward
    x, aux_dict = self.backbone(z=template, x=search, event_z=event_template, event_x=event_search,
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 225, in forward
    x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,)
  File "/data2/local_userdata/heyuting/new_VOT/COESOT-main/CEUTrack/lib/train/../../lib/models/ceutrack/vit_ce.py", line 207, in forward_features
    x = self.cross_mamba(x, event_x)
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/heyuting/anaconda3/envs/mamba/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
    return forward_call(*args, **kwargs)
TypeError: CrossMamba.forward() missing 1 required positional argument: 'm2'，其中，import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
import numbers
from .mamba_cross_simple import Mamba
# 将输入张量从（b,c,h,w)重排为（b,h*w,c)，常用于将图像张量转换为序列格式
def to_3d(x):
    return rearrange(x, 'b c h w -> b (h w) c')
# 将序列张量从（b,h*w,c)重排回（b,c,h,w)，这是to_3d的反向操作,通常在完成序列处理之后用于恢复原始形状。
def to_4d(x, h, w):
    return rearrange(x, 'b (h w) c -> b c h w', h=h, w=w)
# 无偏LayerNorm
# 实现了不包含偏置项的层标准化,计算输入的方差，用于对输入进行规范化处理，再通过权重进行缩放。
class BiasFree_LayerNorm(nn.Module):
    def __init__(self, normalized_shape):
        super(BiasFree_LayerNorm, self).__init__()
        if isinstance(normalized_shape, numbers.Integral):
            normalized_shape = (normalized_shape,)
        normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1

        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.normalized_shape = normalized_shape

    def forward(self, x):
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return x / torch.sqrt(sigma+1e-5) * self.weight

# 包含偏置的层标准化
class WithBias_LayerNorm(nn.Module):
    def __init__(self, normalized_shape):
        super(WithBias_LayerNorm, self).__init__()
        if isinstance(normalized_shape, numbers.Integral):
            normalized_shape = (normalized_shape,)
        normalized_shape = torch.Size(normalized_shape)

        assert len(normalized_shape) == 1

        self.weight = nn.Parameter(torch.ones(normalized_shape))
        self.bias = nn.Parameter(torch.zeros(normalized_shape))
        self.normalized_shape = normalized_shape

    def forward(self, x):
        mu = x.mean(-1, keepdim=True)
        sigma = x.var(-1, keepdim=True, unbiased=False)
        return (x - mu) / torch.sqrt(sigma+1e-5) * self.weight + self.bias

# 对2种不同的层标准化进行的封装 LayerNorm_type参数决定了使用哪种层标准化
# 同时通过先把二维图像数据转换为3维序列，应用层标准化再转换回原来的2维结构。
class LayerNorm(nn.Module):
    def __init__(self, dim, LayerNorm_type):
        super(LayerNorm, self).__init__()
        if LayerNorm_type =='BiasFree':
            self.body = BiasFree_LayerNorm(dim)
        else:
            self.body = WithBias_LayerNorm(dim)

    def forward(self, x):
        h, w = x.shape[-2:]
        return to_4d(self.body(to_3d(x)), h, w)

# CrossMamba是Mamba的变体，bimamba_type='V3'指明了具体的Mmaba版本
# self.cross_mamba()用于处理来自两个不同模态的特征，m1和m2分别是两个模态的特征, m1_resi是m1的残差,并输出融合后的全局特征global_f
# self.norm1和self.norm2分别对m1和m2进行层标准化处理（包含偏置）
# self.dwconv是一个深度可分离卷积层,用于局部空间内增强特征表达能力。
# 先 to_3d 再 to_4d 操作,避免了直接在多维空间上操作的复杂性,并确保了层标准化可以适用于不同位置的特征向量，保持处理的一致性和独立性。
class CrossMamba(nn.Module):
    def __init__(self, dim):
        super(CrossMamba, self).__init__()
        self.cross_mamba = Mamba(dim,bimamba_type="v3") # 使用v3版本,会使用额外的输入extra_emb进行特殊处理
        self.norm1 = LayerNorm(dim,'with_bias')
        self.norm2 = LayerNorm(dim,'with_bias')
        self.dwconv = nn.Conv2d(dim, dim, kernel_size=3, padding=1, groups=dim)
    def forward(self,m1,m1_resi,m2):     
        print('m1.shape',m1.shape) # m1.shape torch.Size([4, 320, 384])
        print('m2.shape',m2.shape) # m2.shape torch.Size([4, 320, 384])
        print(m1_resi) # 0 
        m1_resi = m1+m1_resi    
        print('m1_resi.shape',m1_resi.shape) # m2.shape torch.Size([4, 320, 384])

        global_f = self.cross_mamba(m1, extra_emb=m2)
        print('global_f.shape',global_f.shape)  # global_f.shape torch.Size([4, 320, 384])
        return global_f