原始mamba模型中的配置是：MODEL:
  PRETRAIN_FILE: "hustvlVim-small-midclstok+/vim_s_midclstok_80p5acc.pth"
  EXTRA_MERGER: False
  RETURN_INTER: False
  BACKBONE:
    TYPE: vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2
    STRIDE: 16
    CE_LOC: [3, 6, 9]
    CE_KEEP_RATIO: [1, 1, 1]
    CE_TEMPLATE_RANGE: 'CTR_POINT'，现在模型是import math
import logging
from functools import partial
from collections import OrderedDict
from copy import deepcopy

import torch
import torch.nn as nn
import torch.nn.functional as F

from timm.models.layers import to_2tuple

from lib.models.layers.patch_embed import PatchEmbed, PatchEmbed_event, xcorr_depthwise
from .utils import combine_tokens, recover_tokens
from .vit import VisionTransformer
from ..layers.attn_blocks import CEBlock

from mamba_ssm.modules.mamba_simple import Mamba
from mamba_ssm.utils.generation import GenerationMixin
from mamba_ssm.utils.hf import load_config_hf, load_state_dict_hf
from lib.models.mamba_fetrack.rope import *
import random
from lib.models.layers.head import build_box_head
import importlib
import lib.train.admin.settings as ws_settings
from ..mamba_fetrack.models_mamba import VisionMamba, vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2

_logger = logging.getLogger(__name__)

# 定义 VisionTransformerCE 类
class VisionTransformerCE(VisionTransformer):
    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init='',
                 ce_loc=None, ce_keep_ratio=None, event_embed_dim=768, event_depth=24,
                 rgb_pretrain=None, event_pretrain=None):
        super().__init__()
        if isinstance(img_size, tuple):
            self.img_size = img_size
        else:
            self.img_size = to_2tuple(img_size)
        self.patch_size = patch_size
        self.in_chans = in_chans

        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU

        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches

        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)
        self.pos_embed_event = nn.Parameter(torch.zeros(1, 256, event_embed_dim))
        self.pos_embed_z_event = nn.Parameter(torch.zeros(1, 64, event_embed_dim))
        
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        blocks = []
        ce_index = 0
        self.ce_loc = ce_loc
        for i in range(depth):
            ce_keep_ratio_i = 1.0
            if ce_loc is not None and i in ce_loc:
                ce_keep_ratio_i = ce_keep_ratio[ce_index]
                ce_index += 1

            blocks.append(
                CEBlock(
                    dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate,
                    attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer,
                    keep_ratio_search=ce_keep_ratio_i)
            )

        self.blocks = nn.Sequential(*blocks)
        self.norm = norm_layer(embed_dim)
        
        # 初始化 event_transformer (基于 VisionMamba)
        self.event_transformer = VisionMamba(
            img_size=img_size, 
            patch_size=patch_size, 
            stride=16, 
            depth=event_depth, 
            embed_dim=event_embed_dim, 
            channels=3, 
            num_classes=num_classes,
            if_abs_pos_embed=True,
            drop_rate=drop_rate,
            drop_path_rate=drop_path_rate
        )
        # 加载RGB分支的预训练权重
        if rgb_pretrain and 'npz' not in rgb_pretrain:
            checkpoint = torch.load(rgb_pretrain, map_location='cpu')
            self.load_state_dict(checkpoint['model'], strict=False)
            print('Load RGB pretrained model from: ' + rgb_pretrain)
            
        # 加载事件分支的预训练权重
        if event_pretrain and 'npz' not in event_pretrain:
            checkpoint = torch.load(event_pretrain, map_location='cpu')
            self.event_transformer.load_state_dict(checkpoint['model'], strict=False)
            print('Load Event pretrained model from: ' + event_pretrain)
        #######################################
        self.init_weights(weight_init)
    
    def forward_features(self, z, x, event_z, event_x,
                         mask_z=None, mask_x=None,
                         ce_template_mask=None, ce_keep_rate=None,
                         return_last_attn=False):
        # 分支1 处理流程
        B, H, W = x.shape[0], x.shape[2], x.shape[3]

        x = self.patch_embed(x)
        z = self.patch_embed(z)

        z += self.pos_embed_z
        x += self.pos_embed_x        

        if mask_z is not None and mask_x is not None:
            mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_z = mask_z.flatten(1).unsqueeze(-1)

            mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
            mask_x = mask_x.flatten(1).unsqueeze(-1)

            mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
            mask_x = mask_x.squeeze(-1)

        if self.add_cls_token:
            cls_tokens = self.cls_token.expand(B, -1, -1)
            cls_tokens = cls_tokens + self.cls_pos_embed
        if self.add_sep_seg:
            x += self.search_segment_pos_embed
            z += self.template_segment_pos_embed
        x = combine_tokens(z, x, mode=self.cat_mode)
        if self.add_cls_token:
            x = torch.cat([cls_tokens, x], dim=1)
        x = self.pos_drop(x)

        lens_z = self.pos_embed_z.shape[1]
        lens_x = self.pos_embed_x.shape[1]
        global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
        global_index_t = global_index_t.repeat(B, 1)
        global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
        global_index_s = global_index_s.repeat(B, 1)
        removed_indexes_s = []

        for i, blk in enumerate(self.blocks):
            x, global_index_t, global_index_s, removed_index_s, attn = blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)

            if self.ce_loc is not None and i in self.ce_loc:
                removed_indexes_s.append(removed_index_s)
        x = self.norm(x) # [bs, n_patch, dim] = [bs, 320, 768] 320 = 64 + 256
       
        # 分支2 处理流程
        event_x = self.event_transformer.forward_features(event_z, event_x)
        
        # 拼接两种模态特征
        x_cat = torch.cat([event_x,x], dim=1) 
        x = x_cat

        aux_dict = {
            'attn': attn,
            'removed_indexes_s': removed_indexes_s,  # used for visualization
        }
        
        return x, aux_dict 

    def forward(self, z, x, event_z, event_x,
                ce_template_mask=None, ce_keep_rate=None,
                tnc_keep_rate=None,
                return_last_attn=False):

        x, aux_dict = self.forward_features(z, x, event_z, event_x, ce_template_mask=ce_template_mask, ce_keep_rate=ce_keep_rate,)

        return x, aux_dict


# def _create_vision_transformer(pretrained=False, **kwargs):
#     model = VisionTransformerCE(**kwargs)
    
#     if pretrained:
#         if 'npz' in pretrained:
#             model.load_pretrained(pretrained, prefix='')
#         else:
#             checkpoint = torch.load(pretrained, map_location='cpu')
#             missing_keys, unexpected_keys = model.load_state_dict(checkpoint['model'], strict=False)
#             print('Load pretrained model from: ' + pretrained)

#     return model

# 为进行两种模型权重的加载,修改_create_vision_transformer方法
def _create_vision_transformer(pretrained=False, rgb_pretrain=None, event_pretrain=None, **kwargs):
    model = VisionTransformerCE(rgb_pretrain=rgb_pretrain, event_pretrain=event_pretrain, **kwargs)
    
    return model


# def vit_base_patch16_224_ce(pretrained=False, **kwargs):
#     '''" ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
#     '''
#     model_kwargs = dict(
#         patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs)
#     model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
#     return model

############ 重构整体模型,加载两个分支模型权重
def vit_base_patch16_224_ce(pretrained=False, rgb_pretrain=None, event_pretrain=None, **kwargs):
    ''' ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    '''
    model_kwargs = dict(
        patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, rgb_pretrain=rgb_pretrain, event_pretrain=event_pretrain, **model_kwargs)
    return model


def vit_large_patch16_224_ce(pretrained=False, **kwargs):
    ''' ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    '''
    model_kwargs = dict(
        patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs)
    model = _create_vision_transformer(pretrained=pretrained, **model_kwargs)
    return model
，配置文件是：DATA:
  MAX_SAMPLE_INTERVAL: 100
  MEAN:
  - 0.485
  - 0.456
  - 0.406
  SEARCH:
    CENTER_JITTER: 3
    FACTOR: 4.0
    SCALE_JITTER: 0.25
    SIZE: 256
    NUMBER: 1
  STD:
  - 0.229
  - 0.224
  - 0.225
  TEMPLATE:
    CENTER_JITTER: 0
    FACTOR: 2.0
    SCALE_JITTER: 0
    SIZE: 128
  TRAIN:
    DATASETS_NAME:
    - COESOT
    DATASETS_RATIO:
    - 1
    SAMPLE_PER_EPOCH: 60000
  VAL:
    DATASETS_NAME:
    - COESOT_VAL
    DATASETS_RATIO:
    - 1
    SAMPLE_PER_EPOCH: 5000
MODEL:
  PRETRAIN_FILE: "mae_pretrain_vit_base.pth"
  # 添加RGB分支和Event分支的预训练模型
  RGB_PRETRAIN_FILE: "mae_pretrain_vit_base.pth"
  EVENT_PRETRAIN_FILE: "vim_s_midclstok_80p5acc.pth"
  EXTRA_MERGER: False
  RETURN_INTER: False
  BACKBONE:
    TYPE: vit_base_patch16_224_ce
    STRIDE: 16
    CE_LOC: [3, 6, 9]
    CE_KEEP_RATIO: [1, 1, 1]
    CE_TEMPLATE_RANGE: 'CTR_POINT'
  HEAD:
    TYPE: CENTER
    NUM_CHANNELS: 256
TRAIN:
  BACKBONE_MULTIPLIER: 0.1
  DROP_PATH_RATE: 0.1
  CE_START_EPOCH: 0  # candidate elimination start epoch  20
  CE_WARM_EPOCH: 0  # candidate elimination warm up epoch 50
  BATCH_SIZE: 2 # 32 # 32
  EPOCH: 50
  GIOU_WEIGHT: 1     # 2.0
  L1_WEIGHT: 14       # 5.0
  FOCAL_WEIGHT: 1.0   # 1.5
  GRAD_CLIP_NORM: 0.1
  LR: 0.0001 
  LR_DROP_EPOCH: 40
  NUM_WORKER: 4 # 4
  OPTIMIZER: ADAMW
  PRINT_INTERVAL: 50
  SCHEDULER:
    TYPE: step
    DECAY_RATE: 0.1
#    TYPE: Mstep
#    MILESTONES: [40, 60]
#    GAMMA: 0.1
  VAL_EPOCH_INTERVAL: 1
  WEIGHT_DECAY: 0.0001 #0.0001
  AMP: False
TEST:
  EPOCH: 50
  SEARCH_FACTOR: 4.0
  SEARCH_SIZE: 256
  TEMPLATE_FACTOR: 2.0
  TEMPLATE_SIZE: 128，那么由于event分支采用了viisonmamba进行处理，且加载了visionmamba的预训练权重，那么是否需要将BACKBONE:
    TYPE: vim_small_patch16_224_bimambav2_final_pool_mean_abs_pos_embed_with_midclstok_div2加入DATA:
  MAX_SAMPLE_INTERVAL: 100
  MEAN:
  - 0.485
  - 0.456
  - 0.406
  SEARCH:
    CENTER_JITTER: 3
    FACTOR: 4.0
    SCALE_JITTER: 0.25
    SIZE: 256
    NUMBER: 1
  STD:
  - 0.229
  - 0.224
  - 0.225
  TEMPLATE:
    CENTER_JITTER: 0
    FACTOR: 2.0
    SCALE_JITTER: 0
    SIZE: 128
  TRAIN:
    DATASETS_NAME:
    - COESOT
    DATASETS_RATIO:
    - 1
    SAMPLE_PER_EPOCH: 60000
  VAL:
    DATASETS_NAME:
    - COESOT_VAL
    DATASETS_RATIO:
    - 1
    SAMPLE_PER_EPOCH: 5000
MODEL:
  PRETRAIN_FILE: "mae_pretrain_vit_base.pth"
  # 添加RGB分支和Event分支的预训练模型
  RGB_PRETRAIN_FILE: "mae_pretrain_vit_base.pth"
  EVENT_PRETRAIN_FILE: "vim_s_midclstok_80p5acc.pth"
  EXTRA_MERGER: False
  RETURN_INTER: False
  BACKBONE:
    TYPE: vit_base_patch16_224_ce
    STRIDE: 16
    CE_LOC: [3, 6, 9]
    CE_KEEP_RATIO: [1, 1, 1]
    CE_TEMPLATE_RANGE: 'CTR_POINT'
  HEAD:
    TYPE: CENTER
    NUM_CHANNELS: 256
TRAIN:
  BACKBONE_MULTIPLIER: 0.1
  DROP_PATH_RATE: 0.1
  CE_START_EPOCH: 0  # candidate elimination start epoch  20
  CE_WARM_EPOCH: 0  # candidate elimination warm up epoch 50
  BATCH_SIZE: 2 # 32 # 32
  EPOCH: 50
  GIOU_WEIGHT: 1     # 2.0
  L1_WEIGHT: 14       # 5.0
  FOCAL_WEIGHT: 1.0   # 1.5
  GRAD_CLIP_NORM: 0.1
  LR: 0.0001 
  LR_DROP_EPOCH: 40
  NUM_WORKER: 4 # 4
  OPTIMIZER: ADAMW
  PRINT_INTERVAL: 50
  SCHEDULER:
    TYPE: step
    DECAY_RATE: 0.1
#    TYPE: Mstep
#    MILESTONES: [40, 60]
#    GAMMA: 0.1
  VAL_EPOCH_INTERVAL: 1
  WEIGHT_DECAY: 0.0001 #0.0001
  AMP: False
TEST:
  EPOCH: 50
  SEARCH_FACTOR: 4.0
  SEARCH_SIZE: 256
  TEMPLATE_FACTOR: 2.0
  TEMPLATE_SIZE: 128中？