这是我们的双模态双分支结构的forward函数： def forward_features(self, z, x, event_z, event_x,
mask_z=None, mask_x=None,
ce_template_mask=None, ce_keep_rate=None,
return_last_attn=False
):
# 分支1 处理流程
B, H, W = x.shape[0], x.shape[2], x.shape[3]

x = self.patch_embed(x)
z = self.patch_embed(z)
z += self.pos_embed_z
x += self.pos_embed_x

if mask_z is not None and mask_x is not None:
mask_z = F.interpolate(mask_z[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
mask_z = mask_z.flatten(1).unsqueeze(-1)

mask_x = F.interpolate(mask_x[None].float(), scale_factor=1. / self.patch_size).to(torch.bool)[0]
mask_x = mask_x.flatten(1).unsqueeze(-1)

mask_x = combine_tokens(mask_z, mask_x, mode=self.cat_mode)
mask_x = mask_x.squeeze(-1)

if self.add_cls_token:
cls_tokens = self.cls_token.expand(B, -1, -1)
cls_tokens = cls_tokens + self.cls_pos_embed
if self.add_sep_seg:
x += self.search_segment_pos_embed
z += self.template_segment_pos_embed
x = combine_tokens(z, x, mode=self.cat_mode)
if self.add_cls_token:
x = torch.cat([cls_tokens, x], dim=1)
x = self.pos_drop(x)

lens_z = self.pos_embed_z.shape[1]
lens_x = self.pos_embed_x.shape[1]
global_index_t = torch.linspace(0, lens_z - 1, lens_z).to(x.device)
global_index_t = global_index_t.repeat(B, 1)
global_index_s = torch.linspace(0, lens_x - 1, lens_x).to(x.device)
global_index_s = global_index_s.repeat(B, 1)
removed_indexes_s = []

for i, blk in enumerate(self.blocks):
x, global_index_t, global_index_s, removed_index_s, attn = <br/> blk(x, global_index_t, global_index_s, mask_x, ce_template_mask, ce_keep_rate)


if self.ce_loc is not None and i in self.ce_loc:
removed_indexes_s.append(removed_index_s)
# x = self.norm(x) # # [bs, n_patch, dim] = [bs, 320, 768] 320 = 64 + 256


# # 分支2 处理流程
event_x = self.pos_embed_event(event_x)
event_z = self.pos_embed_event(event_z)
event_x += self.pos_embed_x
event_z += self.pos_embed_z

event_x = combine_tokens(event_z, event_x, mode=self.cat_mode)

if self.add_cls_token:
event_x = torch.cat([cls_tokens, event_x], dim=1)

lens_z = self.pos_embed_z.shape[1]
lens_x = self.pos_embed_x.shape[1]

global_index_t1 = torch.linspace(0, lens_z - 1, lens_z).to(event_x.device)
global_index_t1 = global_index_t1.repeat(B, 1)
global_index_s1 = torch.linspace(0, lens_x - 1, lens_x).to(event_x.device)
global_index_s1 = global_index_s1.repeat(B, 1)

removed_indexes_s1 = []
for i, blk in enumerate(self.blocks):
event_x, global_index_t1, global_index_s1, removed_index_s1, attn = <br/> blk(event_x, global_index_t1, global_index_s1, mask_x, ce_template_mask, ce_keep_rate)

if self.ce_loc is not None and i in self.ce_loc:
removed_indexes_s1.append(removed_index_s1)

# 在所有blocks处理完，引入counter_guide进行模态间交互
x_inter, event_x_inter = self.counter_guide(x,event_x)
# 将交互后的特征增强原始特征
x_enhenced = x + x_inter
event_x_enhenced = event_x + event_x_inter
x = torch.cat([x_enhenced, event_x_enhenced], dim=1)
，现在将import torch,os
import torch.nn as nn
from torch.nn.parameter import Parameter

class Multi_Context(nn.Module):
def init(self, inchannels):
super(Multi_Context, self).init()
self.conv2_1 = nn.Sequential(
nn.Conv2d(in_channels=inchannels, out_channels=inchannels, kernel_size=1, stride=1, padding=0),
nn.BatchNorm2d(inchannels),
nn.ReLU(inplace=True))
self.conv2_2 = nn.Sequential(
nn.Conv2d(in_channels=inchannels, out_channels=inchannels, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(inchannels),
nn.ReLU(inplace=True))
self.conv2_3 = nn.Sequential(
nn.Conv2d(in_channels=inchannels, out_channels=inchannels, kernel_size=5, stride=1, padding=2),
nn.BatchNorm2d(inchannels),
nn.ReLU(inplace=True))
self.conv2 = nn.Sequential(
nn.Conv2d(in_channels=inchannels * 3, out_channels=inchannels, kernel_size=3, padding=1),
nn.BatchNorm2d(inchannels))

def forward(self, x):
x1 = self.conv2_1(x)
x2 = self.conv2_2(x)
x3 = self.conv2_3(x)
x = torch.cat([x1,x2,x3], dim=1)
x = self.conv2(x)
return x

class Adaptive_Weight(nn.Module):
def init(self, inchannels):
super(Adaptive_Weight, self).init()
self.avg = nn.AdaptiveAvgPool2d(1)
self.inchannels = inchannels
self.fc1 = nn.Conv2d(inchannels, inchannels//4, kernel_size=1, bias=False)
self.relu1 = nn.ReLU()
self.fc2 = nn.Conv2d(inchannels//4, 1, kernel_size=1, bias=False)
self.relu2 = nn.ReLU()
self.sigmoid = nn.Sigmoid()

def forward(self, x):
x_avg = self.avg(x)
weight = self.relu1(self.fc1(x_avg))
weight = self.relu2(self.fc2(weight))
weight = self.sigmoid(weight)
out = x * weight
return out

class Counter_attention(nn.Module):
def init(self, inchannels):
super(Counter_attention, self).init()
self.conv1 = nn.Sequential(nn.Conv2d(in_channels=inchannels, out_channels=inchannels, kernel_size=3, padding=1),
nn.BatchNorm2d(inchannels))
self.conv2 = nn.Sequential(nn.Conv2d(in_channels=inchannels, out_channels=inchannels, kernel_size=3, padding=1),
nn.BatchNorm2d(inchannels))
# self.conv3 = nn.Sequential(nn.Conv2d(in_channels=inchannels2, out_channels=inchannels, kernel_size=1),
# nn.BatchNorm2d(inchannels))
self.sig = nn.Sigmoid()
self.mc1 = Multi_Context(inchannels)
self.mc2 = Multi_Context(inchannels)
self.ada_w1 = Adaptive_Weight(inchannels)
self.ada_w2 = Adaptive_Weight(inchannels)
def forward(self, assistant, present):

mc1 = self.mc1(assistant)
pr1 = present * self.sig(mc1)
pr2 = self.conv1(present)
pr2 = present * self.sig(pr2)
out1 = pr1 + pr2 + present


mc2 = self.mc2(present)
as1 = assistant * self.sig(mc2)
as2 = self.conv2(assistant)
as2 = assistant * self.sig(as2)
out2 = as1 + as2 + assistant


out1 = self.ada_w1(out1)
out2 = self.ada_w2(out2)
out = out1 + out2

# out = torch.cat([out1, out2], dim=1)
# out = self.conv3(out)

return out

class Counter_Guide(nn.Module):
def init(self):
super(Counter_Guide, self).init()
self.counter_atten1 = Counter_attention(128)
self.counter_atten2 = Counter_attention(256)



def forward(self, frame1, frame2, event1, event2):
out1 = self.counter_atten1(frame1, event1)
out2 = self.counter_atten2(frame2, event2)

return out1, out2


if name == ‘main’:
net = Counter_Guide()
os.environ[‘CUDA_VISIBLE_DEVICES’] = ‘0’
net = net.cuda()

var1 = torch.FloatTensor(10, 128, 36, 36).cuda()
var2 = torch.FloatTensor(10, 256, 18, 18).cuda()
var3 = torch.FloatTensor(10, 128, 36, 36).cuda()
var4 = torch.FloatTensor(10, 256, 18, 18).cuda()
# var = Variable(var)

out1, out2 = net(var1, var2, var3, var4)

print('************')
print(out1.shape, out2.shape)引入，实现双分支模态特征交互，现在需要在输入counterguide之前将x和event_x维度转换，以适应counterguide模块，我们的特征x和event_x的维度是（32，320，768）