jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def forward(self, x):
        count=0

        size=x.size()
        x = self.resnet.conv1(x)
        x = self.resnet.bn1(x)
        x = self.resnet.relu(x)
        # x = self.resnet.maxpool(x)
        x = self.resnet.layer1(x)
        x = self.resnet.layer2(x)
        x = self.resnet.layer3(x)
        x = self.resnet.layer4(x)

        x = self.final(torch.cat([
        	x,
        	self.layer5a(x),
        	self.layer5b(x),
        	self.layer5c(x),
        	self.layer5d(x),
        ], 1))


        return F.interpolate(x,size[2:], mode='bilinear', align_corners=False)
//  [python]
def forward(self,x):

        feats = self.feats(x)
        fconn = self.fconn(feats)
        upsample = F.upsample_bilinear(fconn, x.size()[2:])
        out = self.activation(upsample)
        return out
//  [python]
def forward(self,x):

        feats = self.feats(x)
        fconn = self.fconn(feats)
        upsample = F.interpolate(fconn, x.size()[2:], mode='bilinear')
        out = self.activation(upsample)
        return out
//  [python]
def __call__(self, x):

        ns = torch.normal(mean=torch.zeros(x.shape[0], x.shape[1], self.noise_res, self.noise_res),
                          std=self.noise_std).to(x.device)

        ns = torch.nn.functional.upsample_bilinear(ns, size=[x.shape[2], x.shape[3]])

        # Roll to randomly translate the generated noise.
        roll_x = np.random.choice(range(x.shape[2]))
        roll_y = np.random.choice(range(x.shape[3]))
        ns = torch.roll(ns, shifts=[roll_x, roll_y], dims=[-2, -1])

        mask = x.sum(dim=1, keepdim=True) > 0.01
        ns *= mask  # Only apply the noise in the foreground.
        res = x + ns
        return res
//  [python]
def __call__(self, x):

        ns = torch.normal(mean=torch.zeros(x.shape[0], x.shape[1], self.noise_res, self.noise_res),
                          std=self.noise_std).to(x.device)

        ns = torch.nn.functional.interpolate(ns, size=[x.shape[2], x.shape[3]], mode='bilinear')

        # Roll to randomly translate the generated noise.
        roll_x = np.random.choice(range(x.shape[2]))
        roll_y = np.random.choice(range(x.shape[3]))
        ns = torch.roll(ns, shifts=[roll_x, roll_y], dims=[-2, -1])

        mask = x.sum(dim=1, keepdim=True) > 0.01
        ns *= mask  # Only apply the noise in the foreground.
        res = x + ns
        return res
//  [python]
def forward(self, x):
        en1 = self.down1(x)
        po1 = self.pool1(en1)
        en2 = self.down2(po1)
        po2 = self.pool2(en2)
        en3 = self.down3(po2)
        po3 = self.pool3(en3)
        en4 = self.down4(po3)
        po4 = self.pool4(en4)

        c1 = self.center(po4)

        dec1 = self.up1(torch.cat([c1, F.upsample_bilinear(en4, c1.size()[2:])], 1))
        dec2 = self.up2(torch.cat([dec1, F.upsample_bilinear(en3, dec1.size()[2:])], 1))
        dec3 = self.up3(torch.cat([dec2, F.upsample_bilinear(en2, dec2.size()[2:])], 1))
        dec4 = self.up4(torch.cat([dec3, F.upsample_bilinear(en1, dec3.size()[2:])], 1))
        
        out = self.output(dec4)
        return self.final(out)
//  [python]
def forward(self, x):
        en1 = self.down1(x)
        po1 = self.pool1(en1)
        en2 = self.down2(po1)
        po2 = self.pool2(en2)
        en3 = self.down3(po2)
        po3 = self.pool3(en3)
        en4 = self.down4(po3)
        po4 = self.pool4(en4)

        c1 = self.center(po4)

        dec1 = self.up1(torch.cat([c1, F.interpolate(en4, c1.size()[2:], mode='bilinear')], 1))
        dec2 = self.up2(torch.cat([dec1, F.interpolate(en3, dec1.size()[2:], mode='bilinear')], 1))
        dec3 = self.up3(torch.cat([dec2, F.interpolate(en2, dec2.size()[2:], mode='bilinear')], 1))
        dec4 = self.up4(torch.cat([dec3, F.interpolate(en1, dec3.size()[2:], mode='bilinear')], 1))
        
        out = self.output(dec4)
        return self.final(out)
//  [python]
def forward(self, x):

        out    = self.preprocess(x)                                       #128

        conv1d = self.conv1d(out)                                         #128
        out    = F.max_pool2d(conv1d, kernel_size=2, stride=2)  # 64

        conv2d = self.conv2d(out) + make_shortcut(out, self.short2d)      # 64
        out    = F.max_pool2d(conv2d, kernel_size=2, stride=2) # 32
        flat2d = make_max_flat(out)

        conv3d = self.conv3d(out) + make_shortcut(out, self.short3d)      # 32
        out    = F.max_pool2d(conv3d, kernel_size=2, stride=2) # 16
        flat3d = make_max_flat(out)

        conv4d = self.conv4d(out) + make_shortcut(out, self.short4d)      # 16
        out    = F.max_pool2d(conv4d, kernel_size=2, stride=2) #  8
        flat4d = make_max_flat(out)

        conv5d = self.conv5d(out) + make_shortcut(out, self.short5d)      #  8
        out    = conv5d                                        #  4
        flat5d = make_max_flat(out)

        out    = F.upsample_bilinear(out,scale_factor=2)      # 16
        out    = out + conv4d
        out    = self.conv4u(out)
        flat4u = make_max_flat(out)

        out    = F.upsample_bilinear(out,scale_factor=2)      # 32
        out    = out + conv3d
        out    = self.conv3u(out)
        flat3u = make_max_flat(out)

        out    = F.upsample_bilinear(out,scale_factor=2)      # 64
        out    = out + conv2d
        out    = self.conv2u(out)
        flat2u = make_max_flat(out)

        out    = F.upsample_bilinear(out,scale_factor=2)      #128
        out    = out + conv1d
        out    = self.conv1u(out)
        flat1u = make_max_flat(out)



        logit2d = self.cls2d(flat2d).unsqueeze(2)
        logit3d = self.cls3d(flat3d).unsqueeze(2)
        logit4d = self.cls4d(flat4d).unsqueeze(2)
        logit5d = self.cls5d(flat5d).unsqueeze(2)

        logit1u = self.cls1u(flat1u).unsqueeze(2)
        logit2u = self.cls2u(flat2u).unsqueeze(2)
        logit3u = self.cls3u(flat3u).unsqueeze(2)
        logit4u = self.cls4u(flat4u).unsqueeze(2)


        logit = torch.cat((logit2d,logit3d,logit4d,logit5d,logit1u,logit2u,logit3u,logit4u),2)

        logit = F.dropout(logit, p=0.15,training=self.training)
        logit = logit.sum(2)
        logit = logit.view(logit.size(0),logit.size(1)) #unsqueeze(2)
        prob = F.sigmoid(logit)

        return prob
//  [python]
def forward(self, x):

        out    = self.preprocess(x)                                       #128

        conv1d = self.conv1d(out)                                         #128
        out    = F.max_pool2d(conv1d, kernel_size=2, stride=2)  # 64

        conv2d = self.conv2d(out) + make_shortcut(out, self.short2d)      # 64
        out    = F.max_pool2d(conv2d, kernel_size=2, stride=2) # 32
        flat2d = make_max_flat(out)

        conv3d = self.conv3d(out) + make_shortcut(out, self.short3d)      # 32
        out    = F.max_pool2d(conv3d, kernel_size=2, stride=2) # 16
        flat3d = make_max_flat(out)

        conv4d = self.conv4d(out) + make_shortcut(out, self.short4d)      # 16
        out    = F.max_pool2d(conv4d, kernel_size=2, stride=2) #  8
        flat4d = make_max_flat(out)

        conv5d = self.conv5d(out) + make_shortcut(out, self.short5d)      #  8
        out    = conv5d                                        #  4
        flat5d = make_max_flat(out)

        out    = F.interpolate(out,scale_factor=2, mode='bilinear')      # 16
        out    = out + conv4d
        out    = self.conv4u(out)
        flat4u = make_max_flat(out)

        out    = F.interpolate(out,scale_factor=2, mode='bilinear')      # 32
        out    = out + conv3d
        out    = self.conv3u(out)
        flat3u = make_max_flat(out)

        out    = F.interpolate(out,scale_factor=2, mode='bilinear')      # 64
        out    = out + conv2d
        out    = self.conv2u(out)
        flat2u = make_max_flat(out)

        out    = F.interpolate(out,scale_factor=2, mode='bilinear')      #128
        out    = out + conv1d
        out    = self.conv1u(out)
        flat1u = make_max_flat(out)



        logit2d = self.cls2d(flat2d).unsqueeze(2)
        logit3d = self.cls3d(flat3d).unsqueeze(2)
        logit4d = self.cls4d(flat4d).unsqueeze(2)
        logit5d = self.cls5d(flat5d).unsqueeze(2)

        logit1u = self.cls1u(flat1u).unsqueeze(2)
        logit2u = self.cls2u(flat2u).unsqueeze(2)
        logit3u = self.cls3u(flat3u).unsqueeze(2)
        logit4u = self.cls4u(flat4u).unsqueeze(2)


        logit = torch.cat((logit2d,logit3d,logit4d,logit5d,logit1u,logit2u,logit3u,logit4u),2)

        logit = F.dropout(logit, p=0.15,training=self.training)
        logit = logit.sum(2)
        logit = logit.view(logit.size(0),logit.size(1)) #unsqueeze(2)
        prob = F.sigmoid(logit)

        return prob
//  [python]
def forward(self,x):
        feats = self.feats(x)
        pool4 = self.pool4(feats)
        pool5 = self.pool5(pool4)

        score_fconn = self.fconn(pool5)
        score_pool4 = self.score_pool4(pool4)

        resize_score_fconn = F.upsample_bilinear(score_fconn, score_pool4.size()[2:])
        prediction = resize_score_fconn + score_pool4
        upsample = F.upsample_bilinear(prediction, x.size()[2:])

        return self.activation(upsample)
//  [python]
def forward(self,x):
        feats = self.feats(x)
        pool4 = self.pool4(feats)
        pool5 = self.pool5(pool4)

        score_fconn = self.fconn(pool5)
        score_pool4 = self.score_pool4(pool4)

        resize_score_fconn = F.interpolate(score_fconn, size=score_pool4.size()[2:], mode='bilinear')
        prediction = resize_score_fconn + score_pool4
        upsample = F.interpolate(prediction, size=x.size()[2:], mode='bilinear')

        return self.activation(upsample)
//  [python]
def validate(**kwargs):
    # Retrieve training configuration
    data_loader = kwargs['data_loader']
    net = kwargs['net']
    loss = kwargs['loss']
    verbose = kwargs['verbose']

    # Default Parameters
    theta_c = 0.5
    crop_size = image_size  # size of cropped images for 'See Better'

    # metrics initialization
    batches = 0
    epoch_loss = 0
    epoch_acc = np.array([0, 0, 0], dtype='float')  # top - 1, 3, 5

    # begin validation
    start_time = time.time()
    net.eval()
    with torch.no_grad():
        for i, (X, y) in enumerate(data_loader):
            batch_start = time.time()
            X = X.to(torch.device(device))
            y = y.to(torch.device(device))

            # Raw Image
            y_pred_raw, feature_matrix, attention_map = net(X)
            # Object Localization and Refinement
            crop_mask = F.upsample_bilinear(
                attention_map, size=(X.size(2), X.size(3))) > theta_c
            crop_images = []
            for batch_index in range(crop_mask.size(0)):
                nonzero_indices = torch.nonzero(crop_mask[batch_index, 0, ...])
                height_min = nonzero_indices[:, 0].min()
                height_max = nonzero_indices[:, 0].max()
                width_min = nonzero_indices[:, 1].min()
                width_max = nonzero_indices[:, 1].max()
                crop_images.append(F.upsample_bilinear(
                    X[batch_index:batch_index + 1, :, height_min:height_max, width_min:width_max], size=crop_size))
            crop_images = torch.cat(crop_images, dim=0)
            y_pred_crop, _, _ = net(crop_images)

            # final prediction
            y_pred = (y_pred_raw + y_pred_crop) / 2

            # loss
            batch_loss = loss(y_pred, y)
            epoch_loss += batch_loss.item()

            # metrics: top-1, top-3, top-5 error
            epoch_acc += accuracy(y_pred, y, topk=(1, 3, 5))

            # end of this batch
            batches += 1
            batch_end = time.time()
            if verbose and ((i + 1) % verbose == 0):
                logging.info('\tBatch %d: Loss %.5f, Accuracy: Top-1 %.3f, Top-3 %.3f, Top-5 %.3f, Time %3.2f' %
                             (i + 1, epoch_loss / batches, epoch_acc[0] / batches, epoch_acc[1] / batches,
                              epoch_acc[2] / batches, batch_end - batch_start))

    # end of validation
    end_time = time.time()

    # metrics for average
    epoch_loss /= batches
    epoch_acc /= batches

    # show information for this epoch
    logging.info('Valid: Loss %.5f,  Accuracy: Top-1 %.2f, Top-3 %.2f, Top-5 %.2f, Time %3.2f' %
                 (epoch_loss, epoch_acc[0], epoch_acc[1], epoch_acc[2], end_time - start_time))

    return epoch_loss, epoch_acc[0]
//  [python]
def validate(**kwargs):
    # Retrieve training configuration
    data_loader = kwargs['data_loader']
    net = kwargs['net']
    loss = kwargs['loss']
    verbose = kwargs['verbose']

    # Default Parameters
    theta_c = 0.5
    crop_size = image_size  # size of cropped images for 'See Better'

    # metrics initialization
    batches = 0
    epoch_loss = 0
    epoch_acc = np.array([0, 0, 0], dtype='float')  # top - 1, 3, 5

    # begin validation
    start_time = time.time()
    net.eval()
    with torch.no_grad():
        for i, (X, y) in enumerate(data_loader):
            batch_start = time.time()
            X = X.to(torch.device(device))
            y = y.to(torch.device(device))

            # Raw Image
            y_pred_raw, feature_matrix, attention_map = net(X)
            # Object Localization and Refinement
            crop_mask = F.interpolate(
                attention_map, size=(X.size(2), X.size(3)), mode='bilinear') > theta_c
            crop_images = []
            for batch_index in range(crop_mask.size(0)):
                nonzero_indices = torch.nonzero(crop_mask[batch_index, 0, ...])
                height_min = nonzero_indices[:, 0].min()
                height_max = nonzero_indices[:, 0].max()
                width_min = nonzero_indices[:, 1].min()
                width_max = nonzero_indices[:, 1].max()
                crop_images.append(F.interpolate(
                    X[batch_index:batch_index + 1, :, height_min:height_max, width_min:width_max], size=crop_size, mode='bilinear'))
            crop_images = torch.cat(crop_images, dim=0)
            y_pred_crop, _, _ = net(crop_images)

            # final prediction
            y_pred = (y_pred_raw + y_pred_crop) / 2

            # loss
            batch_loss = loss(y_pred, y)
            epoch_loss += batch_loss.item()

            # metrics: top-1, top-3, top-5 error
            epoch_acc += accuracy(y_pred, y, topk=(1, 3, 5))

            # end of this batch
            batches += 1
            batch_end = time.time()
            if verbose and ((i + 1) % verbose == 0):
                logging.info('\tBatch %d: Loss %.5f, Accuracy: Top-1 %.3f, Top-3 %.3f, Top-5 %.3f, Time %3.2f' %
                             (i + 1, epoch_loss / batches, epoch_acc[0] / batches, epoch_acc[1] / batches,
                              epoch_acc[2] / batches, batch_end - batch_start))

    # end of validation
    end_time = time.time()

    # metrics for average
    epoch_loss /= batches
    epoch_acc /= batches

    # show information for this epoch
    logging.info('Valid: Loss %.5f,  Accuracy: Top-1 %.2f, Top-3 %.2f, Top-5 %.2f, Time %3.2f' %
                 (epoch_loss, epoch_acc[0], epoch_acc[1], epoch_acc[2], end_time - start_time))

    return epoch_loss, epoch_acc[0]
//  [python]
def forward(self, x):
        h = x.size(2)
        w = x.size(3)

        conv1 = self.conv1(x)
        conv2 = self.conv2(conv1)
        conv3 = self.conv3(conv2)
        conv4 = self.conv4(conv3)
        conv5 = self.conv5(conv4)

        ## side output
        d1 = self.dsn1(conv1)
        d2 = F.upsample_bilinear(self.dsn2(conv2), size=(h,w))
        d3 = F.upsample_bilinear(self.dsn3(conv3), size=(h,w))
        d4 = F.upsample_bilinear(self.dsn4(conv4), size=(h,w))
        d5 = F.upsample_bilinear(self.dsn5(conv5), size=(h,w))

        # dsn fusion output
        fuse = self.fuse(torch.cat((d1, d2, d3, d4, d5), 1))
        
        d1 = F.sigmoid(d1)
        d2 = F.sigmoid(d2)
        d3 = F.sigmoid(d3)
        d4 = F.sigmoid(d4)
        d5 = F.sigmoid(d5)
        fuse = F.sigmoid(fuse)

        return d1, d2, d3, d4, d5, fuse
//  [python]
def forward(self, x):
        h = x.size(2)
        w = x.size(3)

        conv1 = self.conv1(x)
        conv2 = self.conv2(conv1)
        conv3 = self.conv3(conv2)
        conv4 = self.conv4(conv3)
        conv5 = self.conv5(conv4)

        ## side output
        d1 = self.dsn1(conv1)
        d2 = F.interpolate(self.dsn2(conv2), size=(h,w), mode='bilinear')
        d3 = F.interpolate(self.dsn3(conv3), size=(h,w), mode='bilinear')
        d4 = F.interpolate(self.dsn4(conv4), size=(h,w), mode='bilinear')
        d5 = F.interpolate(self.dsn5(conv5), size=(h,w), mode='bilinear')

        # dsn fusion output
        fuse = self.fuse(torch.cat((d1, d2, d3, d4, d5), 1))
        
        d1 = F.sigmoid(d1)
        d2 = F.sigmoid(d2)
        d3 = F.sigmoid(d3)
        d4 = F.sigmoid(d4)
        d5 = F.sigmoid(d5)
        fuse = F.sigmoid(fuse)

        return d1, d2, d3, d4, d5, fuse
//  [python]
def forward(self, x):
        feats = self.feats(x)
        pool3 = self.pool3(feats)
        pool4 = self.pool4(pool3)
        pool5 = self.pool5(pool4)
        fconn = self.fconn(pool5)

        score_pool3 = self.score_pool3(pool3)
        score_pool4 = self.score_pool4(pool4)

        resized_score_pool4 = F.upsample_bilinear(score_pool4, pool3.size()[2:])
        resized_score_fconn = F.upsample_bilinear(fconn, pool3.size()[2:])

        prediction = resized_score_pool4 + resized_score_fconn + score_pool3
        upsample = F.upsample_bilinear(prediction, x.size()[2:])

        return self.activation(upsample)
//  [python]
def forward(self, x):
        feats = self.feats(x)
        pool3 = self.pool3(feats)
        pool4 = self.pool4(pool3)
        pool5 = self.pool5(pool4)
        fconn = self.fconn(pool5)

        score_pool3 = self.score_pool3(pool3)
        score_pool4 = self.score_pool4(pool4)

        resized_score_pool4 = F.interpolate(score_pool4, size=pool3.size()[2:], mode='bilinear', align_corners=False)
        resized_score_fconn = F.interpolate(fconn, size=pool3.size()[2:], mode='bilinear', align_corners=False)

        prediction = resized_score_pool4 + resized_score_fconn + score_pool3
        upsample = F.interpolate(prediction, size=x.size()[2:], mode='bilinear', align_corners=False)

        return self.activation(upsample)
//  [python]
def forward(self, input,lr_jacobian):
        lr_jacobian = F.softmin(lr_jacobian.mean(dim=1,keepdim=True), dim=2)
        #lr_jacobian = F.softmax(self.convs1(lr_jacobian), dim=1)

        features = self.feature_extractor(input)  #b,648,n # b,648,n
        #jocb = self.feature_extractor2(input*lr_jacobian)

        upf = self.upsampling(features*lr_jacobian+features) # b,128,4*n

        coord = self.conv1(upf)
        coord = self.conv2(coord)
        return coord + F.upsample_nearest(input,scale_factor=2)
//  [python]
def forward(self, input,lr_jacobian):
        lr_jacobian = F.softmin(lr_jacobian.mean(dim=1,keepdim=True), dim=2)
        #lr_jacobian = F.softmax(self.convs1(lr_jacobian), dim=1)

        features = self.feature_extractor(input)  #b,648,n # b,648,n
        #jocb = self.feature_extractor2(input*lr_jacobian)

        upf = self.upsampling(features*lr_jacobian+features) # b,128,4*n

        coord = self.conv1(upf)
        coord = self.conv2(coord)
        return coord + F.interpolate(input,scale_factor=2,mode='nearest')
//  [python]
def forward(self, Vt, Vt_1, flo, flo_back, real_projection_t, real_projection_t_1):
        virtual_projection_t = torch_GetVirtualProjection(self.static_options, Vt) 
        virtual_projection_t_1 = torch_GetVirtualProjection(self.static_options, Vt_1) 

        b, h, w = flo.size()[:3]

        grid_t = torch_GetForwardGrid(self.static_options, real_projection_t, virtual_projection_t)[:,:2,:,:].permute(0,1,3,2)
        grid_t = torch.nn.functional.upsample_bilinear(grid_t, size = (h, w)) # [B,C(xy),H,W]

        grid_t_1 = torch_GetForwardGrid(self.static_options, real_projection_t_1, virtual_projection_t_1)[:,:2,:,:].permute(0,1,3,2) 
        grid_t_1 = torch.nn.functional.upsample_bilinear(grid_t_1, size = (h, w)) # [B,C(xy),H,W]
        
        mesh = self.mesh.repeat(b, 1, 1, 1)
        flo = flo + mesh 
        flo_back = flo_back + mesh # [B,H,W,C]

        valid = (flo[:,:,:,0] > 0) * (flo[:,:,:,1] > 0) * (flo[:,:,:,0] < 1) * (flo[:,:,:,1] < 1)
        valid_f = torch.unsqueeze(valid, dim = 3).type(torch.cuda.FloatTensor)
        valid = torch.unsqueeze(valid, dim = 1).type(torch.cuda.FloatTensor)

        valid_back = (flo_back[:,:,:,0] > 0) * (flo_back[:,:,:,1] > 0) * (flo_back[:,:,:,0] < 1) * (flo_back[:,:,:,1] < 1)
        valid_back_f = torch.unsqueeze(valid_back, dim = 3).type(torch.cuda.FloatTensor) 
        valid_back = torch.unsqueeze(valid_back, dim = 1).type(torch.cuda.FloatTensor) # [B,C,H,W]

        flo = (flo * 2 - 1) * valid_f
        flo_back = (flo_back * 2 - 1) * valid_back_f

        forward_t = torch.nn.functional.grid_sample(grid_t, flo, padding_mode="reflection") # default bilinear
        backward_t_1 = torch.nn.functional.grid_sample(grid_t_1, flo_back, padding_mode="reflection") # default bilinear

        forward_diff = ((forward_t - grid_t_1) * valid) ** 2 
        backward_diff = ((backward_t_1 - grid_t) * valid_back) ** 2

        forward_loss = torch.sum(forward_diff, dim = (1,2,3)) / torch.sum(valid, dim = (1,2,3))
        backward_loss = torch.sum(backward_diff, dim = (1,2,3)) / torch.sum(valid_back, dim = (1,2,3))

        loss = forward_loss + backward_loss
        loss = torch.min(loss, loss - loss + 1) #[0]
        loss = torch.sum(loss) / b

        return loss
//  [python]
def forward(self, Vt, Vt_1, flo, flo_back, real_projection_t, real_projection_t_1):
        virtual_projection_t = torch_GetVirtualProjection(self.static_options, Vt) 
        virtual_projection_t_1 = torch_GetVirtualProjection(self.static_options, Vt_1) 

        b, h, w = flo.size()[:3]

        grid_t = torch_GetForwardGrid(self.static_options, real_projection_t, virtual_projection_t)[:,:2,:,:].permute(0,1,3,2)
        grid_t = torch.nn.functional.interpolate(grid_t, size = (h, w), mode='bilinear') # [B,C(xy),H,W]

        grid_t_1 = torch_GetForwardGrid(self.static_options, real_projection_t_1, virtual_projection_t_1)[:,:2,:,:].permute(0,1,3,2) 
        grid_t_1 = torch.nn.functional.interpolate(grid_t_1, size = (h, w), mode='bilinear') # [B,C(xy),H,W]
        
        mesh = self.mesh.repeat(b, 1, 1, 1)
        flo = flo + mesh 
        flo_back = flo_back + mesh # [B,H,W,C]

        valid = (flo[:,:,:,0] > 0) * (flo[:,:,:,1] > 0) * (flo[:,:,:,0] < 1) * (flo[:,:,:,1] < 1)
        valid_f = torch.unsqueeze(valid, dim = 3).type(torch.cuda.FloatTensor)
        valid = torch.unsqueeze(valid, dim = 1).type(torch.cuda.FloatTensor)

        valid_back = (flo_back[:,:,:,0] > 0) * (flo_back[:,:,:,1] > 0) * (flo_back[:,:,:,0] < 1) * (flo_back[:,:,:,1] < 1)
        valid_back_f = torch.unsqueeze(valid_back, dim = 3).type(torch.cuda.FloatTensor) 
        valid_back = torch.unsqueeze(valid_back, dim = 1).type(torch.cuda.FloatTensor) # [B,C,H,W]

        flo = (flo * 2 - 1) * valid_f
        flo_back = (flo_back * 2 - 1) * valid_back_f

        forward_t = torch.nn.functional.grid_sample(grid_t, flo, padding_mode="reflection") # default bilinear
        backward_t_1 = torch.nn.functional.grid_sample(grid_t_1, flo_back, padding_mode="reflection") # default bilinear

        forward_diff = ((forward_t - grid_t_1) * valid) ** 2 
        backward_diff = ((backward_t_1 - grid_t) * valid_back) ** 2

        forward_loss = torch.sum(forward_diff, dim = (1,2,3)) / torch.sum(valid, dim = (1,2,3))
        backward_loss = torch.sum(backward_diff, dim = (1,2,3)) / torch.sum(valid_back, dim = (1,2,3))

        loss = forward_loss + backward_loss
        loss = torch.min(loss, loss - loss + 1) #[0]
        loss = torch.sum(loss) / b

        return loss
//  [python]
def forward(self, x):
        # (x has shape (batch_size, 3, h, w))

        h = x.size()[2]
        w = x.size()[3]

        feature_map = self.resnet(x) # (shape: (batch_size, 512, h/16, w/16)) (assuming self.resnet is ResNet18_OS16 or ResNet34_OS16. If self.resnet is ResNet18_OS8 or ResNet34_OS8, it will be (batch_size, 512, h/8, w/8). If self.resnet is ResNet50-152, it will be (batch_size, 4*512, h/16, w/16))

        output = self.aspp(feature_map) # (shape: (batch_size, num_classes, h/16, w/16))

        output = F.upsample(output, size=(h, w), mode="bilinear") # (shape: (batch_size, num_classes, h, w))

        return output
//  [python]
def forward(self, x):
        # (x has shape (batch_size, 3, h, w))

        h = x.size()[2]
        w = x.size()[3]

        feature_map = self.resnet(x) # (shape: (batch_size, 512, h/16, w/16)) (assuming self.resnet is ResNet18_OS16 or ResNet34_OS16. If self.resnet is ResNet18_OS8 or ResNet34_OS8, it will be (batch_size, 512, h/8, w/8). If self.resnet is ResNet50-152, it will be (batch_size, 4*512, h/16, w/16))

        output = self.aspp(feature_map) # (shape: (batch_size, num_classes, h/16, w/16))

        output = F.interpolate(output, size=(h, w), mode="bilinear") # (shape: (batch_size, num_classes, h, w))

        return output
//  [python]
def forward(self, *features):
        features = features[1:]  # remove first skip with same spatial resolution
        features = features[::-1]  # reverse channels to start from head of encoder

#         for ft in features:
#             print(ft.size())
#         print()

        head = features[0]
        skips = features[1:]

        x = self.center(head)

        xs = []
        for i, decoder_block in enumerate(self.blocks):
            skip = skips[i] if i < len(skips) else None

#             print(i, x.size(), skip.size() if skip is not None else None)
            x = decoder_block(x, skip)
            xs.append(x)

        if self.use_hypercolumns:
            h, w = x.size()[2:]
            x = torch.cat([
                F.upsample_bilinear(xs[-4], size=(h, w)),
                F.upsample_bilinear(xs[-3], size=(h, w)),
                F.upsample_bilinear(xs[-2], size=(h, w)),
                x
            ], 1)

            x = self.hypercolumns_conv(x)

        return x
//  [python]
def forward(self, *features):
        features = features[1:]  # remove first skip with same spatial resolution
        features = features[::-1]  # reverse channels to start from head of encoder

        head = features[0]
        skips = features[1:]

        x = self.center(head)

        xs = []
        for i, decoder_block in enumerate(self.blocks):
            skip = skips[i] if i < len(skips) else None
            x = decoder_block(x, skip)
            xs.append(x)

        if self.use_hypercolumns:
            h, w = x.size()[2:]
            x = torch.cat([
                F.interpolate(xs[-4], size=(h, w), mode='bilinear'),
                F.interpolate(xs[-3], size=(h, w), mode='bilinear'),
                F.interpolate(xs[-2], size=(h, w), mode='bilinear'),
                x
            ], 1)

            x = self.hypercolumns_conv(x)

        return x
//  [python]
def __getitem__(self, index):
        assert index <= len(self), 'index range error'

        img_path = self.img_list[index]
        gt_path = self.img_map[img_path]
        # load image and ground truth
        img, point = load_data((img_path, gt_path), self.train)
        # applu augumentation
        if self.transform is not None:
            img = self.transform(img)

        if self.train:
            # data augmentation -> random scale
            scale_range = [0.7, 1.3]
            min_size = min(img.shape[1:])
            scale = random.uniform(*scale_range)
            # scale the image and points
            if scale * min_size > 128:
                img = torch.nn.functional.upsample_bilinear(img.unsqueeze(0), scale_factor=scale).squeeze(0)
                point *= scale
        # random crop augumentaiton
        if self.train and self.patch:
            img, point = random_crop(img, point)
            for i, _ in enumerate(point):
                point[i] = torch.Tensor(point[i])
        # random flipping
        if random.random() > 0.5 and self.train and self.flip:
            # random flip
            img = torch.Tensor(img[:, :, :, ::-1].copy())
            for i, _ in enumerate(point):
                point[i][:, 0] = 128 - point[i][:, 0]

        if not self.train:
            point = [point]

        img = torch.Tensor(img)
        # pack up related infos
        target = [{} for i in range(len(point))]
        for i, _ in enumerate(point):
            target[i]['point'] = torch.Tensor(point[i])
            image_id = int(img_path.split('/')[-1].split('.')[0].split('_')[-1])
            image_id = torch.Tensor([image_id]).long()
            target[i]['image_id'] = image_id
            target[i]['labels'] = torch.ones([point[i].shape[0]]).long()

        return img, target
//  [python]
def __getitem__(self, index):
        assert index <= len(self), 'index range error'

        img_path = self.img_list[index]
        gt_path = self.img_map[img_path]
        # load image and ground truth
        img, point = load_data((img_path, gt_path), self.train)
        # applu augumentation
        if self.transform is not None:
            img = self.transform(img)

        if self.train:
            # data augmentation -> random scale
            scale_range = [0.7, 1.3]
            min_size = min(img.shape[1:])
            scale = random.uniform(*scale_range)
            # scale the image and points
            if scale * min_size > 128:
                img = torch.nn.functional.interpolate(img.unsqueeze(0), scale_factor=scale, mode='bilinear').squeeze(0)
                point *= scale
        # random crop augumentaiton
        if self.train and self.patch:
            img, point = random_crop(img, point)
            for i, _ in enumerate(point):
                point[i] = torch.Tensor(point[i])
        # random flipping
        if random.random() > 0.5 and self.train and self.flip:
            # random flip
            img = torch.Tensor(img[:, :, :, ::-1].copy())
            for i, _ in enumerate(point):
                point[i][:, 0] = 128 - point[i][:, 0]

        if not self.train:
            point = [point]

        img = torch.Tensor(img)
        # pack up related infos
        target = [{} for i in range(len(point))]
        for i, _ in enumerate(point):
            target[i]['point'] = torch.Tensor(point[i])
            image_id = int(img_path.split('/')[-1].split('.')[0].split('_')[-1])
            image_id = torch.Tensor([image_id]).long()
            target[i]['image_id'] = image_id
            target[i]['labels'] = torch.ones([point[i].shape[0]]).long()

        return img, target
//  [python]
def _forward(self, level, inp):
        # Upper branch
        up1 = inp
        up1 = self._modules['b1_' + str(level)](up1)

        # Lower branch
        low1 = F.avg_pool2d(inp, 2, stride=2)
        low1 = self._modules['b2_' + str(level)](low1)

        if level > 1:
            low2, klow2 = self._forward(level - 1, low1)
            low3 = low2
            low2 = klow2
        else:
            low2 = low1
            low2 = self._modules['b2_plus_' + str(level)](low2)
            low3 = low2

        low3 = self._modules['b3_' + str(level)](low3)

        #up2 = F.interpolate(low3, scale_factor=2, mode='nearest')
        up2 = F.upsample_nearest(low3, scale_factor=2)

        return up1 + up2, low2
//  [python]
def _forward(self, level, inp):
        # Upper branch
        up1 = inp
        up1 = self._modules['b1_' + str(level)](up1)

        # Lower branch
        low1 = F.avg_pool2d(inp, 2, stride=2)
        low1 = self._modules['b2_' + str(level)](low1)

        if level > 1:
            low2, klow2 = self._forward(level - 1, low1)
            low3 = low2
            low2 = klow2
        else:
            low2 = low1
            low2 = self._modules['b2_plus_' + str(level)](low2)
            low3 = low2

        low3 = self._modules['b3_' + str(level)](low3)

        up2 = F.interpolate(low3, scale_factor=2, mode='nearest')
        #up2 = F.upsample_nearest(low3, scale_factor=2)

        return up1 + up2, low2
//  [python]
def forward(self, x):
        residual = x
        
        h, w = x.size(2), x.size(3)
        
        out1 = self.conv1(x)        
        out1 = F.upsample(input=out1, size=(h, w), mode='bilinear')
        out2 = self.conv2(x)        
        out2 = F.upsample(input=out2, size=(h, w), mode='bilinear')
        out3 = self.conv3(x)        
        out3 = F.upsample(input=out3, size=(h, w), mode='bilinear')
        out4 = self.conv4(x)        
        out4 = F.upsample(input=out4, size=(h, w), mode='bilinear')
        
        out = torch.cat([out1, out2, out3, out4, residual], 1)
        return out
//  [python]
def forward(self, x):
        residual = x
        
        h, w = x.size(2), x.size(3)
        
        out1 = self.conv1(x)        
        out1 = F.interpolate(input=out1, size=(h, w), mode='bilinear')
        out2 = self.conv2(x)        
        out2 = F.interpolate(input=out2, size=(h, w), mode='bilinear')
        out3 = self.conv3(x)        
        out3 = F.interpolate(input=out3, size=(h, w), mode='bilinear')
        out4 = self.conv4(x)        
        out4 = F.interpolate(input=out4, size=(h, w), mode='bilinear')
        
        out = torch.cat([out1, out2, out3, out4, residual], 1)
        return out
//  [python]
def forward(self,x,alpha=0.5,downsamples=0,embedding=None,useStylePredictor=True):
        # augments a batch of images with style randomization
        # x: B x C x H x W image tensor
        # alpha: float in [0,1], controls interpolation between random style and original style
        # downsamples: int, number of times to downsample by factor of 2 before applying style transfer
        # embedding: B x 100 tensor, or None. Use this embedding if provided.
        # useStylePredictor: bool. If True, we use the inception based style predictor to compute the original style embedding for the input image, and use that for interpolation. If False, we use the mean ImageNet embedding instead, which is slightly faster.

        # style embedding for when alpha=0:
        base = self.stylePredictor(x) if useStylePredictor else self.imagenet_embedding

        if downsamples:
            assert(x.size(2) % 2**downsamples == 0)
            assert(x.size(3) % 2**downsamples == 0)
            for i in range(downsamples):
                x = nn.functional.avg_pool2d(x,2)

        if embedding is None:
            # sample a random embedding
            embedding = self.sample_embedding(x.size(0))
        # interpolate style embeddings:
        embedding = alpha*embedding + (1-alpha)*base
        
        restyled = self.ghiasi(x,embedding)

        if downsamples:
            restyled = nn.functional.upsample(restyled,scale_factor=2**downsamples,mode='bilinear')
        
        return restyled.detach()
//  [python]
def forward(self,x,alpha=0.5,downsamples=0,embedding=None,useStylePredictor=True):
        # augments a batch of images with style randomization
        # x: B x C x H x W image tensor
        # alpha: float in [0,1], controls interpolation between random style and original style
        # downsamples: int, number of times to downsample by factor of 2 before applying style transfer
        # embedding: B x 100 tensor, or None. Use this embedding if provided.
        # useStylePredictor: bool. If True, we use the inception based style predictor to compute the original style embedding for the input image, and use that for interpolation. If False, we use the mean ImageNet embedding instead, which is slightly faster.

        # style embedding for when alpha=0:
        base = self.stylePredictor(x) if useStylePredictor else self.imagenet_embedding

        if downsamples:
            assert(x.size(2) % 2**downsamples == 0)
            assert(x.size(3) % 2**downsamples == 0)
            for i in range(downsamples):
                x = nn.functional.avg_pool2d(x,2)

        if embedding is None:
            # sample a random embedding
            embedding = self.sample_embedding(x.size(0))
        # interpolate style embeddings:
        embedding = alpha*embedding + (1-alpha)*base
        
        restyled = self.ghiasi(x,embedding)

        if downsamples:
            restyled = nn.functional.interpolate(restyled,scale_factor=2**downsamples,mode='bilinear')
        
        return restyled.detach()
//  [python]
def forward(self, x):
    # Normalize x
    x = (x + 1.) / 2.0
    x = (x - self.mean) / self.std
    # Upsample if necessary
    if x.shape[2] != 299 or x.shape[3] != 299:
      if torch.__version__ in ['0.4.0']:
        x = F.upsample_bilinear(x, size=(299, 299))
      else:
        x = F.interpolate(x, size=(299, 299), mode='bilinear', align_corners=True)
    # 299 x 299 x 3
    x = self.net.Conv2d_1a_3x3(x)
    # 149 x 149 x 32
    x = self.net.Conv2d_2a_3x3(x)
    # 147 x 147 x 32
    x = self.net.Conv2d_2b_3x3(x)
    # 147 x 147 x 64
    x = F.max_pool2d(x, kernel_size=3, stride=2)
    # 73 x 73 x 64
    x = self.net.Conv2d_3b_1x1(x)
    # 73 x 73 x 80
    x = self.net.Conv2d_4a_3x3(x)
    # 71 x 71 x 192
    x = F.max_pool2d(x, kernel_size=3, stride=2)
    # 35 x 35 x 192
    x = self.net.Mixed_5b(x)
    # 35 x 35 x 256
    x = self.net.Mixed_5c(x)
    # 35 x 35 x 288
    x = self.net.Mixed_5d(x)
    # 35 x 35 x 288
    x = self.net.Mixed_6a(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6b(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6c(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6d(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6e(x)
    # 17 x 17 x 768
    # 17 x 17 x 768
    x = self.net.Mixed_7a(x)
    # 8 x 8 x 1280
    x = self.net.Mixed_7b(x)
    # 8 x 8 x 2048
    x = self.net.Mixed_7c(x)
    # 8 x 8 x 2048
    pool = torch.mean(x.view(x.size(0), x.size(1), -1), 2)
    # 1 x 1 x 2048
    logits = self.net.fc(F.dropout(pool, training=False).view(pool.size(0), -1))
    # 1000 (num_classes)
    return pool, logits
//  [python]
def forward(self, x):
    # Normalize x
    x = (x + 1.) / 2.0
    x = (x - self.mean) / self.std
    # Upsample if necessary
    if x.shape[2] != 299 or x.shape[3] != 299:
      x = F.interpolate(x, size=(299, 299), mode='bilinear', align_corners=True)
    # 299 x 299 x 3
    x = self.net.Conv2d_1a_3x3(x)
    # 149 x 149 x 32
    x = self.net.Conv2d_2a_3x3(x)
    # 147 x 147 x 32
    x = self.net.Conv2d_2b_3x3(x)
    # 147 x 147 x 64
    x = F.max_pool2d(x, kernel_size=3, stride=2)
    # 73 x 73 x 64
    x = self.net.Conv2d_3b_1x1(x)
    # 73 x 73 x 80
    x = self.net.Conv2d_4a_3x3(x)
    # 71 x 71 x 192
    x = F.max_pool2d(x, kernel_size=3, stride=2)
    # 35 x 35 x 192
    x = self.net.Mixed_5b(x)
    # 35 x 35 x 256
    x = self.net.Mixed_5c(x)
    # 35 x 35 x 288
    x = self.net.Mixed_5d(x)
    # 35 x 35 x 288
    x = self.net.Mixed_6a(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6b(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6c(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6d(x)
    # 17 x 17 x 768
    x = self.net.Mixed_6e(x)
    # 17 x 17 x 768
    # 17 x 17 x 768
    x = self.net.Mixed_7a(x)
    # 8 x 8 x 1280
    x = self.net.Mixed_7b(x)
    # 8 x 8 x 2048
    x = self.net.Mixed_7c(x)
    # 8 x 8 x 2048
    pool = torch.mean(x.view(x.size(0), x.size(1), -1), 2)
    # 1 x 1 x 2048
    logits = self.net.fc(F.dropout(pool, training=False).view(pool.size(0), -1))
    # 1000 (num_classes)
    return pool, logits
//  [python]
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        """

        Parameters
        ----------
        x: torch.Tensor
            Input image

        Returns
        -------
        all_features: torch.Tensor
            Features per landmark
        maps: torch.Tensor
            Attention maps per landmark
        scores: torch.Tensor
            Classification scores per landmark
        """
        # Pretrained ResNet part of the model
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)
        x = self.layer1(x)
        x = self.layer2(x)
        l3 = self.layer3(x)
        x = self.layer4(l3)
        x = torch.nn.functional.upsample_bilinear(x, size=(l3.shape[-2], l3.shape[-1]))
        x = torch.cat((x, l3), dim=1)

        # Compute per landmark attention maps
        # (b - a)^2 = b^2 - 2ab + a^2, b = feature maps resnet, a = convolution kernel
        batch_size = x.shape[0]
        ab = self.fc_landmarks(x)
        b_sq = x.pow(2).sum(1, keepdim=True)
        b_sq = b_sq.expand(-1, self.num_landmarks + 1, -1, -1)
        a_sq = self.fc_landmarks.weight.pow(2).sum(1).unsqueeze(1).expand(-1, batch_size, x.shape[-2], x.shape[-1])
        a_sq = a_sq.permute(1, 0, 2, 3)
        maps = b_sq - 2 * ab + a_sq
        maps = -maps

        # Softmax so that the attention maps for each pixel add up to 1
        maps = self.softmax(maps)

        # Use maps to get weighted average features per landmark
        feature_tensor = x
        all_features = ((maps).unsqueeze(1) * feature_tensor.unsqueeze(2)).mean(-1).mean(-1)

        # Classification based on the landmarks
        all_features_modulated = all_features * self.modulation
        all_features_modulated = self.dropout_full_landmarks(all_features_modulated.permute(0,2,1)).permute(0,2,1)
        scores = self.fc_class_landmarks(all_features_modulated.permute(0, 2, 1)).permute(0, 2, 1)

        return all_features, maps, scores
//  [python]
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        """

        Parameters
        ----------
        x: torch.Tensor
            Input image

        Returns
        -------
        all_features: torch.Tensor
            Features per landmark
        maps: torch.Tensor
            Attention maps per landmark
        scores: torch.Tensor
            Classification scores per landmark
        """
        # Pretrained ResNet part of the model
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)
        x = self.layer1(x)
        x = self.layer2(x)
        l3 = self.layer3(x)
        x = self.layer4(l3)
        x = torch.nn.functional.interpolate(x, size=(l3.shape[-2], l3.shape[-1]), mode='bilinear')
        x = torch.cat((x, l3), dim=1)

        # Compute per landmark attention maps
        # (b - a)^2 = b^2 - 2ab + a^2, b = feature maps resnet, a = convolution kernel
        batch_size = x.shape[0]
        ab = self.fc_landmarks(x)
        b_sq = x.pow(2).sum(1, keepdim=True)
        b_sq = b_sq.expand(-1, self.num_landmarks + 1, -1, -1)
        a_sq = self.fc_landmarks.weight.pow(2).sum(1).unsqueeze(1).expand(-1, batch_size, x.shape[-2], x.shape[-1])
        a_sq = a_sq.permute(1, 0, 2, 3)
        maps = b_sq - 2 * ab + a_sq
        maps = -maps

        # Softmax so that the attention maps for each pixel add up to 1
        maps = self.softmax(maps)

        # Use maps to get weighted average features per landmark
        feature_tensor = x
        all_features = ((maps).unsqueeze(1) * feature_tensor.unsqueeze(2)).mean(-1).mean(-1)

        # Classification based on the landmarks
        all_features_modulated = all_features * self.modulation
        all_features_modulated = self.dropout_full_landmarks(all_features_modulated.permute(0,2,1)).permute(0,2,1)
        scores = self.fc_class_landmarks(all_features_modulated.permute(0, 2, 1)).permute(0, 2, 1)

        return all_features, maps, scores
//  [python]
def _upsample_add(self, x, y):
        _,_,H,W = y.size()
        return F.upsample(x, size=(H,W), mode='bilinear') + y
//  [python]
def forward(self, x):
        x = self.layer1(x) 
        x = self.layer1plus(x)  
        x = self.layer2(x)
        x = self.layer2plus_to_5(x)  
        x = self.de_pred(x)

        x = F.upsample(x,scale_factor=16)

        return x
//  [python]
def forward(self, x):
        x = self.layer1(x) 
        x = self.layer1plus(x)  
        x = self.layer2(x)
        x = self.layer2plus_to_5(x)  
        x = self.de_pred(x)

        x = F.interpolate(x,scale_factor=16)

        return x
//  [python]
def forward(self, inputs):
        visual_feats = inputs["visual"]
        language_dict_features = inputs["lang"]

        next_x = []
        for level, feature in enumerate(visual_feats):

            conv_args = dict()
            if self.offset is not None:
                offset_mask = self.offset(feature)
                offset = offset_mask[:, :18, :, :]
                mask = offset_mask[:, 18:, :, :].sigmoid()
                conv_args = dict(offset=offset, mask=mask)

            temp_fea = [self.DyConv[1](feature, **conv_args)]

            if level > 0:
                temp_fea.append(self.DyConv[2](visual_feats[level - 1], **conv_args))
            if level < len(visual_feats) - 1:
                offset = conv_args.get('offset', None)
                mask = conv_args.get('mask', None)
                out_h, out_w = visual_feats[level + 1].shape[2:]
                offset_nchw = reorder(offset, out_h, out_w)
                mask_nchw = reorder(mask, out_h, out_w)
                newconv_args = dict(offset=offset_nchw, mask=mask_nchw)
                temp_fea.append(F.upsample_bilinear(self.DyConv[0](visual_feats[level + 1], **newconv_args),
                                                    size=[feature.size(2), feature.size(3)]))
            mean_fea = torch.mean(torch.stack(temp_fea), dim=0, keepdim=False)

            if self.AttnConv is not None:
                attn_fea = []
                res_fea = []
                for fea in temp_fea:
                    res_fea.append(fea)
                    attn_fea.append(self.AttnConv(fea))

                res_fea = torch.stack(res_fea)
                spa_pyr_attn = self.h_sigmoid(torch.stack(attn_fea))

                mean_fea = torch.mean(res_fea * spa_pyr_attn, dim=0, keepdim=False)

            next_x.append(mean_fea)

        next_x = [self.relu(item) for item in next_x]

        features_dict = {"visual": next_x,
                         "lang": language_dict_features}

        return features_dict
//  [python]
def forward(self, inputs):
        visual_feats = inputs["visual"]
        language_dict_features = inputs["lang"]

        next_x = []
        for level, feature in enumerate(visual_feats):

            conv_args = dict()
            if self.offset is not None:
                offset_mask = self.offset(feature)
                offset = offset_mask[:, :18, :, :]
                mask = offset_mask[:, 18:, :, :].sigmoid()
                conv_args = dict(offset=offset, mask=mask)

            temp_fea = [self.DyConv[1](feature, **conv_args)]

            if level > 0:
                temp_fea.append(self.DyConv[2](visual_feats[level - 1], **conv_args))
            if level < len(visual_feats) - 1:
                offset = conv_args.get('offset', None)
                mask = conv_args.get('mask', None)
                out_h, out_w = visual_feats[level + 1].shape[2:]
                offset_nchw = reorder(offset, out_h, out_w)
                mask_nchw = reorder(mask, out_h, out_w)
                newconv_args = dict(offset=offset_nchw, mask=mask_nchw)
                temp_fea.append(F.interpolate(self.DyConv[0](visual_feats[level + 1], **newconv_args),
                                                    size=[feature.size(2), feature.size(3)], mode='bilinear'))
            mean_fea = torch.mean(torch.stack(temp_fea), dim=0, keepdim=False)

            if self.AttnConv is not None:
                attn_fea = []
                res_fea = []
                for fea in temp_fea:
                    res_fea.append(fea)
                    attn_fea.append(self.AttnConv(fea))

                res_fea = torch.stack(res_fea)
                spa_pyr_attn = self.h_sigmoid(torch.stack(attn_fea))

                mean_fea = torch.mean(res_fea * spa_pyr_attn, dim=0, keepdim=False)

            next_x.append(mean_fea)

        next_x = [self.relu(item) for item in next_x]

        features_dict = {"visual": next_x,
                         "lang": language_dict_features}

        return features_dict