jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def imenlarge2(self, im):
        _, _, M, N = im.shape
        t1 = F.upsample(im, size=(int(4*M-3), int(4*N-3)), mode='bilinear')
        t2 = torch.zeros([1, 1, 4*M-1, 4*N-1]).type(self.samplet.type())
        t2[:, :, 1: -1, 1:-1] = t1
        t2[:, :, 0, :] = 2*t2[:, :, 1, :] - t2[:, :, 2, :]
        t2[:, :, -1, :] = 2*t2[:, :, -2, :] - t2[:, :, -3, :]
        t2[:, :, :, 0] = 2*t2[:, :, :, 1] - t2[:, :, :, 2]
        t2[:, :, :, -1] = 2*t2[:, :, :, -2] - t2[:, :, :, -3]
        imu = t2[:, :, ::2, ::2]

        return imu
//  [python]
def imenlarge2(self, im):
        _, _, M, N = im.shape
        t1 = F.interpolate(im, size=(int(4*M-3), int(4*N-3)), mode='bilinear')
        t2 = torch.zeros([1, 1, 4*M-1, 4*N-1]).type(self.samplet.type())
        t2[:, :, 1: -1, 1:-1] = t1
        t2[:, :, 0, :] = 2*t2[:, :, 1, :] - t2[:, :, 2, :]
        t2[:, :, -1, :] = 2*t2[:, :, -2, :] - t2[:, :, -3, :]
        t2[:, :, :, 0] = 2*t2[:, :, :, 1] - t2[:, :, :, 2]
        t2[:, :, :, -1] = 2*t2[:, :, :, -2] - t2[:, :, :, -3]
        imu = t2[:, :, ::2, ::2]

        return imu
//  [python]
def _forward(self, level, inp):
        # Upper branch
        up1 = inp
        up1 = self._modules['b1_' + str(level)](up1)
        up1 = self.dropout(up1)
        # Lower branch
        low1 = F.max_pool2d(inp, 2, stride=2)
        low1 = self._modules['b2_' + str(level)](low1)

        if level > 1:
            low2 = self._forward(level - 1, low1)
        else:
            low2 = low1
            low2 = self._modules['b2_plus_' + str(level)](low2)

        low3 = low2
        low3 = self._modules['b3_' + str(level)](low3)
        up1size = up1.size()
        rescale_size = (up1size[2], up1size[3])
        up2 = F.upsample(low3, size=rescale_size, mode='bilinear')

        return up1 + up2
//  [python]
def _forward(self, level, inp):
        # Upper branch
        up1 = inp
        up1 = self._modules['b1_' + str(level)](up1)
        up1 = self.dropout(up1)
        # Lower branch
        low1 = F.max_pool2d(inp, 2, stride=2)
        low1 = self._modules['b2_' + str(level)](low1)

        if level > 1:
            low2 = self._forward(level - 1, low1)
        else:
            low2 = low1
            low2 = self._modules['b2_plus_' + str(level)](low2)

        low3 = low2
        low3 = self._modules['b3_' + str(level)](low3)
        up1size = up1.size()
        rescale_size = (up1size[2], up1size[3])
        up2 = F.interpolate(low3, size=rescale_size, mode='bilinear')

        return up1 + up2
//  [python]
def gt_preprocess(gt,size):
    if len(gt.shape) > 2:
        gt = gt[:, :, 0]

    gt_tensor = torch.unsqueeze(torch.tensor(gt, dtype=torch.uint8),0)

    if(len(size)<2):
        return gt_tensor.type(torch.uint8), gt.shape[0:2]
    else:
        gt_tensor = torch.unsqueeze(torch.tensor(gt_tensor, dtype=torch.float32),0)
        gt_tensor = F.upsample(gt_tensor, size, mode="bilinear")
        gt_tensor = torch.squeeze(gt_tensor,0)

    return gt_tensor.type(torch.uint8), gt.shape[0:2]
//  [python]
def gt_preprocess(gt,size):
    if len(gt.shape) > 2:
        gt = gt[:, :, 0]

    gt_tensor = torch.unsqueeze(torch.tensor(gt, dtype=torch.uint8),0)

    if(len(size)<2):
        return gt_tensor.type(torch.uint8), gt.shape[0:2]
    else:
        gt_tensor = torch.unsqueeze(torch.tensor(gt_tensor, dtype=torch.float32),0)
        gt_tensor = F.interpolate(gt_tensor, size, mode="bilinear")
        gt_tensor = torch.squeeze(gt_tensor,0)

    return gt_tensor.type(torch.uint8), gt.shape[0:2]
//  [python]
def __call__(self,sample):
        imidx, image, label, shape =  sample['imidx'], sample['image'], sample['label'], sample['shape']

        # import time
        # start = time.time()

        image = torch.squeeze(F.upsample(torch.unsqueeze(image,0),self.size,mode='bilinear'),dim=0)
        label = torch.squeeze(F.upsample(torch.unsqueeze(label,0),self.size,mode='bilinear'),dim=0)

        # print("time for resize: ", time.time()-start)

        return {'imidx':imidx,'image':image, 'label':label, 'shape':shape}
//  [python]
def __call__(self,sample):
        imidx, image, label, shape =  sample['imidx'], sample['image'], sample['label'], sample['shape']

        image = torch.squeeze(F.interpolate(torch.unsqueeze(image,0),self.size,mode='bilinear'),dim=0)
        label = torch.squeeze(F.interpolate(torch.unsqueeze(label,0),self.size,mode='bilinear'),dim=0)

        return {'imidx':imidx,'image':image, 'label':label, 'shape':shape}
//  [python]
def im_preprocess(im,size):
    if len(im.shape) < 3:
        im = im[:, :, np.newaxis]
    if im.shape[2] == 1:
        im = np.repeat(im, 3, axis=2)
    im_tensor = torch.tensor(im.copy(), dtype=torch.float32)
    im_tensor = torch.transpose(torch.transpose(im_tensor,1,2),0,1)
    if(len(size)<2):
        return im_tensor, im.shape[0:2]
    else:
        im_tensor = torch.unsqueeze(im_tensor,0)
        im_tensor = F.upsample(im_tensor, size, mode="bilinear")
        im_tensor = torch.squeeze(im_tensor,0)

    return im_tensor.type(torch.uint8), im.shape[0:2]
//  [python]
def im_preprocess(im,size):
    if len(im.shape) < 3:
        im = im[:, :, np.newaxis]
    if im.shape[2] == 1:
        im = np.repeat(im, 3, axis=2)
    im_tensor = torch.tensor(im.copy(), dtype=torch.float32)
    im_tensor = torch.transpose(torch.transpose(im_tensor,1,2),0,1)
    if(len(size)<2):
        return im_tensor, im.shape[0:2]
    else:
        im_tensor = torch.unsqueeze(im_tensor,0)
        im_tensor = F.interpolate(im_tensor, size, mode="bilinear")
        im_tensor = torch.squeeze(im_tensor,0)

    return im_tensor.type(torch.uint8), im.shape[0:2]
//  [python]
def predict_img(net, full_img, gpu=False):
    img = resize_and_crop(full_img)

    left = get_square(img, 0)
    right = get_square(img, 1)

    right = normalize(right)
    left = normalize(left)

    right = np.transpose(right, axes=[2, 0, 1])
    left = np.transpose(left, axes=[2, 0, 1])

    X_l = torch.FloatTensor(left).unsqueeze(0)
    X_r = torch.FloatTensor(right).unsqueeze(0)

    if gpu:
        X_l = Variable(X_l, volatile=True).cuda()
        X_r = Variable(X_r, volatile=True).cuda()
    else:
        X_l = Variable(X_l, volatile=True)
        X_r = Variable(X_r, volatile=True)

    y_l = F.sigmoid(net(X_l))
    y_r = F.sigmoid(net(X_r))
    y_l = F.upsample_bilinear(y_l, scale_factor=2).data[0][0].cpu().numpy()
    y_r = F.upsample_bilinear(y_r, scale_factor=2).data[0][0].cpu().numpy()

    y = merge_masks(y_l, y_r, full_img.size[0])
    yy = dense_crf(np.array(full_img).astype(np.uint8), y)

    return yy > 0.5
//  [python]
def predict_img(net, full_img, gpu=False):
    img = resize_and_crop(full_img)

    left = get_square(img, 0)
    right = get_square(img, 1)

    right = normalize(right)
    left = normalize(left)

    right = np.transpose(right, axes=[2, 0, 1])
    left = np.transpose(left, axes=[2, 0, 1])

    X_l = torch.FloatTensor(left).unsqueeze(0)
    X_r = torch.FloatTensor(right).unsqueeze(0)

    if gpu:
        X_l = Variable(X_l, volatile=True).cuda()
        X_r = Variable(X_r, volatile=True).cuda()
    else:
        X_l = Variable(X_l, volatile=True)
        X_r = Variable(X_r, volatile=True)

    y_l = F.sigmoid(net(X_l))
    y_r = F.sigmoid(net(X_r))
    y_l = F.interpolate(y_l, scale_factor=2, mode='bilinear').data[0][0].cpu().numpy()
    y_r = F.interpolate(y_r, scale_factor=2, mode='bilinear').data[0][0].cpu().numpy()

    y = merge_masks(y_l, y_r, full_img.size[0])
    yy = dense_crf(np.array(full_img).astype(np.uint8), y)

    return yy > 0.5
//  [python]
def forward(self, x, t) -> torch.Tensor:
        x_skip = x

        t = self.f_t(F.silu(t))
        t_1, t_2 = t.chunk(2, dim=1)
        t_1 = t_1.unsqueeze(2).unsqueeze(3) + 1
        t_2 = t_2.unsqueeze(2).unsqueeze(3)

        gn_1 = F.silu(self.gn_1(x))
        upsample = F.upsample_nearest(gn_1, scale_factor=2)
        f_1 = self.f_1(upsample)
        gn_2 = self.gn_2(f_1)

        f_2 = self.f_2(F.silu(t_2 + (t_1 * gn_2)))

        return f_2 + F.upsample_nearest(x_skip, scale_factor=2)
//  [python]
def forward(self, x, t) -> torch.Tensor:
        x_skip = x

        t = self.f_t(F.silu(t))
        t_1, t_2 = t.chunk(2, dim=1)
        t_1 = t_1.unsqueeze(2).unsqueeze(3) + 1
        t_2 = t_2.unsqueeze(2).unsqueeze(3)

        gn_1 = F.silu(self.gn_1(x))
        upsample = F.interpolate(gn_1, scale_factor=2, mode='nearest')
        f_1 = self.f_1(upsample)
        gn_2 = self.gn_2(f_1)

        f_2 = self.f_2(F.silu(t_2 + (t_1 * gn_2)))

        return f_2 + F.interpolate(x_skip, scale_factor=2, mode='nearest')
//  [python]
def forward(self, x):
        x_size = x.size()
        out = [x]
        for f in self.features:
            out.append(F.upsample(f(x), x_size[2:], mode='bilinear'))
        out = torch.cat(out, 1)
        return out
//  [python]
def forward(self, x):
        x_size = x.size()
        out = [x]
        for f in self.features:
            out.append(F.interpolate(f(x), x_size[2:], mode='bilinear'))
        out = torch.cat(out, 1)
        return out
//  [python]
def forward(self, x, t, features) -> torch.Tensor:
        converted = hasattr(self, "converted") and self.converted

        x = torch.cat([x, F.upsample_nearest(features, scale_factor=8)], dim=1)

        if converted:
            t = self.time_embedding(self.time_proj(t))
        else:
            t = self.embed_time(t)

        x = self.embed_image(x)

        skips = [x]
        for i, down in enumerate(self.down):
            if converted and i in [0, 1, 2, 3]:
                x, skips_ = down(x, t)
                for skip in skips_:
                    skips.append(skip)
            else:
                for block in down:
                    x = block(x, t)
                    skips.append(x)
            print(x.float().abs().sum())

        if converted:
            x = self.mid(x, t)
        else:
            for i in range(2):
                x = self.mid[i](x, t)
        print(x.float().abs().sum())

        for i, up in enumerate(self.up[::-1]):
            if converted and i in [0, 1, 2, 3]:
                skip_4 = skips.pop()
                skip_3 = skips.pop()
                skip_2 = skips.pop()
                skip_1 = skips.pop()
                skips_ = (skip_1, skip_2, skip_3, skip_4)
                x = up(x, skips_, t)
            else:
                for block in up:
                    if isinstance(block, ConvResblock):
                        x = torch.concat([x, skips.pop()], dim=1)
                    x = block(x, t)

        return self.output(x)
//  [python]
def forward(self, x, t, features) -> torch.Tensor:
        converted = hasattr(self, "converted") and self.converted

        x = torch.cat([x, F.interpolate(features, scale_factor=8, mode='nearest')], dim=1)

        if converted:
            t = self.time_embedding(self.time_proj(t))
        else:
            t = self.embed_time(t)

        x = self.embed_image(x)

        skips = [x]
        for i, down in enumerate(self.down):
            if converted and i in [0, 1, 2, 3]:
                x, skips_ = down(x, t)
                for skip in skips_:
                    skips.append(skip)
            else:
                for block in down:
                    x = block(x, t)
                    skips.append(x)
            print(x.float().abs().sum())

        if converted:
            x = self.mid(x, t)
        else:
            for i in range(2):
                x = self.mid[i](x, t)
        print(x.float().abs().sum())

        for i, up in enumerate(self.up[::-1]):
            if converted and i in [0, 1, 2, 3]:
                skip_4 = skips.pop()
                skip_3 = skips.pop()
                skip_2 = skips.pop()
                skip_1 = skips.pop()
                skips_ = (skip_1, skip_2, skip_3, skip_4)
                x = up(x, skips_, t)
            else:
                for block in up:
                    if isinstance(block, ConvResblock):
                        x = torch.concat([x, skips.pop()], dim=1)
                    x = block(x, t)

        return self.output(x)
//  [python]
def forward(self, x):
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        if self.training and self.use_aux:
            aux = self.aux_logits(x)
        x = self.layer4(x)
        x = self.ppm(x)
        x = self.final(x)
        if self.training and self.use_aux:
            return F.upsample(x, self.input_size, mode='bilinear'), F.upsample(aux, self.input_size, mode='bilinear')
        return F.upsample(x, self.input_size, mode='bilinear')
//  [python]
def forward(self, x):
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        if self.training and self.use_aux:
            aux = self.aux_logits(x)
        x = self.layer4(x)
        x = self.ppm(x)
        x = self.final(x)
        if self.training and self.use_aux:
            return F.interpolate(x, size=self.input_size, mode='bilinear'), F.interpolate(aux, size=self.input_size, mode='bilinear')
        return F.interpolate(x, size=self.input_size, mode='bilinear')
//  [python]
def forward(self, x):
        x_size = x.size()
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        if self.training and self.use_aux:
            aux = self.aux_logits(x)
        x = self.layer4(x)
        x = self.ppm(x)
        x = self.final(x)
        if self.training and self.use_aux:
            return F.upsample(x, x_size[2:], mode='bilinear'), F.upsample(aux, x_size[2:], mode='bilinear')
        return F.upsample(x, x_size[2:], mode='bilinear')
//  [python]
def forward(self, x):
        x_size = x.size()
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        if self.training and self.use_aux:
            aux = self.aux_logits(x)
        x = self.layer4(x)
        x = self.ppm(x)
        x = self.final(x)
        if self.training and self.use_aux:
            return F.interpolate(x, x_size[2:], mode='bilinear'), F.interpolate(aux, x_size[2:], mode='bilinear')
        return F.interpolate(x, x_size[2:], mode='bilinear')
//  [python]
def forward(self, left, right):

        refimg_fea     = self.feature_extraction(left)
        targetimg_fea  = self.feature_extraction(right)
 
        #matching
        cost = Variable(torch.FloatTensor(refimg_fea.size()[0], refimg_fea.size()[1]*2, self.maxdisp/4,  refimg_fea.size()[2],  refimg_fea.size()[3]).zero_(), volatile= not self.training).cuda()

        for i in range(self.maxdisp/4):
            if i > 0 :
             cost[:, :refimg_fea.size()[1], i, :,i:]   = refimg_fea[:,:,:,i:]
             cost[:, refimg_fea.size()[1]:, i, :,i:] = targetimg_fea[:,:,:,:-i]
            else:
             cost[:, :refimg_fea.size()[1], i, :,:]   = refimg_fea
             cost[:, refimg_fea.size()[1]:, i, :,:]   = targetimg_fea
        cost = cost.contiguous()

        cost0 = self.dres0(cost)
        cost0 = self.dres1(cost0) + cost0
        cost0 = self.dres2(cost0) + cost0 
        cost0 = self.dres3(cost0) + cost0 
        cost0 = self.dres4(cost0) + cost0

        cost = self.classify(cost0)
        cost = F.upsample(cost, [self.maxdisp,left.size()[2],left.size()[3]], mode='trilinear')
        cost = torch.squeeze(cost,1)
        pred = F.softmax(cost)
        pred = disparityregression(self.maxdisp)(pred)

        return pred
//  [python]
def forward(self, left, right):

        refimg_fea     = self.feature_extraction(left)
        targetimg_fea  = self.feature_extraction(right)
 
        #matching
        cost = Variable(torch.FloatTensor(refimg_fea.size()[0], refimg_fea.size()[1]*2, self.maxdisp/4,  refimg_fea.size()[2],  refimg_fea.size()[3]).zero_(), volatile= not self.training).cuda()

        for i in range(self.maxdisp/4):
            if i > 0 :
             cost[:, :refimg_fea.size()[1], i, :,i:]   = refimg_fea[:,:,:,i:]
             cost[:, refimg_fea.size()[1]:, i, :,i:] = targetimg_fea[:,:,:,:-i]
            else:
             cost[:, :refimg_fea.size()[1], i, :,:]   = refimg_fea
             cost[:, refimg_fea.size()[1]:, i, :,:]   = targetimg_fea
        cost = cost.contiguous()

        cost0 = self.dres0(cost)
        cost0 = self.dres1(cost0) + cost0
        cost0 = self.dres2(cost0) + cost0 
        cost0 = self.dres3(cost0) + cost0 
        cost0 = self.dres4(cost0) + cost0

        cost = self.classify(cost0)
        cost = F.interpolate(cost, [self.maxdisp,left.size()[2],left.size()[3]], mode='trilinear')
        cost = torch.squeeze(cost,1)
        pred = F.softmax(cost)
        pred = disparityregression(self.maxdisp)(pred)

        return pred
//  [python]
def forward(self, x):
        fm0 = self.layer0(x)
        fm1 = self.layer1(fm0)
        fm2 = self.layer2(fm1)
        fm3 = self.layer3(fm2)
        fm4 = self.layer4(fm3)
        gcfm1 = self.brm1(self.gcm1(fm4))
        gcfm2 = self.brm2(self.gcm2(fm3))
        gcfm3 = self.brm3(self.gcm3(fm2))
        gcfm4 = self.brm4(self.gcm4(fm1))
        fs1 = self.brm5(F.upsample_bilinear(gcfm1, fm3.size()[2:]) + gcfm2)
        fs2 = self.brm6(F.upsample_bilinear(fs1, fm2.size()[2:]) + gcfm3)
        fs3 = self.brm7(F.upsample_bilinear(fs2, fm1.size()[2:]) + gcfm4)
        fs4 = self.brm8(F.upsample_bilinear(fs3, fm0.size()[2:]))
        out = self.brm9(F.upsample_bilinear(fs4, self.input_size))
        return out
//  [python]
def forward(self, x):
        fm0 = self.layer0(x)
        fm1 = self.layer1(fm0)
        fm2 = self.layer2(fm1)
        fm3 = self.layer3(fm2)
        fm4 = self.layer4(fm3)
        gcfm1 = self.brm1(self.gcm1(fm4))
        gcfm2 = self.brm2(self.gcm2(fm3))
        gcfm3 = self.brm3(self.gcm3(fm2))
        gcfm4 = self.brm4(self.gcm4(fm1))
        fs1 = self.brm5(F.interpolate(gcfm1, size=fm3.size()[2:], mode='bilinear') + gcfm2)
        fs2 = self.brm6(F.interpolate(fs1, size=fm2.size()[2:], mode='bilinear') + gcfm3)
        fs3 = self.brm7(F.interpolate(fs2, size=fm1.size()[2:], mode='bilinear') + gcfm4)
        fs4 = self.brm8(F.interpolate(fs3, size=fm0.size()[2:], mode='bilinear'))
        out = self.brm9(F.interpolate(fs4, size=self.input_size, mode='bilinear'))
        return out
//  [python]
def __call__(
        self,
        features: torch.Tensor,
        schedule=[1.0, 0.5],
        generator=None,
    ):
        features = self.ldm_transform_latent(features)
        ts = self.round_timesteps(
            torch.arange(0, 1024),
            1024,
            self.n_distilled_steps,
            truncate_start=False,
        )
        shape = (
            features.size(0),
            3,
            8 * features.size(2),
            8 * features.size(3),
        )
        x_start = torch.zeros(shape, device=features.device, dtype=features.dtype)
        schedule_timesteps = [int((1024 - 1) * s) for s in schedule]
        for i in schedule_timesteps:
            t = ts[i].item()
            t_ = torch.tensor([t] * features.shape[0]).to(self.device)
            # noise = torch.randn_like(x_start)
            noise = torch.randn(x_start.shape, dtype=x_start.dtype, generator=generator).to(device=x_start.device)
            x_start = (
                _extract_into_tensor(self.sqrt_alphas_cumprod, t_, x_start.shape) * x_start
                + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t_, x_start.shape) * noise
            )
            c_in = _extract_into_tensor(self.c_in, t_, x_start.shape)

            import torch.nn.functional as F

            from diffusers import UNet2DModel

            if isinstance(self.ckpt, UNet2DModel):
                input = torch.concat([c_in * x_start, F.upsample_nearest(features, scale_factor=8)], dim=1)
                model_output = self.ckpt(input, t_).sample
            else:
                model_output = self.ckpt(c_in * x_start, t_, features=features)

            B, C = x_start.shape[:2]
            model_output, _ = torch.split(model_output, C, dim=1)
            pred_xstart = (
                _extract_into_tensor(self.c_out, t_, x_start.shape) * model_output
                + _extract_into_tensor(self.c_skip, t_, x_start.shape) * x_start
            ).clamp(-1, 1)
            x_start = pred_xstart
        return x_start
//  [python]
def __call__(
        self,
        features: torch.Tensor,
        schedule=[1.0, 0.5],
        generator=None,
    ):
        features = self.ldm_transform_latent(features)
        ts = self.round_timesteps(
            torch.arange(0, 1024),
            1024,
            self.n_distilled_steps,
            truncate_start=False,
        )
        shape = (
            features.size(0),
            3,
            8 * features.size(2),
            8 * features.size(3),
        )
        x_start = torch.zeros(shape, device=features.device, dtype=features.dtype)
        schedule_timesteps = [int((1024 - 1) * s) for s in schedule]
        for i in schedule_timesteps:
            t = ts[i].item()
            t_ = torch.tensor([t] * features.shape[0]).to(self.device)
            # noise = torch.randn_like(x_start)
            noise = torch.randn(x_start.shape, dtype=x_start.dtype, generator=generator).to(device=x_start.device)
            x_start = (
                _extract_into_tensor(self.sqrt_alphas_cumprod, t_, x_start.shape) * x_start
                + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t_, x_start.shape) * noise
            )
            c_in = _extract_into_tensor(self.c_in, t_, x_start.shape)

            import torch.nn.functional as F

            from diffusers import UNet2DModel

            if isinstance(self.ckpt, UNet2DModel):
                input = torch.concat([c_in * x_start, F.interpolate(features, scale_factor=8, mode='nearest')], dim=1)
                model_output = self.ckpt(input, t_).sample
            else:
                model_output = self.ckpt(c_in * x_start, t_, features=features)

            B, C = x_start.shape[:2]
            model_output, _ = torch.split(model_output, C, dim=1)
            pred_xstart = (
                _extract_into_tensor(self.c_out, t_, x_start.shape) * model_output
                + _extract_into_tensor(self.c_skip, t_, x_start.shape) * x_start
            ).clamp(-1, 1)
            x_start = pred_xstart
        return x_start
//  [python]
def forward(self, inputs):
        assert len(inputs) == len(self.in_channels)

        # build laterals
        laterals = [
            lateral_conv(inputs[i + self.start_level])
            for i, lateral_conv in enumerate(self.lateral_convs)
        ]

        # build top-down path
        used_backbone_levels = len(laterals)
        # import pdb;pdb.set_trace()
        for i in range(used_backbone_levels - 1, 0, -1):
            laterals[i - 1] += F.upsample_nearest(
                laterals[i], scale_factor=2)
            # laterals[i - 1] += interpolate(
            #     laterals[i], scale_factor=2, mode='nearest')

        # build outputs
        # part 1: from original levels
        outs = [
            self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels)
        ]
        # part 2: add extra levels
        if self.num_outs > len(outs):
            # use max pool to get more levels on top of outputs
            # (e.g., Faster R-CNN, Mask R-CNN)
            if not self.add_extra_convs:
                for i in range(self.num_outs - used_backbone_levels):
                    outs.append(F.max_pool2d(outs[-1], 1, stride=2))
            # add conv layers on top of original feature maps (RetinaNet)
            else:
                orig = inputs[self.backbone_end_level - 1]
                outs.append(self.fpn_convs[used_backbone_levels](orig))
                for i in range(used_backbone_levels + 1, self.num_outs):
                    # BUG: we should add relu before each extra conv
                    outs.append(self.fpn_convs[i](outs[-1]))
        return tuple(outs)
//  [python]
def forward(self, inputs):
        assert len(inputs) == len(self.in_channels)

        # build laterals
        laterals = [
            lateral_conv(inputs[i + self.start_level])
            for i, lateral_conv in enumerate(self.lateral_convs)
        ]

        # build top-down path
        used_backbone_levels = len(laterals)
        # import pdb;pdb.set_trace()
        for i in range(used_backbone_levels - 1, 0, -1):
            laterals[i - 1] += F.interpolate(
                laterals[i], scale_factor=2, mode='nearest')

        # build outputs
        # part 1: from original levels
        outs = [
            self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels)
        ]
        # part 2: add extra levels
        if self.num_outs > len(outs):
            # use max pool to get more levels on top of outputs
            # (e.g., Faster R-CNN, Mask R-CNN)
            if not self.add_extra_convs:
                for i in range(self.num_outs - used_backbone_levels):
                    outs.append(F.max_pool2d(outs[-1], 1, stride=2))
            # add conv layers on top of original feature maps (RetinaNet)
            else:
                orig = inputs[self.backbone_end_level - 1]
                outs.append(self.fpn_convs[used_backbone_levels](orig))
                for i in range(used_backbone_levels + 1, self.num_outs):
                    # BUG: we should add relu before each extra conv
                    outs.append(self.fpn_convs[i](outs[-1]))
        return tuple(outs)
//  [python]
def forward(self, x, size):
        x = F.upsample(x, size=size, mode='bilinear')
        x_conv1 = self.relu(self.bn1(self.conv1(x)))
        bran1 = self.bn1_2(self.conv1_2(x_conv1))
        bran2 = self.bn2(self.conv2(x))

        out = self.relu(bran1 + bran2)

        return out
//  [python]
def forward(self, x, size):
        x = F.interpolate(x, size=size, mode='bilinear')
        x_conv1 = self.relu(self.bn1(self.conv1(x)))
        bran1 = self.bn1_2(self.conv1_2(x_conv1))
        bran2 = self.bn2(self.conv2(x))

        out = self.relu(bran1 + bran2)

        return out
//  [python]
def forward(self, x):
        enc1 = self.enc1(x)
        enc2 = self.enc2(enc1)
        enc3 = self.enc3(enc2)
        enc4 = self.enc4(enc3)
        center = self.center(enc4)
        dec4 = self.dec4(torch.cat([center, F.upsample(enc4, center.size()[2:], mode='bilinear')], 1))
        dec3 = self.dec3(torch.cat([dec4, F.upsample(enc3, dec4.size()[2:], mode='bilinear')], 1))
        dec2 = self.dec2(torch.cat([dec3, F.upsample(enc2, dec3.size()[2:], mode='bilinear')], 1))
        dec1 = self.dec1(torch.cat([dec2, F.upsample(enc1, dec2.size()[2:], mode='bilinear')], 1))
        final = self.final(dec1)
        return F.upsample(final, x.size()[2:], mode='bilinear')
//  [python]
def forward(self, x):
        enc1 = self.enc1(x)
        enc2 = self.enc2(enc1)
        enc3 = self.enc3(enc2)
        enc4 = self.enc4(enc3)
        center = self.center(enc4)
        dec4 = self.dec4(torch.cat([center, F.interpolate(enc4, center.size()[2:], mode='bilinear')], 1))
        dec3 = self.dec3(torch.cat([dec4, F.interpolate(enc3, dec4.size()[2:], mode='bilinear')], 1))
        dec2 = self.dec2(torch.cat([dec3, F.interpolate(enc2, dec3.size()[2:], mode='bilinear')], 1))
        dec1 = self.dec1(torch.cat([dec2, F.interpolate(enc1, dec2.size()[2:], mode='bilinear')], 1))
        final = self.final(dec1)
        return F.interpolate(final, x.size()[2:], mode='bilinear')
//  [python]
def forward(self, x):
        imsize = x.size()[2:]
        features = self.base_forward(x)

        x = list(self.head(features))

        x[0] = F.upsample(x[0], imsize, **self._up_kwargs)
        if self.aux:
            auxout = self.auxlayer(features[2])
            auxout = F.upsample(auxout, imsize, **self._up_kwargs)
            x.append(auxout)
        return tuple(x)
//  [python]
def forward(self, x):
        imsize = x.size()[2:]
        features = self.base_forward(x)

        x = list(self.head(features))

        x[0] = F.interpolate(x[0], imsize, **self._up_kwargs)
        if self.aux:
            auxout = self.auxlayer(features[2])
            auxout = F.interpolate(auxout, imsize, **self._up_kwargs)
            x.append(auxout)
        return tuple(x)
//  [python]
def forward(self, x):    
        '''
            Attention, input size should be the 32x. 
        '''
        dec1 = self.dec1(x)
        dec2 = self.dec2(dec1)
        dec3 = self.dec3(dec2)
        dec4 = self.dec4(dec3)
        dec5 = self.dec5(dec4)
        enc5 = self.enc5(dec5)  
   
        enc4 = self.enc4(torch.cat([dec4, enc5], 1))
        enc3 = self.enc3(torch.cat([dec3, enc4], 1))
        enc2 = self.enc2(torch.cat([dec2, enc3], 1))
        enc1 = self.enc1(torch.cat([dec1, enc2], 1))

        return F.upsample_bilinear(self.final(enc1), x.size()[2:])
//  [python]
def forward(self, x):    
        '''
            Attention, input size should be the 32x. 
        '''
        dec1 = self.dec1(x)
        dec2 = self.dec2(dec1)
        dec3 = self.dec3(dec2)
        dec4 = self.dec4(dec3)
        dec5 = self.dec5(dec4)
        enc5 = self.enc5(dec5)  
   
        enc4 = self.enc4(torch.cat([dec4, enc5], 1))
        enc3 = self.enc3(torch.cat([dec3, enc4], 1))
        enc2 = self.enc2(torch.cat([dec2, enc3], 1))
        enc1 = self.enc1(torch.cat([dec1, enc2], 1))

        return F.interpolate(self.final(enc1), size=x.size()[2:], mode='bilinear')
//  [python]
def compute_clip_loss(img, text):
    # img = clip_transform(img)
    img = torch.nn.functional.upsample_bilinear(img, (224, 224))
    tokenized_text = clip.tokenize([text]).to(device)

    img_logits, _text_logits = clip_model(img, tokenized_text)

    return 1/img_logits * 100
//  [python]
def compute_clip_loss(img, text):
    # img = clip_transform(img)
    img = torch.nn.functional.interpolate(img, (224, 224), mode='bilinear')
    tokenized_text = clip.tokenize([text]).to(device)

    img_logits, _text_logits = clip_model(img, tokenized_text)

    return 1/img_logits * 100
//  [python]
def forward(self, x):
        conv1 = self.conv_block1(x)
        conv2 = self.conv_block2(conv1)
        conv3 = self.conv_block3(conv2)
        conv4 = self.conv_block4(conv3)
        conv5 = self.conv_block5(conv4)

        score = self.classifier(conv5)

        if self.learned_billinear:
            upscore2 = self.upscore2(score)
            score_pool4c = self.score_pool4(conv4)[:, :, 5:5+upscore2.size()[2],
                                                         5:5+upscore2.size()[3]]
            upscore_pool4 = self.upscore4(upscore2 + score_pool4c)
            
            score_pool3c = self.score_pool3(conv3)[:, :, 9:9+upscore_pool4.size()[2],
                                                         9:9+upscore_pool4.size()[3]]

            out = self.upscore8(score_pool3c + upscore_pool4)[:, :, 31:31+x.size()[2],
                                                                    31:31+x.size()[3]]
            return out.contiguous()
                                                         

        else:
            score_pool4 = self.score_pool4(conv4)
            score_pool3 = self.score_pool3(conv3)
            score = F.upsample_bilinear(score, score_pool4.size()[2:])
            score += score_pool4
            score = F.upsample_bilinear(score, score_pool3.size()[2:])
            score += score_pool3
            out = F.upsample_bilinear(score, x.size()[2:])

        return out
//  [python]
def forward(self, x):
        conv1 = self.conv_block1(x)
        conv2 = self.conv_block2(conv1)
        conv3 = self.conv_block3(conv2)
        conv4 = self.conv_block4(conv3)
        conv5 = self.conv_block5(conv4)

        score = self.classifier(conv5)

        if self.learned_billinear:
            upscore2 = self.upscore2(score)
            score_pool4c = self.score_pool4(conv4)[:, :, 5:5+upscore2.size()[2],
                                                         5:5+upscore2.size()[3]]
            upscore_pool4 = self.upscore4(upscore2 + score_pool4c)
            
            score_pool3c = self.score_pool3(conv3)[:, :, 9:9+upscore_pool4.size()[2],
                                                         9:9+upscore_pool4.size()[3]]

            out = self.upscore8(score_pool3c + upscore_pool4)[:, :, 31:31+x.size()[2],
                                                                    31:31+x.size()[3]]
            return out.contiguous()
                                                         

        else:
            score_pool4 = self.score_pool4(conv4)
            score_pool3 = self.score_pool3(conv3)
            score = F.interpolate(score, size=score_pool4.size()[2:], mode='bilinear')
            score += score_pool4
            score = F.interpolate(score, size=score_pool3.size()[2:], mode='bilinear')
            score += score_pool3
            out = F.interpolate(score, size=x.size()[2:], mode='bilinear')

        return out
//  [python]
def compute_perceptual_loss(gen_img, ref_img):
    gen_img = torch.nn.functional.upsample_bilinear(img, (224, 224))
    loss = 0
    len_vgg_layer_mappings = int(max(vgg_layer_name_mapping.keys()))

    ref_feats = ref_img
    gen_feats = gen_img

    for idx, (name, module) in enumerate(vgg_layers._modules.items()):
        ref_feats = module(ref_feats)
        gen_feats = module(gen_feats)
        if name in vgg_layer_name_mapping.keys():
            loss += torch.nn.functional.mse_loss(ref_feats, gen_feats)
        
        if idx >= len_vgg_layer_mappings:
            break
    
    return loss/len_vgg_layer_mappings
//  [python]
def compute_perceptual_loss(gen_img, ref_img):
    gen_img = torch.nn.functional.interpolate(img, (224, 224), mode='bilinear')
    loss = 0
    len_vgg_layer_mappings = int(max(vgg_layer_name_mapping.keys()))

    ref_feats = ref_img
    gen_feats = gen_img

    for idx, (name, module) in enumerate(vgg_layers._modules.items()):
        ref_feats = module(ref_feats)
        gen_feats = module(gen_feats)
        if name in vgg_layer_name_mapping.keys():
            loss += torch.nn.functional.mse_loss(ref_feats, gen_feats)
        
        if idx >= len_vgg_layer_mappings:
            break
    
    return loss/len_vgg_layer_mappings
//  [python]
def forward(self, x):
        x = self.linear_fuse(x[1:, ...])
        l, bt, c = x.size()
        b, t = bt // (self.num_frames // self.alpha), self.num_frames // self.alpha
        x = x.view(l, b, t, c).permute(1, 3, 2, 0).reshape(b, c, t, int(math.sqrt(l)), int(math.sqrt(l)))
        # return x.repeat_interleave(self.alpha // self.temporal_alpha, dim=2)
        return torch.nn.functional.upsample_nearest(x, size=(x.size(2)*(self.alpha), x.size(3), x.size(4)))
//  [python]
def forward(self, x):
        x = self.linear_fuse(x[1:, ...])
        l, bt, c = x.size()
        b, t = bt // (self.num_frames // self.alpha), self.num_frames // self.alpha
        x = x.view(l, b, t, c).permute(1, 3, 2, 0).reshape(b, c, t, int(math.sqrt(l)), int(math.sqrt(l)))
        # return x.repeat_interleave(self.alpha // self.temporal_alpha, dim=2)
        return torch.nn.functional.interpolate(x, size=(x.size(2)*(self.alpha), x.size(3), x.size(4)), mode='nearest')
//  [python]
def forward(self, features, attentions, counterfactual=False):
        B, C, H, W = features.size()
        _, M, AH, AW = attentions.size()

        # match size
        if AH != H or AW != W:
            attentions = F.upsample_bilinear(attentions, size=(H, W))

        # feature_matrix: (B, M, C) -> (B, M * C)
        if self.pool is None:
            feature_matrix = (torch.einsum('imjk,injk->imn', (attentions, features)) / float(H * W)).view(B, -1)
        else:
            feature_matrix = []
            for i in range(M):
                AiF = self.pool(features * attentions[:, i:i + 1, ...]).view(B, -1)
                feature_matrix.append(AiF)
            feature_matrix = torch.cat(feature_matrix, dim=1)

        # sign-sqrt
        feature_matrix_raw = torch.sign(feature_matrix) * torch.sqrt(torch.abs(feature_matrix) + EPSILON)

        # l2 normalization along dimension M and C
        feature_matrix = F.normalize(feature_matrix_raw, dim=-1)

        if counterfactual:
            if self.training:
                fake_att = torch.zeros_like(attentions).uniform_(0, 2)
            else:
                fake_att = torch.ones_like(attentions)
            # mean_feature = features.mean(3).mean(2).view(B, 1, C)
            # counterfactual_feature = mean_feature.expand(B, M, C).contiguous().view(B, -1)
            counterfactual_feature = (torch.einsum('imjk,injk->imn', (fake_att, features)) / float(H * W)).view(B, -1)

            counterfactual_feature = torch.sign(counterfactual_feature) * torch.sqrt(
                torch.abs(counterfactual_feature) + EPSILON)

            counterfactual_feature = F.normalize(counterfactual_feature, dim=-1)
            return feature_matrix, counterfactual_feature
        else:
            return feature_matrix
//  [python]
def forward(self, features, attentions, counterfactual=False):
        B, C, H, W = features.size()
        _, M, AH, AW = attentions.size()

        # match size
        if AH != H or AW != W:
            attentions = F.interpolate(attentions, size=(H, W), mode='bilinear')

        # feature_matrix: (B, M, C) -> (B, M * C)
        if self.pool is None:
            feature_matrix = (torch.einsum('imjk,injk->imn', (attentions, features)) / float(H * W)).view(B, -1)
        else:
            feature_matrix = []
            for i in range(M):
                AiF = self.pool(features * attentions[:, i:i + 1, ...]).view(B, -1)
                feature_matrix.append(AiF)
            feature_matrix = torch.cat(feature_matrix, dim=1)

        # sign-sqrt
        feature_matrix_raw = torch.sign(feature_matrix) * torch.sqrt(torch.abs(feature_matrix) + EPSILON)

        # l2 normalization along dimension M and C
        feature_matrix = F.normalize(feature_matrix_raw, dim=-1)

        if counterfactual:
            if self.training:
                fake_att = torch.zeros_like(attentions).uniform_(0, 2)
            else:
                fake_att = torch.ones_like(attentions)
            # mean_feature = features.mean(3).mean(2).view(B, 1, C)
            # counterfactual_feature = mean_feature.expand(B, M, C).contiguous().view(B, -1)
            counterfactual_feature = (torch.einsum('imjk,injk->imn', (fake_att, features)) / float(H * W)).view(B, -1)

            counterfactual_feature = torch.sign(counterfactual_feature) * torch.sqrt(
                torch.abs(counterfactual_feature) + EPSILON)

            counterfactual_feature = F.normalize(counterfactual_feature, dim=-1)
            return feature_matrix, counterfactual_feature
        else:
            return feature_matrix
//  [python]
def forward(self, x):
        conv1 = self.conv_block1(x)
        conv2 = self.conv_block2(conv1)
        conv3 = self.conv_block3(conv2)
        conv4 = self.conv_block4(conv3)
        conv5 = self.conv_block5(conv4)

        embed = self.embedding_layer(conv5)

        if self.learned_billinear:
            upembed2 = self.upembed2(embed)
            embed_pool4c = self.embed_pool4(conv4)[:, :, 5:5+upembed2.size()[2],
                                                         5:5+upembed2.size()[3]]
            upembed_pool4 = self.upembed4(upembed2 + embed_pool4c)
            
            embed_pool3c = self.embed_pool3(conv3)[:, :, 9:9+upembed_pool4.size()[2],
                                                         9:9+upembed_pool4.size()[3]]

            out = self.upembed8(embed_pool3c + upembed_pool4)[:, :, 31:31+x.size()[2],
                                                                    31:31+x.size()[3]]
            return out.contiguous()                  

        else:
            embed_pool4 = self.embed_pool4(conv4)
            embed_pool3 = self.embed_pool3(conv3)
            embed = F.upsample_bilinear(embed, embed_pool4.size()[2:])
            embed += embed_pool4
            embed = F.upsample_bilinear(embed, embed_pool3.size()[2:])
            embed += embed_pool3
            out = F.upsample_bilinear(embed, x.size()[2:])

        return out
//  [python]
def forward(self, x):
        conv1 = self.conv_block1(x)
        conv2 = self.conv_block2(conv1)
        conv3 = self.conv_block3(conv2)
        conv4 = self.conv_block4(conv3)
        conv5 = self.conv_block5(conv4)

        embed = self.embedding_layer(conv5)

        if self.learned_billinear:
            upembed2 = self.upembed2(embed)
            embed_pool4c = self.embed_pool4(conv4)[:, :, 5:5+upembed2.size()[2],
                                                         5:5+upembed2.size()[3]]
            upembed_pool4 = self.upembed4(upembed2 + embed_pool4c)
            
            embed_pool3c = self.embed_pool3(conv3)[:, :, 9:9+upembed_pool4.size()[2],
                                                         9:9+upembed_pool4.size()[3]]

            out = self.upembed8(embed_pool3c + upembed_pool4)[:, :, 31:31+x.size()[2],
                                                                    31:31+x.size()[3]]
            return out.contiguous()                  

        else:
            embed_pool4 = self.embed_pool4(conv4)
            embed_pool3 = self.embed_pool3(conv3)
            embed = F.interpolate(embed, size=embed_pool4.size()[2:], mode='bilinear')
            embed += embed_pool4
            embed = F.interpolate(embed, size=embed_pool3.size()[2:], mode='bilinear')
            embed += embed_pool3
            out = F.interpolate(embed, size=x.size()[2:], mode='bilinear')

        return out
//  [python]
def forward(self, x):
        """Applies network layers and ops on input image(s) x.

        Args:
            x: input image or batch of images. Shape: [batch,3*batch,300,300].

        Return:
            Depending on phase:
            test:
                Variable(tensor) of output class label predictions,
                confidence score, and corresponding location predictions for
                each object detected. Shape: [batch,topk,7]

            train:
                list of concat outputs from:
                    1: confidence layers, Shape: [batch*num_priors,num_classes]
                    2: localization layers, Shape: [batch,num_priors*4]
                    3: priorbox layers, Shape: [2,num_priors*4]
        """
        sources = list()
        loc = list()
        conf = list()

        # apply vgg up to conv4_3 relu
        for k in range(23):
            x = self.base[k](x)

        s1 = self.reduce(x)

        # apply vgg up to fc7
        for k in range(23, len(self.base)):
            x = self.base[k](x)
        s2 = self.up_reduce(x)
        s2 = F.upsample(s2, scale_factor=2, mode='bilinear', align_corners=True)
        s = torch.cat((s1,s2),1)

        ss = self.Norm(s)
        sources.append(ss)

        # apply extra layers and cache source layer outputs
        for k, v in enumerate(self.extras):
            x = v(x)
            if k < self.indicator or k%2 ==0:
                sources.append(x)

        # apply multibox head to source layers
        for (x, l, c) in zip(sources, self.loc, self.conf):
            loc.append(l(x).permute(0, 2, 3, 1).contiguous())
            conf.append(c(x).permute(0, 2, 3, 1).contiguous())

        #print([o.size() for o in loc])


        loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
        conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)

        if self.phase == "test":
            output = (
                loc.view(loc.size(0), -1, 4),                   # loc preds
                self.softmax(conf.view(-1, self.num_classes)),  # conf preds
            )
        else:
            output = (
                loc.view(loc.size(0), -1, 4),
                conf.view(conf.size(0), -1, self.num_classes),
            )
        return output
//  [python]
def forward(self, x):
        """Applies network layers and ops on input image(s) x.

        Args:
            x: input image or batch of images. Shape: [batch,3*batch,300,300].

        Return:
            Depending on phase:
            test:
                Variable(tensor) of output class label predictions,
                confidence score, and corresponding location predictions for
                each object detected. Shape: [batch,topk,7]

            train:
                list of concat outputs from:
                    1: confidence layers, Shape: [batch*num_priors,num_classes]
                    2: localization layers, Shape: [batch,num_priors*4]
                    3: priorbox layers, Shape: [2,num_priors*4]
        """
        sources = list()
        loc = list()
        conf = list()

        # apply vgg up to conv4_3 relu
        for k in range(23):
            x = self.base[k](x)

        s1 = self.reduce(x)

        # apply vgg up to fc7
        for k in range(23, len(self.base)):
            x = self.base[k](x)
        s2 = self.up_reduce(x)
        s2 = F.interpolate(s2, scale_factor=2, mode='bilinear', align_corners=True)
        s = torch.cat((s1,s2),1)

        ss = self.Norm(s)
        sources.append(ss)

        # apply extra layers and cache source layer outputs
        for k, v in enumerate(self.extras):
            x = v(x)
            if k < self.indicator or k%2 ==0:
                sources.append(x)

        # apply multibox head to source layers
        for (x, l, c) in zip(sources, self.loc, self.conf):
            loc.append(l(x).permute(0, 2, 3, 1).contiguous())
            conf.append(c(x).permute(0, 2, 3, 1).contiguous())

        #print([o.size() for o in loc])


        loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
        conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)

        if self.phase == "test":
            output = (
                loc.view(loc.size(0), -1, 4),                   # loc preds
                self.softmax(conf.view(-1, self.num_classes)),  # conf preds
            )
        else:
            output = (
                loc.view(loc.size(0), -1, 4),
                conf.view(conf.size(0), -1, self.num_classes),
            )
        return output
//  [python]
def forward(self, x):
        x = self.features(x)
        x = F.upsample_bilinear(x, scale_factor=2)
        x = self.reg_layer(x)
        return torch.nn.functional.relu(x)
//  [python]
def forward(self, x):
        x = self.features(x)
        x = F.interpolate(x, scale_factor=2, mode='bilinear')
        x = self.reg_layer(x)
        return torch.nn.functional.relu(x)
//  [python]
def _upsample_add(self, x, y):
        '''Upsample and add two feature maps.
        Args:
          x: (Variable) top feature map to be upsampled.
          y: (Variable) lateral feature map.
        Returns:
          (Variable) added feature map.
        Note in PyTorch, when input size is odd, the upsampled feature map
        with `F.upsample(..., scale_factor=2, mode='nearest')`
        maybe not equal to the lateral feature map size.
        e.g.
        original input size: [N,_,15,15] ->
        conv2d feature map size: [N,_,8,8] ->
        upsampled feature map size: [N,_,16,16]
        So we choose bilinear upsample which supports arbitrary output sizes.
        '''
        _,_,H,W = y.size()
        return F.upsample(x, size=(H,W), mode='bilinear') + y
//  [python]
def _upsample_add(self, x, y):
        _,_,H,W = y.size()
        return F.interpolate(x, size=(H,W), mode='bilinear') + y
//  [python]
def forward(self, x):
        """Forward pass of the network.
        """
        N = x.size()[0]
        x1 = self.backbone.features(x)

        H = x1.size()[2]
        W = x1.size()[3]
        assert x1.size()[1] == 512

        x2 = self.sfeatures(x)
        H2 = x2.size()[2]
        W2 = x2.size()[3]
        assert x2.size()[1] == 128

        if (H != H2) | (W != W2):
            x2 = F.upsample_bilinear(x2, (H, W))

        x1 = x1.view(N, 512, H * W)
        x2 = x2.view(N, 128, H * W)
        x = torch.bmm(x1, torch.transpose(x2, 1, 2)) / (H * W)  # Bilinear
        assert x.size() == (N, 512, 128)
        x = x.view(N, 512 * 128)
        #x = torch.sqrt(x + 1e-8)
        x = F.normalize(x)

        output = []

        if not self.config.JL:
            for idx, fc in enumerate(self.fc):
                if not self.config.train:
                    for W in fc.parameters():
                        fc.weight.data = F.normalize(W, p=2, dim=1)
                output.append(fc(x))
        else:
            for idx, fc in enumerate(self.fc):
                output.append(fc(x))

        sfeat = 0
        return output, sfeat