Spaces:
Runtime error
Runtime error
File size: 3,017 Bytes
30a0ec5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
"""
****************** COPYRIGHT AND CONFIDENTIALITY INFORMATION ******************
Copyright (c) 2018 [Thomson Licensing]
All Rights Reserved
This program contains proprietary information which is a trade secret/business \
secret of [Thomson Licensing] and is protected, even if unpublished, under \
applicable Copyright laws (including French droit d'auteur) and/or may be \
subject to one or more patent(s).
Recipient is to retain this program in confidence and is not permitted to use \
or make copies thereof other than as permitted in a written agreement with \
[Thomson Licensing] unless otherwise expressly allowed by applicable laws or \
by [Thomson Licensing] under express agreement.
Thomson Licensing is a company of the group TECHNICOLOR
*******************************************************************************
This scripts permits one to reproduce training and experiments of:
Engilberge, M., Chevallier, L., Pérez, P., & Cord, M. (2018, April).
Finding beans in burgers: Deep semantic-visual embedding with localization.
In Proceedings of CVPR (pp. 3984-3993)
Author: Martin Engilberge
"""
import torch.nn as nn
import torch
class ContrastiveLoss(nn.Module):
def __init__(self, margin=0.2):
super(ContrastiveLoss, self).__init__()
self.margin = margin
def forward(self, imgs, caps):
scores = torch.mm(imgs, caps.t())
diag = scores.diag()
cost_s = torch.clamp((self.margin - diag).expand_as(scores) + scores, min=0)
# compare every diagonal score to scores in its row (i.e, all
# contrastive sentences for each image)
cost_im = torch.clamp((self.margin - diag.view(-1, 1)).expand_as(scores) + scores, min=0)
# clear diagonals
diag_s = torch.diag(cost_s.diag())
diag_im = torch.diag(cost_im.diag())
cost_s = cost_s - diag_s
cost_im = cost_im - diag_im
return cost_s.sum() + cost_im.sum()
class HardNegativeContrastiveLoss(nn.Module):
def __init__(self, nmax=1, margin=0.2):
super(HardNegativeContrastiveLoss, self).__init__()
self.margin = margin
self.nmax = nmax
def forward(self, imgs, caps):
scores = torch.mm(imgs, caps.t())
diag = scores.diag()
# Reducing the score on diagonal so there are not selected as hard negative
scores = (scores - 2 * torch.diag(scores.diag()))
sorted_cap, _ = torch.sort(scores, 0, descending=True)
sorted_img, _ = torch.sort(scores, 1, descending=True)
# Selecting the nmax hardest negative examples
max_c = sorted_cap[:self.nmax, :]
max_i = sorted_img[:, :self.nmax]
# Margin based loss with hard negative instead of random negative
neg_cap = torch.sum(torch.clamp(max_c + (self.margin - diag).view(1, -1).expand_as(max_c), min=0))
neg_img = torch.sum(torch.clamp(max_i + (self.margin - diag).view(-1, 1).expand_as(max_i), min=0))
loss = neg_cap + neg_img
return loss
|