# -*- coding: utf-8 -*- import random import torch import torch.nn as nn from models.base import CaptionModel from utils.model_util import repeat_tensor import models.transformer_decoder class TransformerModel(CaptionModel): def __init__(self, encoder: nn.Module, decoder: nn.Module, **kwargs): if not hasattr(self, "compatible_decoders"): self.compatible_decoders = ( models.transformer_decoder.TransformerDecoder, ) super().__init__(encoder, decoder, **kwargs) def seq_forward(self, input_dict): cap = input_dict["cap"] cap_padding_mask = (cap == self.pad_idx).to(cap.device) cap_padding_mask = cap_padding_mask[:, :-1] output = self.decoder( { "word": cap[:, :-1], "attn_emb": input_dict["attn_emb"], "attn_emb_len": input_dict["attn_emb_len"], "cap_padding_mask": cap_padding_mask } ) return output def prepare_decoder_input(self, input_dict, output): decoder_input = { "attn_emb": input_dict["attn_emb"], "attn_emb_len": input_dict["attn_emb_len"] } t = input_dict["t"] ############### # determine input word ################ if input_dict["mode"] == "train" and random.random() < input_dict["ss_ratio"]: # training, scheduled sampling word = input_dict["cap"][:, :t+1] else: start_word = torch.tensor([self.start_idx,] * input_dict["attn_emb"].size(0)).unsqueeze(1).long() if t == 0: word = start_word else: word = torch.cat((start_word, output["seq"][:, :t]), dim=-1) # word: [N, T] decoder_input["word"] = word cap_padding_mask = (word == self.pad_idx).to(input_dict["attn_emb"].device) decoder_input["cap_padding_mask"] = cap_padding_mask return decoder_input def prepare_beamsearch_decoder_input(self, input_dict, output_i): decoder_input = {} t = input_dict["t"] i = input_dict["sample_idx"] beam_size = input_dict["beam_size"] ############### # prepare attn embeds ################ if t == 0: attn_emb = repeat_tensor(input_dict["attn_emb"][i], beam_size) attn_emb_len = repeat_tensor(input_dict["attn_emb_len"][i], beam_size) output_i["attn_emb"] = attn_emb output_i["attn_emb_len"] = attn_emb_len decoder_input["attn_emb"] = output_i["attn_emb"] decoder_input["attn_emb_len"] = output_i["attn_emb_len"] ############### # determine input word ################ start_word = torch.tensor([self.start_idx,] * beam_size).unsqueeze(1).long() if t == 0: word = start_word else: word = torch.cat((start_word, output_i["seq"]), dim=-1) decoder_input["word"] = word cap_padding_mask = (word == self.pad_idx).to(input_dict["attn_emb"].device) decoder_input["cap_padding_mask"] = cap_padding_mask return decoder_input class M2TransformerModel(CaptionModel): def __init__(self, encoder: nn.Module, decoder: nn.Module, **kwargs): if not hasattr(self, "compatible_decoders"): self.compatible_decoders = ( models.transformer_decoder.M2TransformerDecoder, ) super().__init__(encoder, decoder, **kwargs) self.check_encoder_compatibility() def check_encoder_compatibility(self): assert isinstance(self.encoder, models.encoder.M2TransformerEncoder), \ f"only M2TransformerModel is compatible with {self.__class__.__name__}" def seq_forward(self, input_dict): cap = input_dict["cap"] output = self.decoder( { "word": cap[:, :-1], "attn_emb": input_dict["attn_emb"], "attn_emb_mask": input_dict["attn_emb_mask"], } ) return output def prepare_decoder_input(self, input_dict, output): decoder_input = { "attn_emb": input_dict["attn_emb"], "attn_emb_mask": input_dict["attn_emb_mask"] } t = input_dict["t"] ############### # determine input word ################ if input_dict["mode"] == "train" and random.random() < input_dict["ss_ratio"]: # training, scheduled sampling word = input_dict["cap"][:, :t+1] else: start_word = torch.tensor([self.start_idx,] * input_dict["attn_emb"].size(0)).unsqueeze(1).long() if t == 0: word = start_word else: word = torch.cat((start_word, output["seq"][:, :t]), dim=-1) # word: [N, T] decoder_input["word"] = word return decoder_input def prepare_beamsearch_decoder_input(self, input_dict, output_i): decoder_input = {} t = input_dict["t"] i = input_dict["sample_idx"] beam_size = input_dict["beam_size"] ############### # prepare attn embeds ################ if t == 0: attn_emb = repeat_tensor(input_dict["attn_emb"][i], beam_size) attn_emb_mask = repeat_tensor(input_dict["attn_emb_mask"][i], beam_size) output_i["attn_emb"] = attn_emb output_i["attn_emb_mask"] = attn_emb_mask decoder_input["attn_emb"] = output_i["attn_emb"] decoder_input["attn_emb_mask"] = output_i["attn_emb_mask"] ############### # determine input word ################ start_word = torch.tensor([self.start_idx,] * beam_size).unsqueeze(1).long() if t == 0: word = start_word else: word = torch.cat((start_word, output_i["seq"]), dim=-1) decoder_input["word"] = word return decoder_input class EventEncoder(nn.Module): """ Encode the Label information in AudioCaps and AudioSet """ def __init__(self, emb_dim, vocab_size=527): super(EventEncoder, self).__init__() self.label_embedding = nn.Parameter( torch.randn((vocab_size, emb_dim)), requires_grad=True) def forward(self, word_idxs): indices = word_idxs / word_idxs.sum(dim=1, keepdim=True) embeddings = indices @ self.label_embedding return embeddings class EventCondTransformerModel(TransformerModel): def __init__(self, encoder: nn.Module, decoder: nn.Module, **kwargs): if not hasattr(self, "compatible_decoders"): self.compatible_decoders = ( models.transformer_decoder.EventTransformerDecoder, ) super().__init__(encoder, decoder, **kwargs) self.label_encoder = EventEncoder(decoder.emb_dim, 527) self.train_forward_keys += ["events"] self.inference_forward_keys += ["events"] # def seq_forward(self, input_dict): # cap = input_dict["cap"] # cap_padding_mask = (cap == self.pad_idx).to(cap.device) # cap_padding_mask = cap_padding_mask[:, :-1] # output = self.decoder( # { # "word": cap[:, :-1], # "attn_emb": input_dict["attn_emb"], # "attn_emb_len": input_dict["attn_emb_len"], # "cap_padding_mask": cap_padding_mask # } # ) # return output def prepare_decoder_input(self, input_dict, output): decoder_input = super().prepare_decoder_input(input_dict, output) decoder_input["events"] = self.label_encoder(input_dict["events"]) return decoder_input def prepare_beamsearch_decoder_input(self, input_dict, output_i): decoder_input = super().prepare_beamsearch_decoder_input(input_dict, output_i) t = input_dict["t"] i = input_dict["sample_idx"] beam_size = input_dict["beam_size"] if t == 0: output_i["events"] = repeat_tensor(self.label_encoder(input_dict["events"])[i], beam_size) decoder_input["events"] = output_i["events"] return decoder_input class KeywordCondTransformerModel(TransformerModel): def __init__(self, encoder: nn.Module, decoder: nn.Module, **kwargs): if not hasattr(self, "compatible_decoders"): self.compatible_decoders = ( models.transformer_decoder.KeywordProbTransformerDecoder, ) super().__init__(encoder, decoder, **kwargs) self.train_forward_keys += ["keyword"] self.inference_forward_keys += ["keyword"] def seq_forward(self, input_dict): cap = input_dict["cap"] cap_padding_mask = (cap == self.pad_idx).to(cap.device) cap_padding_mask = cap_padding_mask[:, :-1] keyword = input_dict["keyword"] output = self.decoder( { "word": cap[:, :-1], "attn_emb": input_dict["attn_emb"], "attn_emb_len": input_dict["attn_emb_len"], "keyword": keyword, "cap_padding_mask": cap_padding_mask } ) return output def prepare_decoder_input(self, input_dict, output): decoder_input = super().prepare_decoder_input(input_dict, output) decoder_input["keyword"] = input_dict["keyword"] return decoder_input def prepare_beamsearch_decoder_input(self, input_dict, output_i): decoder_input = super().prepare_beamsearch_decoder_input(input_dict, output_i) t = input_dict["t"] i = input_dict["sample_idx"] beam_size = input_dict["beam_size"] if t == 0: output_i["keyword"] = repeat_tensor(input_dict["keyword"][i], beam_size) decoder_input["keyword"] = output_i["keyword"] return decoder_input