actual bigmed model uploaded needs sample fixing

this is without CLIP folder as the model will be cloned from github

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+PathVQA_2Decoders_1024_30iterations_Trial4_CLIPVIT32.pth.tar filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1 +1,3 @@
-.idea/
+.idea/
+__pycache__/
+CLIP/

MED_VQA_Huggyface_Gradio.py

@@ -0,0 +1,181 @@
+##### VQA MED Demo
+
+import gradio as gr
+from transformers import ViltProcessor, ViltForQuestionAnswering
+import torch
+import torch.nn as nn
+from transformers import CLIPTokenizer
+from CLIP import clip
+from Transformers_for_Caption import Transformer_Caption
+import numpy as np
+import torchvision.transforms as transforms
+
+class Config(object):
+    def __init__(self):
+        # Learning Rates
+        # Transformer
+        self.hidden_dim = 512
+        self.pad_token_id = 0
+        self.max_position_embeddings = 76
+        self.layer_norm_eps = 1e-12
+        self.dropout = 0.1
+        self.vocab_size = 49408
+
+        self.enc_layers = 1
+        self.dec_layers = 1
+        self.dim_feedforward = 1024 #2048
+        self.nheads = 4
+        self.pre_norm = True
+        # Dataset
+        #self.dir = os.getcwd() + '/data/coco'
+        self.limit = -1
+
+
+
+##### OUR MODEL
+
+class VQA_Net(nn.Module):
+    def __init__(self, num_classes):
+        super(VQA_Net,self).__init__()
+        #self.VIT = deit_base_distilled_patch16_224(pretrained=True)
+        #self.VIT =vit_base_patch16_224_dino(pretrained=True)
+        #self.VIT = vit_base_patch32_sam_224(pretrained=True)    ###### please not that we used only 6 layers
+        #self.VIT=maxvit_rmlp_nano_rw_256(pretrained=True)
+        #self.VIT = vit_base_patch8_224(pretrained=True)
+        #self.VIT=m = tf_efficientnetv2_m(pretrained=True, features_only=True, out_indices=(1,3), feature_location='expansion')
+        self.backbone, _ = clip.load('ViT-B/32', 'cpu', jit=False)
+        self.input_proj = nn.LayerNorm(512)  # nn.Sequential(nn.LayerNorm(768),nn.Linear(768,768),nn.GELU(),nn.Dropout(0.1))
+        self.transformer_decoder = Transformer_Caption(config,num_decoder_layers=2)
+        self.mlp = nn.Sequential(nn.Sequential(nn.Linear(512, num_classes)))  # MLP(256, 512, 30522, 1) 49408)
+        #self.samples_proj = nn.Sequential(nn.Linear(768,512))
+        self.samples_proj = nn.Identity()
+        self.question_proj = nn.Identity() #nn.Sequential(nn.Linear(512, 512,bias=False))  # nn.Sequential(nn.LayerNorm(768),nn.Linear(768,768),nn.GELU(),nn.Dropout(0.1))
+        #self.tokenizer=CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+
+    def forward(self, samples, question_in, answer_out, mask_answer):
+        # print('Here')
+        #print(samples.shape)
+        _, _,samples = self.backbone.encode_image(samples)
+
+        #samples=self.VIT(samples)
+        #print(samples.shape)
+        samples=samples.float()
+        #samples = self.VIT(samples)
+        #print(`samples.shape)
+        #samples = samples.view(-1, 512, 8 * 8)
+        # print(img_seq.shape)
+        #samples = samples.permute(0, 2, 1)
+        #samples=samples[:,0:,:] @ self.samples_proj
+        samples = self.samples_proj(samples)
+        #print(samples.shape)
+        #print(samples.shape)
+        _, _,question_in = self.backbone.encode_text(question_in)
+        #print(question_in.shape)
+        #samples = self.samples_proj(samples.float())
+        question_in = self.question_proj(question_in.float())
+        #print(question_in.shape)
+        #print(samples.shape)
+        samples = torch.cat((samples, question_in), dim=1)
+        #print(samples.shape)
+
+        # src, mask = features[-1].decompose()
+        # assert mask is not None
+        hs = self.transformer_decoder(self.input_proj(samples.permute(1, 0, 2).float()), answer_out, tgt_mask=mask_answer)
+        out = self.mlp(hs.permute(1, 0, 2))
+        # print(out.shape)
+        return out
+
+config = Config()
+Tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+My_VQA = VQA_Net(num_classes=len(Tokenizer))
+My_VQA.load_state_dict(torch.load("./PathVQA_2Decoders_1024_30iterations_Trial4_CLIPVIT32.pth.tar",map_location= torch.device("cuda" if torch.cuda.is_available() else "cpu")))
+
+
+tfms = transforms.Compose([
+    #transforms.Lambda(under_max),
+    transforms.Resize((224, 224)),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                         std=[0.229, 0.224, 0.225])
+    # transforms.Normalize(0.5, 0.5),
+])
+
+
+def answer_question(image, text_question):
+    with torch.no_grad():
+        for iter in range(1):
+            start_token = Tokenizer.convert_tokens_to_ids("<|startoftext|>")
+            # end_token = Tokenizer.convert_tokens_to_ids("<|endoftext|>")
+            # start_token=tokenizer.convert_tokens_to_ids(tokenizer._cls_token)
+            caption = torch.zeros((1, config.max_position_embeddings), dtype=torch.long)
+            cap_mask = torch.ones((1, config.max_position_embeddings), dtype=torch.bool)
+            caption[:, 0] = start_token
+            cap_mask[:, 0] = False
+            print(text_question)
+            if text_question.find('?') > -1:
+                text_question = text_question.split('?')[0].lower()
+            text_question= np.array(Tokenizer.encode_plus(text_question, max_length=77, pad_to_max_length=True,return_attention_mask=True,
+                                       return_token_type_ids=False, truncation=True)['input_ids'])
+            #print(torch.Tensor(text_question).unsqueeze(0).long())
+            for i in range(config.max_position_embeddings - 1):
+                predictions = My_VQA(image.unsqueeze(0),torch.Tensor(text_question).unsqueeze(0).long(), caption,cap_mask)
+                predictions = predictions[:, i, :]
+                predicted_id = torch.argmax(predictions, axis=-1)
+                caption[:, i + 1] = predicted_id[0]
+                cap_mask[:, i + 1] = False
+                if predicted_id[0] == 49407:
+                    break
+        #print('question:')
+        #print(batch_test['question'])
+        cap_result_intermediate = Tokenizer.decode(caption[0].tolist(), skip_special_tokens=True)
+        #print('+++++++++++++++++++++++++++++++++++')
+        #print("True:")
+        # print(ref_sentence)
+        cap_result = cap_result_intermediate.split('!')
+        #ref_sentence = batch_test['answer'].lower()
+        #print(ref_sentence)
+        #print("Predict:")
+        #print(cap_result)
+        # image_disp=inv_Normalize(batch_test['image'])[0].permute(1,2,0).detach().cpu().numpy()
+        # print('************************')
+        # plt.imshow(image_disp)
+        return cap_result
+
+
+def infer_answer_question(image, text):
+    if text is None:
+        cap_result = "please write a question"
+    elif image is None:
+        cap_result = "please upload an image"
+    else:
+        image_encoded = tfms(image)
+        print(image_encoded)
+        cap_result=answer_question(image_encoded,text)[0]
+
+    return cap_result
+
+
+image = gr.inputs.Image(type="pil")
+question = gr.inputs.Textbox(label="Question")
+answer = gr.outputs.Textbox(label="Predicted answer")
+examples = [["train_0000.jpg", "Where are liver stem cells (oval cells) located?"],
+            ["train_0001.jpg", "What are stained here with an immunohistochemical stain for cytokeratin 7?"],
+            ["train_0002.jpg", "What are bile duct cells and canals of Hering stained here with for cytokeratin 7?"],
+            ["train_0003.jpg", "Are bile duct cells and canals of Hering stained here with an immunohistochemical stain for cytokeratin 7?"],
+            ["train_0018.jpg", "Is there an infarct in the brain hypertrophy?"],
+            ["train_0019.jpg", "What is ischemic coagulative necrosis?"]]
+
+title = "Interactive Vsisual Question Answering demo(BigMed@ai: Artificial Intelligence for Large-Scale Medical Image Analysis)"
+description = "<div style='display: flex;align-items: center;justify-content: space-between;'><p style='width:60vw;'>Gradio Demo for VQA medical model trained on PathVQA dataset, To use it, upload your image and type a question and click 'submit', or click one of the examples to load them.</p><a href='https://github.com/dandelin/ViLT' target='_blank' class='link'><img src='file/GitHub.png' style='justify-self:margin-top:0.5em;center; width:calc(200px + 5vw);'></a></div>"
+### link to paper and github code
+article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2102.03334' target='_blank'>BigMed@ai</a> | <a href='https://github.com/dandelin/ViLT' target='_blank'>Github Repo</a></p>"
+
+interface = gr.Interface(fn=infer_answer_question,
+                         inputs=[image, question],
+                         outputs=answer,
+                         examples=examples,
+                         title=title,
+                         description=description,
+                         article=article,
+                         enable_queue=True)
+interface.launch(debug=True)

PathVQA_2Decoders_1024_30iterations_Trial4_CLIPVIT32.pth.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79262e9686303e4e8c515078b820341394b6b380382be0819c2c01d9dd9eaa51
+size 589964081

README.md

@@ -5,7 +5,7 @@ colorFrom: yellow
 colorTo: red
 sdk: gradio
 sdk_version: 3.15.0
-app_file: app.py
+app_file: MED_VQA_Huggyface_Gradio.py
 pinned: false
 ---
 

Transformers_for_Caption.py

@@ -0,0 +1,364 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import copy
+from typing import Optional, List
+
+import torch
+import torch.nn.functional as F
+from torch import nn, Tensor
+
+
+class Transformer_Caption(nn.Module):
+
+    def __init__(self, config,d_model=512, nhead=4, num_encoder_layers=1,
+                 num_decoder_layers=2, dim_feedforward=1024, dropout=0.1,
+                 activation="gelu", normalize_before=False,
+                 return_intermediate_dec=False):
+        super().__init__()
+
+        encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward,
+                                                dropout, activation, normalize_before)
+        encoder_norm = nn.LayerNorm(d_model) if normalize_before else None
+        self.encoder = TransformerEncoder(
+            encoder_layer, num_encoder_layers, encoder_norm)
+
+        self.embeddings = DecoderEmbeddings(config)
+        decoder_layer = TransformerDecoderLayer(d_model, nhead, dim_feedforward,
+                                                dropout, activation, normalize_before)
+        decoder_norm = nn.LayerNorm(d_model)
+        self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, decoder_norm,
+                                          return_intermediate=return_intermediate_dec)
+        print("Num decoders:")
+        print(num_decoder_layers)
+        self._reset_parameters()
+
+        self.d_model = d_model
+        self.nhead = nhead
+
+    def _reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                nn.init.xavier_uniform_(p)
+
+    def forward(self, src, tgt, tgt_mask):
+        # flatten NxCxHxW to HWxNxC
+        #print("HERRRRRR")
+        #print(src.shape)
+        h, bs, w = src.shape
+        #src = src.permute(1, 0, 2)
+        #print("SRCCCCCCCC")
+        #print(src.shape)
+        #pos_embed = pos_embed.flatten(2).permute(2, 0, 1)
+        #mask = mask.flatten(1)
+        #print(num_decoder_layers)
+
+        tgt = self.embeddings(tgt).permute(1, 0, 2)
+        query_embed = self.embeddings.position_embeddings.weight.unsqueeze(1)
+        query_embed = query_embed.repeat(1, bs, 1)
+        #print("firstmyyyyyyyyyyyyyy")
+        #print(tgt.shape)
+        #print(tgt_mask.shape)
+        #print(pos_embed.shape)
+        #print(query_embed.shape)
+        #print(generate_square_subsequent_mask(len(tgt)).to(tgt.device).shape)
+        #print(src.shape)
+
+        #memory = self.encoder(src, src_key_padding_mask=None, pos=None)
+        #memory = self.encoder(src)
+        #print("then....")
+        #print(tgt_mask.shape)
+        hs = self.decoder(tgt, src, memory_key_padding_mask=None, tgt_key_padding_mask=tgt_mask,
+                          pos=None, query_pos=query_embed,
+                          tgt_mask=generate_square_subsequent_mask(len(tgt)).to(tgt.device))
+        #hs = self.decoder(tgt, memory, tgt_key_padding_mask=tgt_mask,query_pos=query_embed,tgt_mask=generate_square_subsequent_mask(len(tgt)).to(tgt.device))
+
+        return hs
+
+
+class TransformerEncoder(nn.Module):
+
+    def __init__(self, encoder_layer, num_layers, norm=None):
+        super().__init__()
+        self.layers = _get_clones(encoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+
+    def forward(self, src,
+                mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None):
+        output = src
+
+        for layer in self.layers:
+            output = layer(output, src_mask=mask,
+                           src_key_padding_mask=src_key_padding_mask, pos=pos)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output
+
+
+class TransformerDecoder(nn.Module):
+
+    def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False):
+        super().__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.return_intermediate = return_intermediate
+
+    def forward(self, tgt, memory,
+                tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None,
+                tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None,
+                query_pos: Optional[Tensor] = None):
+        output = tgt
+
+        intermediate = []
+
+        for layer in self.layers:
+            output = layer(output, memory, tgt_mask=tgt_mask,
+                           memory_mask=memory_mask,
+                           tgt_key_padding_mask=tgt_key_padding_mask,
+                           memory_key_padding_mask=memory_key_padding_mask,
+                           pos=pos, query_pos=query_pos)
+            if self.return_intermediate:
+                intermediate.append(self.norm(output))
+
+        if self.norm is not None:
+            output = self.norm(output)
+            if self.return_intermediate:
+                intermediate.pop()
+                intermediate.append(output)
+
+        if self.return_intermediate:
+            return torch.stack(intermediate)
+
+        return output
+
+
+class TransformerEncoderLayer(nn.Module):
+
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.activation = _get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+    def with_pos_embed(self, tensor, pos: Optional[Tensor]):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(self,
+                     src,
+                     src_mask: Optional[Tensor] = None,
+                     src_key_padding_mask: Optional[Tensor] = None,
+                     pos: Optional[Tensor] = None):
+        q = k = self.with_pos_embed(src, pos)
+        src2 = self.self_attn(q, k, value=src, attn_mask=src_mask,
+                              key_padding_mask=src_key_padding_mask)[0]
+        src = src + self.dropout1(src2)
+        src = self.norm1(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
+        src = src + self.dropout2(src2)
+        src = self.norm2(src)
+        return src
+
+    def forward_pre(self, src,
+                    src_mask: Optional[Tensor] = None,
+                    src_key_padding_mask: Optional[Tensor] = None,
+                    pos: Optional[Tensor] = None):
+        src2 = self.norm1(src)
+        q = k = self.with_pos_embed(src2, pos)
+        src2 = self.self_attn(q, k, value=src2, attn_mask=src_mask,
+                              key_padding_mask=src_key_padding_mask)[0]
+        src = src + self.dropout1(src2)
+        src2 = self.norm2(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src2))))
+        src = src + self.dropout2(src2)
+        return src
+
+    def forward(self, src,
+                src_mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None):
+        if self.normalize_before:
+            return self.forward_pre(src, src_mask, src_key_padding_mask, pos)
+        return self.forward_post(src, src_mask, src_key_padding_mask, pos)
+
+
+class TransformerDecoderLayer(nn.Module):
+
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        self.multihead_attn = nn.MultiheadAttention(
+            d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+        self.dropout3 = nn.Dropout(dropout)
+
+        self.activation = _get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+
+    def with_pos_embed(self, tensor, pos: Optional[Tensor]):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(self, tgt, memory,
+                     tgt_mask: Optional[Tensor] = None,
+                     memory_mask: Optional[Tensor] = None,
+                     tgt_key_padding_mask: Optional[Tensor] = None,
+                     memory_key_padding_mask: Optional[Tensor] = None,
+                     pos: Optional[Tensor] = None,
+                     query_pos: Optional[Tensor] = None):
+        #print(tgt.shape)
+        #print(query_pos.shape)
+
+        q = k = self.with_pos_embed(tgt, query_pos)
+        tgt2 = self.self_attn(q, k, value=tgt, attn_mask=tgt_mask,
+                              key_padding_mask=tgt_key_padding_mask)[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm1(tgt)
+        tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt, query_pos),
+                                   key=self.with_pos_embed(memory, pos),
+                                   value=memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask)[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+        return tgt
+
+    def forward_pre(self, tgt, memory,
+                    tgt_mask: Optional[Tensor] = None,
+                    memory_mask: Optional[Tensor] = None,
+                    tgt_key_padding_mask: Optional[Tensor] = None,
+                    memory_key_padding_mask: Optional[Tensor] = None,
+                    pos: Optional[Tensor] = None,
+                    query_pos: Optional[Tensor] = None):
+        tgt2 = self.norm1(tgt)
+        q = k = self.with_pos_embed(tgt2, query_pos)
+        tgt2 = self.self_attn(q, k, value=tgt2, attn_mask=tgt_mask,
+                              key_padding_mask=tgt_key_padding_mask)[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt2 = self.norm2(tgt)
+        tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt2, query_pos),
+                                   key=self.with_pos_embed(memory, pos),
+                                   value=memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask)[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt2 = self.norm3(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2))))
+        tgt = tgt + self.dropout3(tgt2)
+        return tgt
+
+    def forward(self, tgt, memory,
+                tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None,
+                tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None,
+                query_pos: Optional[Tensor] = None):
+        if self.normalize_before:
+            return self.forward_pre(tgt, memory, tgt_mask, memory_mask,
+                                    tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos)
+        return self.forward_post(tgt, memory, tgt_mask, memory_mask,
+                                 tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos)
+
+
+class DecoderEmbeddings(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.word_embeddings = nn.Embedding(
+            config.vocab_size, config.hidden_dim, padding_idx=config.pad_token_id)
+        self.position_embeddings = nn.Embedding(
+            config.max_position_embeddings, config.hidden_dim
+        )
+
+        self.LayerNorm = torch.nn.LayerNorm(
+            config.hidden_dim, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x):
+        input_shape = x.size()
+        x=x.long()
+        #print(x.shape)
+        seq_length = input_shape[1]
+        device = x.device
+
+        position_ids = torch.arange(
+            seq_length, dtype=torch.long, device=device)
+        position_ids = position_ids.unsqueeze(0).expand(input_shape)
+        input_embeds = self.word_embeddings(x)
+        position_embeds = self.position_embeddings(position_ids)
+
+
+        embeddings = input_embeds + position_embeds
+        embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings)
+
+        #print(embeddings)
+
+        return embeddings
+
+
+def _get_clones(module, N):
+    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
+
+
+def _get_activation_fn(activation):
+    """Return an activation function given a string"""
+    if activation == "relu":
+        return F.relu
+    if activation == "gelu":
+        return F.gelu
+    if activation == "glu":
+        return F.glu
+    raise RuntimeError(F"activation should be relu/gelu, not {activation}.")
+
+
+def generate_square_subsequent_mask(sz):
+    r"""Generate a square mask for the sequence. The masked positions are filled with float('-inf').
+        Unmasked positions are filled with float(0.0).
+    """
+    mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)
+    mask = mask.float().masked_fill(mask == 0, float(
+        '-inf')).masked_fill(mask == 1, float(0.0))
+    return mask
+
+
+def build_transformer(config):
+    return Transformer_Caption(
+        config,
+        d_model=config.hidden_dim,
+        dropout=config.dropout,
+        nhead=config.nheads,
+        dim_feedforward=config.dim_feedforward,
+        num_encoder_layers=config.enc_layers,
+        num_decoder_layers=config.dec_layers,
+        normalize_before=config.pre_norm,
+        return_intermediate_dec=False,
+    )

requirements.txt

@@ -1,2 +1,5 @@
-torch == 1.13.1
-transformers == 4.25.1
+clip-by-openai
+transformers
+torch
+numpy
+ftfy