Upload loconet

config.json

@@ -0,0 +1,17 @@
+{
+  "adjust_attention": false,
+  "architectures": [
+    "loconet"
+  ],
+  "auto_map": {
+    "AutoConfig": "config_loconet.LoCoNetConfig",
+    "AutoModel": "modeling_loconet.loconet"
+  },
+  "av": "speaker_temporal",
+  "av_layers": 3,
+  "clip_length": 200,
+  "model_type": "loconet",
+  "num_speakers": 3,
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.1"
+}

config_loconet.py

@@ -0,0 +1,23 @@
+from transformers import PretrainedConfig
+from typing import List
+
+
+class LoCoNetConfig(PretrainedConfig):
+    model_type = "loconet"
+
+    def __init__(
+        self,
+        num_speakers: int = 3,
+        clip_length: int = 200,
+        av: str = "speaker_temporal",
+        av_layers: int = 3,
+        adjust_attention: bool = False,
+        **kwargs,
+    ):
+
+        self.num_speakers = num_speakers
+        self.clip_length = clip_length
+        self.av = av
+        self.av_layers = av_layers
+        self.adjust_attention = adjust_attention
+        super().__init__(**kwargs)

modeling_loconet.py

@@ -0,0 +1,45 @@
+from config_loconet import LoCoNetConfig
+from transformers import PreTrainedModel
+from loconet_encoder import locoencoder
+from loss_multi import lossAV, lossA, lossV
+
+
+class loconet(PreTrainedModel):
+    config_class = LoCoNetConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.model = locoencoder(config)
+
+    def forward(self, audioFeature, visualFeature, masks, labels=None):
+        b, s, t = visualFeature.shape[:3]
+        visualFeature = visualFeature.view(b * s, *visualFeature.shape[2:])
+        labels = labels.view(b * s, *labels.shape[2:])
+        masks = masks.view(b * s, *masks.shape[2:])
+
+        audioEmbed = self.model.forward_audio_frontend(audioFeature)    # B, C, T, 4
+        visualEmbed = self.model.forward_visual_frontend(visualFeature)
+        audioEmbed = audioEmbed.repeat(s, 1, 1)
+
+        audioEmbed, visualEmbed = self.model.forward_cross_attention(audioEmbed, visualEmbed)
+        outsAV = self.model.forward_audio_visual_backend(audioEmbed, visualEmbed, b, s)
+        outsA = self.model.forward_audio_backend(audioEmbed)
+        outsV = self.model.forward_visual_backend(visualEmbed)
+        num_frames = masks.sum()
+
+        if labels is not None:
+
+            labels = labels.reshape((-1))
+            masks = masks.reshape((-1))
+            nlossAV, _, _, prec = self.lossAV.forward(outsAV, labels, masks)
+            nlossA = self.lossA.forward(outsA, labels, masks)
+            nlossV = self.lossV.forward(outsV, labels, masks)
+
+            nloss = nlossAV + 0.4 * nlossA + 0.4 * nlossV
+
+            return {"loss": nloss, "logits": outsAV}
+
+        else:
+
+            return {"logits": outsAV}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6918e8391d48c40cfd90b332687508bb4b2269879ba1303dacb5a26937ecda87
+size 137464429