Repository: Project-MONAI/MONAI

Issue:
class MultiModal - updating in the for loop
**Describe the bug**
It looks like that in the 'class MultiModal(BertPreTrainedModel):' inside the 'forward' function in lines 240, 242, and 244, the variables are not getting updated. As the input argument, they receive the very initial input and not the output of the previous layer which is supposed to be the reason for using a 'for loop'.
Is that a bug, or is there something that I'm missing out on?



Discussion:
@ahatamiz this looks like a bug, could you help?
https://github.com/Project-MONAI/MONAI/blob/82ec2fef6e0636a0290a454b82a262d6871a84d5/monai/networks/nets/transchex.py#L237-L245

Resolving patch:
diff --git a/monai/networks/nets/transchex.py b/monai/networks/nets/transchex.py
--- a/monai/networks/nets/transchex.py
+++ b/monai/networks/nets/transchex.py
@@ -185,12 +185,15 @@ class BertMixedLayer(nn.Module):
 
     def __init__(self, config) -> None:
         super().__init__()
-        self.att = BertAttention(config)
-        self.output = BertOutput(config)
+        self.att_x = BertAttention(config)
+        self.output_x = BertOutput(config)
+        self.att_y = BertAttention(config)
+        self.output_y = BertOutput(config)
 
     def forward(self, x, y):
-        output = self.att(x, y)
-        return self.output(output, x)
+        output_x = self.att_x(x, y)
+        output_y = self.att_y(y, x)
+        return self.output_x(output_x, x), self.output_y(output_y, y)
 
 
 class Pooler(nn.Module):
@@ -237,12 +240,12 @@ def __init__(
     def forward(self, input_ids, token_type_ids=None, vision_feats=None, attention_mask=None):
         language_features = self.embeddings(input_ids, token_type_ids)
         for layer in self.vision_encoder:
-            hidden_state_vision = layer(vision_feats, None)[0]
+            vision_feats = layer(vision_feats, None)[0]
         for layer in self.language_encoder:
-            hidden_state_language = layer(language_features, attention_mask)[0]
+            language_features = layer(language_features, attention_mask)[0]
         for layer in self.mixed_encoder:
-            hidden_state_mixed = layer(hidden_state_language, hidden_state_vision)
-        return hidden_state_mixed
+            language_features, vision_feats = layer(language_features, vision_feats)
+        return language_features, vision_feats
 
 
 class Transchex(torch.nn.Module):
@@ -367,9 +370,9 @@ def forward(self, input_ids, token_type_ids=None, vision_feats=None):
         vision_feats = self.vision_proj(vision_feats).flatten(2).transpose(1, 2)
         vision_feats = self.norm_vision_pos(vision_feats)
         vision_feats = vision_feats + self.pos_embed_vis
-        hidden_state_mixed = self.multimodal(
+        hidden_state_lang, hidden_state_vis = self.multimodal(
             input_ids=input_ids, token_type_ids=token_type_ids, vision_feats=vision_feats, attention_mask=attention_mask
         )
-        pooled_features = self.pooler(hidden_state_mixed)
+        pooled_features = self.pooler(hidden_state_lang)
         logits = self.cls_head(self.drop(pooled_features))
         return logits


Test patch:
diff --git a/tests/utils.py b/tests/utils.py
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -114,6 +114,8 @@ def skip_if_downloading_fails():
     except (ContentTooShortError, HTTPError, ConnectionError) as e:
         raise unittest.SkipTest(f"error while downloading: {e}") from e
     except RuntimeError as rt_e:
+        if "unexpected EOF" in str(rt_e):
+            raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e  # incomplete download
         if "network issue" in str(rt_e):
             raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e
         if "gdown dependency" in str(rt_e):  # no gdown installed