bibtext: @InProceedings{ Bui_VisualTextual_MICCAI2024,
                   author = { Bui, Phuoc-Nguyen and Le, Duc-Tai and Choo, Hyunseung },
                   title = { { Visual-Textual Matching Attention for Lesion Segmentation in Chest Images } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15009 },
                   month = {October},
                   pages = { pending },
                }
abstract: Lesion segmentation in chest images is crucial for AI-assisted diagnostic systems of pulmonary conditions. The multi-modal approach, which combines image and text description, has achieved notable performance in medical image segmentation. However, the existing methods mainly focus on improving the decoder using the text information while the encoder remains unexplored. In this study, we introduce a Multi-Modal Input UNet model, namely MMI-UNet, which utilizes visual-textual matching (VTM) features for infected areas segmentation in chest X-ray images. These VTM features, which contain visual features that are relevant to the text description, are created by a combination of self-attention and cross-attention mechanisms in a novel Image-Text Matching (ITM) module integrated into the encoder. Empirically, extensive evaluations on the QaTa-Cov19 and MosMedData+ datasets demonstrate MMI-UNet’s state-of-the-art performance over both uni-modal and previous multi-modal methods. Furthermore, our method also outperforms the best uni-modal method even with 15% of the training data. These findings highlight the interpretability of our vision-language model, advancing the explainable diagnosis of pulmonary diseases and reducing the labeling cost for segmentation tasks in the medical field. The source code is made publicly available on GitHub.
title: Visual-Textual Matching Attention for Lesion Segmentation in Chest Images
authors: Bui, Phuoc-Nguyen Le, Duc-Tai Choo, Hyunseung