bibtext: @InProceedings{ Yu_CPCLIP_MICCAI2024,
                   author = { Yu, Xiaowei and Wu, Zihao and Zhang, Lu and Zhang, Jing and Lyu, Yanjun and Zhu, Dajiang },
                   title = { { CP-CLIP: Core-Periphery Feature Alignment CLIP for Zero-Shot Medical Image Analysis } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15003 },
                   month = {October},
                   pages = { pending },
                }
abstract: Multi-modality learning, exemplified by the language and
  image pair pre-trained CLIP model, has demonstrated remarkable performance in enhancing zero-shot capabilities and has gained significant attention in the field. However, simply applying language-image pre-trained CLIP to medical image analysis encounters substantial domain shifts, resulting in significant performance degradation due to inherent disparities between natural (non-medical) and medical image characteristics. To address this challenge and uphold or even enhance CLIP’s zero-shot
  capability in medical image analysis, we develop a novel framework, Core-Periphery feature alignment for CLIP (CP-CLIP), tailored for handling medical images and corresponding clinical reports. Leveraging the foundational core-periphery organization that has been widely observed in brain networks, we augment CLIP by integrating a novel core-peripheryguided neural network. This auxiliary CP network not only aligns text and image features into a unified latent space more efficiently but also ensures the alignment is driven by domain-specific core information, e.g., in medical images and clinical reports. In this way, our approach effectively
  mitigates and further enhances CLIP’s zero-shot performance in
  medical image analysis. More importantly, our designed CP-CLIP exhibits excellent explanatory capability, enabling the automatic identification of critical regions in clinical analysis. Extensive experimentation and evaluation across five public datasets underscore the superiority of our CP-CLIP in zero-shot medical image prediction and critical area detection, showing its promising utility in multimodal feature alignment in current medical applications.
title: CP-CLIP: Core-Periphery Feature Alignment CLIP for Zero-Shot Medical Image Analysis
authors: Yu, Xiaowei Wu, Zihao Zhang, Lu Zhang, Jing Lyu, Yanjun Zhu, Dajiang