bibtext: @InProceedings{ Fan_Aligning_MICCAI2024,
                   author = { Fang, Xiao and Lin, Yi and Zhang, Dong and Cheng, Kwang-Ting and Chen, Hao },
                   title = { { Aligning Medical Images with General Knowledge from Large Language Models } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15010 },
                   month = {October},
                   pages = { pending },
                }
abstract: Pre-trained large vision-language models (VLMs) like CLIP have revolutionized visual representation learning using natural language as supervisions, and have demonstrated promising generalization ability. In this work, we propose ViP, a novel visual symptom-guided prompt learning framework for medical image analysis, which facilitates general knowledge transfer from CLIP. ViP consists of two key components: a visual symptom generator (VSG) and a dual-prompt network. Specifically, VSG aims to extract explicable visual symptoms from pre-trained large language models, while the dual-prompt network uses these visual symptoms to guide the training on two learnable prompt modules, i.e., context prompt and merge prompt, to better adapt our framework to medical image analysis via large VLMs. Extensive experimental results demonstrate that ViP can achieve competitive performance compared to the state-of-the-art methods on two challenging datasets. We provide the source code in the supplementary material.
title: Aligning Medical Images with General Knowledge from Large Language Models
authors: Fang, Xiao Lin, Yi Zhang, Dong Cheng, Kwang-Ting Chen, Hao