bibtext: @InProceedings{ Sha_Fewshot_MICCAI2024,
                   author = { Shakeri, Fereshteh and Huang, Yunshi and Silva-Rodriguez, Julio and Bahig, Houda and Tang, An and Dolz, Jose and Ben Ayed, Ismail },
                   title = { { Few-shot Adaptation of Medical Vision-Language Models } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15012 },
                   month = {October},
                   pages = { pending },
                }
abstract: Integrating image and text data through multi-modal learning has emerged as a new approach in medical imaging research, following its successful deployment in computer vision. While considerable efforts have been dedicated to establishing medical foundation models and their zero-shot transfer to downstream tasks, the popular few-shot setting remains relatively unexplored. Following on from the currently strong emergence of this setting in computer vision, we introduce the first structured benchmark for adapting medical vision-language models (VLMs) in a strict few-shot regime and investigate various adaptation strategies commonly used in the context of natural images. Furthermore, we evaluate a simple generalization of the linear-probe adaptation baseline, which seeks an optimal blending of the visual prototypes and text embeddings via learnable class-wise multipliers. Surprisingly, such a text-informed linear probe yields competitive performances in comparison to convoluted prompt-learning and adapter-based strategies, while running considerably faster and accommodating the black-box setting. Our extensive experiments span three different medical modalities and specialized foundation models, nine downstream tasks, and several state-of-the-art few-shot adaptation methods. We made our benchmark and code publicly available to trigger further developments in this emergent subject.
title: Few-shot Adaptation of Medical Vision-Language Models
authors: Shakeri, Fereshteh Huang, Yunshi Silva-Rodriguez, Julio Bahig, Houda Tang, An Dolz, Jose Ben Ayed, Ismail