bibtext: @InProceedings{ Sun_Continually_MICCAI2024,
                   author = { Sun, Yihua and Khor, Hee Guan and Wang, Yuanzheng and Wang, Zhuhao and Zhao, Hongliang and Zhang, Yu and Ma, Longfei and Zheng, Zhuozhao and Liao, Hongen },
                   title = { { Continually Tuning a Large Language Model for Multi-domain Radiology Report Generation } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15005 },
                   month = {October},
                   pages = { pending },
                }
abstract: Large language models (LLMs) have demonstrated potential across various tasks, including vision-language applications like chest X-ray (XR) report generation (RG) in healthcare. Recent RG approaches focus on optimizing model performance for a single dataset with a single XR modality, often neglecting the critical area of computed tomography (CT) report generation. The challenge is compounded by medical datasets being isolated across different centers, making comprehensive collection difficult. Furthermore, LLMs trained on datasets sequentially can experience catastrophic forgetting. In this paper, we move beyond conventional approaches of training on a single dataset, and focus on improving the overall performance on sequentially collected multi-center datasets. We incorporate four datasets with diverse languages and image modalities for the experiments. Our approach utilizes a minimal number of task-specific learnable weights within an LLM-based RG method for each domain, maintaining the majority of weights frozen to avoid forgetting. Utilizing LLMs’ multilingual generalizability, we align models and facilitate knowledge sharing through a multi-label supervised contrastive loss within the LLM hidden space. We design a 2D-3D adapter for the image encoder to transfer from XR to CT RG tasks. A CT disease graph is established for transferring knowledge from XR to CT RG tasks, using CT’s most relevant XR disease class centers in a triplet loss. Extensive experiments validate our design.
title: Continually Tuning a Large Language Model for Multi-domain Radiology Report Generation
authors: Sun, Yihua Khor, Hee Guan Wang, Yuanzheng Wang, Zhuhao Zhao, Hongliang Zhang, Yu Ma, Longfei Zheng, Zhuozhao Liao, Hongen