bibtext: @InProceedings{ Din_HiA_MICCAI2024,
                   author = { Ding, Xinpeng and Chu, Yongqiang and Pi, Renjie and Wang, Hualiang and Li, Xiaomeng },
                   title = { { HiA: Towards Chinese Multimodal LLMs for Comparative High-Resolution Joint Diagnosis } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15012 },
                   month = {October},
                   pages = { pending },
                }
abstract: Multimodal large language models (MLLMs) have been explored in the Chinese medical domain for comprehending complex healthcare. However, due to the flaws in training data and architecture design, current Chinese medical MLLMs suffer from several limitations: cultural biases from English machine translations, limited comparative ability from single image input and difficulty in identifying small lesions with low-resolution images. To address these problems, we first introduce a new instruction-following dataset, Chili-Joint (Chinese Interleaved Image-Text Dataset for Joint Diagnosis) collected from the hospital in mainland China, avoiding cultural biases and errors caused by machine translation. Besides one single image input, Chili-Joint also has multiple images obtained at various intervals during a patient’s treatment, thus facilitating an evaluation of the treatment’s outcomes. We further propose a novel HiA (High-resolution instruction-aware Adapter) to incorporate high-resolutioninstruction-aware visual features into LLMs to facilitate the current MLLMs to observe the small lesions as well as the comparative analysis. Extensive experiments on Chili-Joint demonstrate our HiA can be a plug-and-play method to improve the performance of current MLLMs for medical analysis. The code is available at https://github.com/xmed-lab/HiA.
title: HiA: Towards Chinese Multimodal LLMs for Comparative High-Resolution Joint Diagnosis
authors: Ding, Xinpeng Chu, Yongqiang Pi, Renjie Wang, Hualiang Li, Xiaomeng