bibtext: @InProceedings{ Shi_MoRA_MICCAI2024,
                   author = { Shi, Zhiyi and Kim, Junsik and Li, Wanhua and Li, Yicong and Pfister, Hanspeter },
                   title = { { MoRA: LoRA Guided Multi-Modal Disease Diagnosis with Missing Modality } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15003 },
                   month = {October},
                   pages = { pending },
                }
abstract: Multi-modal pre-trained models efficiently extract and fuse features from different modalities with low memory requirements for fine-tuning. Despite this efficiency, their application in disease diagnosis is under-explored. A significant challenge is the frequent occurrence of missing modalities, which impairs performance. Additionally, fine-tuning the entire pre-trained model demands substantial computational resources. To address these issues, we introduce Modality-aware Low-Rank Adaptation (MoRA), a computationally efficient method. MoRA projects each input to a low intrinsic dimension but uses different modality-aware up-projections for modality-specific adaptation in cases of missing modalities. Practically, MoRA integrates into the first block of the model, significantly improving performance when a modality is missing. It requires minimal computational resources, with less than 1.6\% of the trainable parameters needed compared to training the entire model. Experimental results show that MoRA outperforms existing techniques in disease diagnosis, demonstrating superior performance, robustness, and training efficiency. The code link is: https://github.com/zhiyiscs/MoRA.
title: MoRA: LoRA Guided Multi-Modal Disease Diagnosis with Missing Modality
authors: Shi, Zhiyi Kim, Junsik Li, Wanhua Li, Yicong Pfister, Hanspeter