bibtext: @InProceedings{ Zha_Knowledgedriven_MICCAI2024,
                   author = { Zhang, Yupei and Wang, Xiaofei and Meng, Fangliangzi and Tang, Jin and Li, Chao },
                   title = { { Knowledge-driven Subspace Fusion and Gradient Coordination for Multi-modal Learning } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15004 },
                   month = {October},
                   pages = { pending },
                }
abstract: Most recently, molecular pathology has played a crucial role in cancer diagnosis and prognosis assessment. Deep learning-based methods have been proposed for integrating multi-modal genomic and histology data for efficient molecular pathology analysis. However, current multi-modal approaches simply treat each modality equally, ignoring the modal unique information and the complex correlation across modalities, which hinders the effective multi-modal feature representation for downstream tasks. Besides, considering the intrinsic complexity in tumour ecosystem, where both tumour cells and tumor microenvironment (TME) contribute to the cancer status, it is challenging to utilize a single embedding space to model the mixed genomic profiles of the tumour ecosystem. To tackle these challenges, in this paper, we propose a biologically interpretative and robust multi-modal learning framework to efficiently integrate histology images and genomics data. 
   Specifically, to enhance cross-modal interactions, we design a knowledge-driven subspace fusion scheme, consisting a cross-modal deformable attention module and a gene-guided consistency strategy, which Additionally, in pursuit of dynamically  optimizing the subspace knowledge, we further propose a novel gradient coordinatio n learning strategy. Extensive experiments on two public datasets demonstrate the effectiveness of our proposed method, outperforming state-of-the-art techniques in three downstream tasks of glioma diagnosis, tumour grading, and survival analysis.
title: Knowledge-driven Subspace Fusion and Gradient Coordination for Multi-modal Learning
authors: Zhang, Yupei Wang, Xiaofei Meng, Fangliangzi Tang, Jin Li, Chao