bibtext: @InProceedings{ Xu_Temporal_MICCAI2024,
                   author = { Xu, Jingwen and Zhu, Ye and Lyu, Fei and Wong, Grace Lai-Hung and Yuen, Pong C. },
                   title = { { Temporal Neighboring Multi-Modal Transformer with Missingness-Aware Prompt for Hepatocellular Carcinoma Prediction } },
                   booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2024},
                   year = {2024},
                   publisher = {Springer Nature Switzerland},
                   volume = { LNCS 15001 },
                   month = {October},
                   pages = { pending },
                }
abstract: Early prediction of hepatocellular carcinoma (HCC) is necessary to facilitate appropriate surveillance strategy and reduce cancer mortality. Incorporating CT scans and clinical time series can greatly increase the accuracy of predictive models. However, there are two challenges to effective multi-modal learning: (a) CT scans and clinical time series can be asynchronous and irregularly sampled. (b) CT scans are often missing compared with clinical time series. To tackle the above challenges, we propose a Temporal Neighboring Multi-modal Transformer with Missingness-Aware Prompt (\textbf{TNformer-MP}) to integrate clinical time series and available CT scans for HCC prediction. Specifically, to explore the inter-modality temporal correspondence, TNformer-MP exploits a Temporal Neighboring Multimodal Tokenizer (\textbf{TN-MT}) to fuse the CT embedding into its multiple-scale neighboring tokens from clinical time series. To mitigate the performance drop caused by missing CT modality, TNformer-MP exploits a Missingness-aware Prompt-driven Multimodal Tokenizer (\textbf{MP-MT}) that adopts missingness-aware prompts to adjust the encoding of clinical time series tokens. Experiments conducted on a largescale multimodal datasets of 36,353 patients show that our method achieves superior performance with existing methods.
title: Temporal Neighboring Multi-Modal Transformer with Missingness-Aware Prompt for Hepatocellular Carcinoma Prediction
authors: Xu, Jingwen Zhu, Ye Lyu, Fei Wong, Grace Lai-Hung Yuen, Pong C.