Kaggle competition: TReNDS Neuroimaging
<SOS> metric: WMAE Kaggle data source: trends-neuroimaging<filter>
!pip install -q joypy
xnli_valid_without_dups=xnli_valid_df[(~xnli_valid_df['combo'].isin(dup_pairs['combo'].tolist())) ]<drop_column>
from sklearn.model_selection import KFold, train_test_split
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import joypy
from tqdm.notebook import tqdm
from glob import glob
import gc
import nilearn as nl
import nilearn.plotting as nlplt
import nibabel as nib
import h5py
import lightgbm as lgb
from scipy.stats import skew, kurtosis
import os, random
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.utils import Sequence
xnli_valid_without_dups.drop(['combo'],axis=1,inplace=True )<feature_engineering>
MAIN_DATA_PATH = '/kaggle/input/trends-assessment-prediction/'
train_scores_df = pd.read_csv(MAIN_DATA_PATH + 'train_scores.csv')
 icn_numbers_df = pd.read_csv(MAIN_DATA_PATH + 'ICN_numbers.csv')
 loading_df = pd.read_csv(MAIN_DATA_PATH + 'loading.csv')
 fnc_df = pd.read_csv(MAIN_DATA_PATH + 'fnc.csv' )
xnli_valid_without_dups['dataset']="xnli"<concatenate>
print("Kurtosis(Fisher's definition)")
 train_scores_df.kurtosis()
train_df=pd.concat([train_df,xnli_valid_without_dups])
 train_df.shape<create_dataframe>
round(train_scores_df.isna().sum() / len(train_scores_df)* 100, 2 )
mnli=nlp.load_dataset(path='glue', name='mnli')
 mnli<prepare_output>
train_scores_df.fillna(train_scores_df.mean() , inplace=True )
mnli_train=pd.DataFrame(mnli['train'])
 mnli_train.head()<merge>
train_scores_df.isna().sum()
test_dup=pd.merge(test[['premise','hypothesis']],mnli_train[['premise','hypothesis']],how="outer",indicator=True)
 test_dup['_merge'].unique()<drop_column>
features_df = pd.merge(train_scores_df, loading_df, on=['Id'], how='left')
 features_df.head()
mnli_train.drop(['idx'],axis=1,inplace=True )<feature_engineering>
fnc_df.isna().sum().sum()
mnli_train['lang_abv']="en"
mnli_train['dataset']="mnli"<concatenate>
features_df = pd.merge(features_df, fnc_df, how='left', on='Id')
 features_df.head()
train_df=pd.concat([train_df,mnli_train])
 train_df.shape<import_modules>
!wget https://github.com/Chaogan-Yan/DPABI/raw/master/Templates/ch2better.nii
from transformers import BertTokenizer,TFBertModel
import tensorflow as tf
from transformers import AutoTokenizer 
<define_variables>
mask_filename = '.. /input/trends-assessment-prediction/fMRI_mask.nii'
smri_filename = 'ch2better.nii'
mask_niimg = nl.image.load_img(mask_filename)
 def load_subject(filename, mask_niimg):
subject_data = None
with h5py.File(filename, 'r')as f:
subject_data = f['SM_feature'][() ]
subject_data = np.moveaxis(subject_data, [0, 1, 2, 3], [3, 2, 1, 0])
 subject_niimg = nl.image.new_img_like(mask_niimg, subject_data, affine=mask_niimg.affine, copy_header=True)
 return subject_niimg
MODEL_NAME="jplu/tf-xlm-roberta-large"
MAX_LEN=64
BATCH_SIZE=64<load_pretrained>
fMRI_train_data_path = '.. /input/trends-assessment-prediction/fMRI_train/'
filenames = random.choices(os.listdir(fMRI_train_data_path), k=4 )
tokeniser=AutoTokenizer.from_pretrained(MODEL_NAME)
<import_modules>
fnc_df = pd.read_csv(".. /input/trends-assessment-prediction/fnc.csv")
 loading_df = pd.read_csv(".. /input/trends-assessment-prediction/loading.csv")
 labels_df = pd.read_csv(".. /input/trends-assessment-prediction/train_scores.csv")
 fnc_features, loading_features = list(fnc_df.columns[1:]), list(loading_df.columns[1:])
 df = fnc_df.merge(loading_df, on="Id")
 labels_df["is_train"] = True
df = df.merge(labels_df, on="Id", how="left")
 target_cols = ['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']
test_df = df[df["is_train"] != True].copy()
train_df = df[df["is_train"] == True].copy()
from tensorflow.keras import Input, Model, Sequential
from tensorflow.keras.layers import Dense, Dropout
from keras.optimizers import Adam
from transformers import TFAutoModel<define_search_model>
y_train_df = train_df[target_cols]
train_df = train_df.drop(target_cols + ['is_train'], axis=1)
 test_df = test_df.drop(target_cols + ['is_train'], axis=1)
 FNC_SCALE = 1/500
test_df[fnc_features] *= FNC_SCALE
train_df[fnc_features] *= FNC_SCALE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("In TPU STRATERGY")
 print('Number of replicas:', strategy.num_replicas_in_sync)
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync )<choose_model_class>
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))
def createModel() :
with strategy.scope() :
model=TFAutoModel.from_pretrained(MODEL_NAME)
 input_ids = tf.keras.layers.Input(shape=(MAX_LEN,), name='input_token', dtype='int32')
 embedding=model(input_ids)[0]
cls_token=embedding[:,0,:]
output_layer = Dense(3, activation='softmax' )(cls_token)
 classification_model= Model(inputs=input_ids, outputs = output_layer)
 classification_model.compile(Adam(lr=1e-5),loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 return classification_model 
<create_dataframe>
param = {'objective':'regression',
'metric':'rmse',
'bossting_type':'gbdt',
'learning_rate':0.01,
'max_depth':-1}
output = pd.DataFrame()
for target in ['age','domain1_var1','domain1_var2','domain2_var1','domain2_var2']:
X_train, X_val, y_train, y_val = train_test_split(train_df.iloc[:,1:], y_train_df[target], test_size=0.2, shuffle=True, random_state=20)
 train_data = lgb.Dataset(X_train, label=y_train)
 val_data = lgb.Dataset(X_val, label=y_val)
 model = lgb.train(param,
train_data,
10000,
early_stopping_rounds=15,
valid_sets=[val_data],
verbose_eval=50)
 temp = pd.DataFrame(test_df['Id'].apply(lambda x:str(x)+ '_'+ target))
temp['Predicted'] = model.predict(test_df.iloc[:,1:])
 output = pd.concat([output,temp] )
tokeniser,train_dataset,valid_dataset,train_rows=createTFDataSet(train_df,external_dataset=['xnli','mnli'] )<train_model>
sample_submission = pd.read_csv("/kaggle/input/trends-assessment-prediction/sample_submission.csv")
 output = sample_submission.drop('Predicted',axis=1 ).merge(output,on='Id',how='left' )
n_steps = train_rows//BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=5
)<categorify>
sub_df = pd.read_csv(".. /input/trends-svc-rapids-ai/submission.csv" )
test_encoded=tokeniser.batch_encode_plus(test[['hypothesis','premise']].values.tolist() ,pad_to_max_length=True,max_length=MAX_LEN,truncation=True)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_encoded['input_ids'])
.batch(BATCH_SIZE)
)
<predict_on_test>
final_sub = pd.DataFrame(data={
'id':sample_submission['Id'],
'Predicted': 0.7*output['Predicted'] + 0.3*sub_df['Predicted']
} )
<create_dataframe><EOS> 
sub_df.to_csv("submission.csv", index=False )