Kaggle competition: TReNDS Neuroimaging
<SOS> metric: WMAE Kaggle data source: trends-neuroimaging<choose_model_class>
!cp.. /input/rapids/rapids.0.13.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null
sys.path = ["/opt/conda/envs/rapids/lib/python3.6/site-packages"] + sys.path
sys.path = ["/opt/conda/envs/rapids/lib/python3.6"] + sys.path
sys.path = ["/opt/conda/envs/rapids/lib"] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/
with strategy.scope() :
roberta_model = build_roberta(max_len=80)
 filepath='roberta_best_weight.hdf5'
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
 n_steps = len(x_train)//BATCH_SIZE
roberta_model.fit(tf_train_dataset,steps_per_epoch=n_steps,
validation_data=tf_val_dataset,callbacks=[checkpoint],epochs=10 )<find_best_model_class>
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))
def model_cv_score(model):
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=98)
 E_cv = 0
for fold,(train_idx,val_idx)in enumerate(skf.split(df_train['premise'],df_train['label'])) :
print('
Fold %s:'%(fold))
K.clear_session()
train_premise = df_train['premise'].iloc[train_idx]
train_hypothesis = df_train['hypothesis'].iloc[train_idx]
train_label = df_train['label'].iloc[train_idx]
val_premise = df_train['premise'].iloc[val_idx]
val_hypothesis = df_train['hypothesis'].iloc[val_idx]
val_label = df_train['label'].iloc[val_idx]
cv_train_input = bert_encode(train_premise.values, train_hypothesis.values, tokenizer)
 cv_val_input = bert_encode(val_premise.values, val_hypothesis.values, tokenizer)
 filepath='best_weight_fold%s.h5'%(fold)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
 model.fit(cv_train_input, train_label.values, epochs =2,
batch_size = 16, callbacks=[checkpoint],
validation_data=(cv_val_input,val_label.values))
print('
Predicting OOF Error...... ')
 model.load_weights('best_weight_fold%s.h5'%(fold))
val_pred = [np.argmax(p)for p in model.predict(cv_val_input)]
oof_error = np.sum([pred!=true for pred,true in zip(val_pred,val_label)])/len(val_label)
 print('Fold_%s Error: %s' %(fold,oof_error))
E_cv += oof_error
print('
E_cv = %s'%(E_cv/5))<predict_on_test>
fnc_df = cudf.read_csv(".. /input/trends-assessment-prediction/fnc.csv")
 loading_df = cudf.read_csv(".. /input/trends-assessment-prediction/loading.csv")
 fnc_features, loading_features = list(fnc_df.columns[1:]), list(loading_df.columns[1:])
 df = fnc_df.merge(loading_df, on="Id")
 labels_df = cudf.read_csv(".. /input/trends-assessment-prediction/train_scores.csv")
 labels_df["is_train"] = True
df = df.merge(labels_df, on="Id", how="left")
 test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()
test_text = df_test[['premise', 'hypothesis']].values.tolist()
encoded_test_text = roberta_encode(test_text,tokenizer,max_len=80)
 tf_test_dataset =(tf.data.Dataset 
.from_tensor_slices(encoded_test_text['input_ids'])
.batch(BATCH_SIZE))
roberta_model.load_weights('./roberta_best_weight.hdf5')
 pred = [np.argmax(res)for res in roberta_model.predict(tf_test_dataset)]<save_to_csv>
FNC_SCALE = 1/600
df[fnc_features] *= FNC_SCALE
test_df[fnc_features] *= FNC_SCALE
submission = pd.DataFrame({'id':df_test.id,'prediction':pred})
 submission.to_csv('submission.csv',index=False )<install_modules>
import nilearn as nl
import nibabel as nib
from nilearn import image
from nilearn import plotting
from nilearn import datasets
from nilearn import surface
import nilearn.plotting as nlplt
!pip install transformers==3.0.2
!pip install nlp<set_options>
fmri_mask = '.. /input/trends-assessment-prediction/fMRI_mask.nii'
np.random.seed(123 )<set_options>
smri = 'ch2better.nii'
mask_img = nl.image.load_img(fmri_mask)
 def load_subject(filename, mask_img):
subject_data = None
with h5py.File(filename, 'r')as f:
subject_data = f['SM_feature'][() ]
subject_data = np.moveaxis(subject_data, [0,1,2,3], [3,2,1,0])
 subject_img = nl.image.new_img_like(mask_img, subject_data, affine=mask_img.affine, copy_header=True)
 return subject_img
files = random.choices(os.listdir('.. /input/trends-assessment-prediction/fMRI_train/'), k = 3)
 for file in files:
subject = os.path.join('.. /input/trends-assessment-prediction/fMRI_train/', file)
 subject_img = load_subject(subject, mask_img)
 print("Image shape is %s" %(str(subject_img.shape)))
 num_components = subject_img.shape[-1]
print("Detected {num_components} spatial maps".format(num_components=num_components))
rsn = subject_img
first_rsn = image.index_img(rsn, 0)
 print(first_rsn.shape)
 plotting.plot_glass_brain(first_rsn,display_mode='lyrz')
 print("-"*50 )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_pretrained>
%%time
warnings.filterwarnings("ignore", message="Expected column")
 df_model1 = df.copy()
df_model2 = df.copy()
df_model3 = df.copy()
NUM_FOLDS = 7
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 features = loading_features + fnc_features
blend_weights = {"age": [0.4, 0.55, 0.05],
"domain1_var1": [0.55, 0.15, 0.3],
"domain1_var2": [0.45, 0.0, 0.55],
"domain2_var1": [0.55, 0.15, 0.3],
"domain2_var2": [0.5, 0.05, 0.45]}
overall_score = 0
for target, c, w in [("age", 60, 0.3),("domain1_var1", 12, 0.175),("domain1_var2", 8, 0.175),("domain2_var1", 9, 0.175),("domain2_var2", 12, 0.175)]:
y_oof = np.zeros(df.shape[0])
 y_oof_model_1 = np.zeros(df.shape[0])
 y_oof_model_2 = np.zeros(df.shape[0])
 y_oof_model_3 = np.zeros(df.shape[0])
 y_test = np.zeros(( test_df.shape[0], NUM_FOLDS))
for f,(train_ind, val_ind)in enumerate(kf.split(df, df)) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
train_df = train_df[train_df[target].notnull() ]
model_1 = SVR(C=c, cache_size=3000.0)
 model_1.fit(train_df[features].values, train_df[target].values)
 model_2 = Ridge(alpha = 0.0001)
 model_2.fit(train_df[features].values, train_df[target].values)
 model_3 = BaggingRegressor(Ridge(alpha = 0.0001), n_estimators=30, random_state=42, max_samples=0.3, max_features=0.3)
 model_3.fit(train_df.to_pandas() [features].values, train_df.to_pandas() [target].values)
 val_pred_1 = model_1.predict(val_df[features])
 val_pred_2 = model_2.predict(val_df[features])
 val_pred_3 = model_3.predict(val_df.to_pandas() [features])
 val_pred_3 = cudf.from_pandas(pd.Series(val_pred_3))
test_pred_1 = model_1.predict(test_df[features])
 test_pred_2 = model_2.predict(test_df[features])
 test_pred_3 = model_3.predict(test_df.to_pandas() [features])
 test_pred_3 = cudf.from_pandas(pd.Series(test_pred_3))
val_pred = blend_weights[target][0]*val_pred_1+blend_weights[target][1]*val_pred_2+blend_weights[target][2]*val_pred_3
val_pred = cp.asnumpy(val_pred.values.flatten())
 test_pred = blend_weights[target][0]*test_pred_1+blend_weights[target][1]*test_pred_2+blend_weights[target][2]*test_pred_3
test_pred = cp.asnumpy(test_pred.values.flatten())
 y_oof[val_ind] = val_pred
y_oof_model_1[val_ind] = val_pred_1
y_oof_model_2[val_ind] = val_pred_2
y_oof_model_3[val_ind] = val_pred_3
y_test[:, f] = test_pred
df["pred_{}".format(target)] = y_oof
df_model1["pred_{}".format(target)] = y_oof_model_1
df_model2["pred_{}".format(target)] = y_oof_model_2
df_model3["pred_{}".format(target)] = y_oof_model_3
test_df[target] = y_test.mean(axis=1)
 score = metric(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values)
 overall_score += w*score
score_model1 = metric(df_model1[df_model1[target].notnull() ][target].values, df_model1[df_model1[target].notnull() ]["pred_{}".format(target)].values)
 score_model2 = metric(df_model2[df_model2[target].notnull() ][target].values, df_model2[df_model1[target].notnull() ]["pred_{}".format(target)].values)
 score_model3 = metric(df_model3[df_model3[target].notnull() ][target].values, df_model3[df_model1[target].notnull() ]["pred_{}".format(target)].values)
 print(f"For {target}:")
 print("SVR:", np.round(score_model1, 6))
print("Ridge:", np.round(score_model2, 6))
print("BaggingRegressor:", np.round(score_model3, 6))
print("Ensemble:", np.round(score, 6))
print()
print("Overall score:", np.round(overall_score, 6))
MODEL = 'joeddav/xlm-roberta-large-xnli'
EPOCHS = 4
MAXLEN = 120
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
tokenizer = AutoTokenizer.from_pretrained(MODEL)
 auto = tf.data.experimental.AUTOTUNE<choose_model_class>
sub_df = cudf.melt(test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]], id_vars=["Id"], value_name="Predicted")
 sub_df["Id"] = sub_df["Id"].astype("str")+ "_" + sub_df["variable"].astype("str")
 sub_df = sub_df.drop("variable", axis=1 ).sort_values("Id")
 assert sub_df.shape[0] == test_df.shape[0]*5
sub_df.head(10 )
def create_model(transformer_layer,learning_rate):
input_ids = Input(shape =(MAXLEN,), dtype = tf.int32)
 roberta = TFAutoModel.from_pretrained(transformer_layer)
 roberta = roberta([input_ids])[0]
out = GlobalAveragePooling1D()(roberta)
 out = Dense(3, activation = 'softmax' )(out)
 model = Model(inputs = input_ids, outputs = out)
 model.compile(
 optimizer = Adam(lr = learning_rate),
loss = 'sparse_categorical_crossentropy',
metrics = ['accuracy'])
 return model
def tokeniZer(text,tokenizer):
encoded = tokenizer.batch_encode_plus(text, padding=True, max_length=MAXLEN, truncation=True)
 return np.array(encoded['input_ids'])
 def tokeniZer_old(dataset,tokenizer):
text = dataset[['premise', 'hypothesis']].values.tolist()
encoded = tokenizer.batch_encode_plus(text, padding=True, max_length=MAXLEN, truncation=True)
 x = encoded['input_ids'], encoded['attention_mask']
y = None
if 'label' in dataset.columns:
y = dataset.label.values
return encoded['input_ids'], encoded['attention_mask'], y
def load_mnli(use_validation=True):
result = []
dataset = load_dataset(path='glue', name='mnli')
 print(dataset['train'])
 keys = ['train', 'validation_matched','validation_mismatched'] if use_validation else ['train']
for k in keys:
for record in dataset[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result.append(( c1,c2,c3,'en'))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
def load_snli(use_validation=True):
result = []
dataset = load_dataset(path='snli')
 keys = ['train', 'validation'] if use_validation else ['train']
for k in keys:
for record in dataset[k]:
c1, c2, c3 = record['premise'], record['hypothesis'], record['label']
if c1 and c2 and c3 in {0,1,2}:
result.append(( c1,c2,c3,'en'))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result
def load_xnli() :
result = []
dataset = load_dataset(path='xnli')
 for k in dataset.keys() :
for record in dataset[k]:
hp, pr, lb = record['hypothesis'], record['premise'], record['label']
if hp and pr and lb in {0,1,2}:
for lang, translation in zip(hp['language'], hp['translation']):
pr_lang = pr.get(lang, None)
 if pr_lang is None:
continue
result.append(( pr_lang, translation, lb,lang))
result = pd.DataFrame(result, columns=['premise','hypothesis','label','lang_abv'])
 return result<load_from_csv>
sub_df.to_csv("submission_rapids_ensemble_with_baggingregressor.csv", index=False )