Kaggle competition: TReNDS Neuroimaging
<SOS> metric: WMAE Kaggle data source: trends-neuroimaging<split>
import numpy as np
import pandas as pd
import os
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded_bert['input_ids'], train.label.values,
test_size=0.2, random_state=2020 
)
 x_test = test_encoded_bert['input_ids']<split>
loading = pd.read_csv('/kaggle/input/trends-assessment-prediction/loading.csv')
 features = pd.read_csv('/kaggle/input/trends-assessment-prediction/train_scores.csv')
 submission = pd.read_csv('/kaggle/input/trends-assessment-prediction/sample_submission.csv')
 fnc = pd.read_csv("/kaggle/input/trends-assessment-prediction/fnc.csv")
 score = pd.read_csv("/kaggle/input/trends-assessment-prediction/train_scores.csv" )
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
 )<categorify>
df = fnc.merge(loading, on="Id")
 score["is_train"] = True
df = df.merge(score, on="Id", how="left")
 test_df = df[df["is_train"] != True].copy()
df = df[df["is_train"] == True].copy()
with strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(model_bert)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer_encoder(input_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()<train_model>
FNC_SCALE = 1/400
df[fnc_features] *= FNC_SCALE
test_df[fnc_features] *= FNC_SCALE
n_steps = len(x_train)// batch_size
train_history_bert = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)<load_pretrained>
features = loading_features + fnc_features
tokenizer_xml_roberta = AutoTokenizer.from_pretrained(model_xml_roberta )<categorify>
def metric(y_true, y_pred):
return np.mean(np.sum(np.abs(y_true - y_pred), axis=0)/np.sum(y_true, axis=0))
train_text_xml_roberta = train[['premise', 'hypothesis']].values.tolist()
test_text_xml_roberta = test[['premise', 'hypothesis']].values.tolist()
train_encoded_xml_roberta = tokenizer_xml_roberta.batch_encode_plus(
 train_text_xml_roberta,
pad_to_max_length=True,
max_length=max_len 
)
 test_encoded_xml_roberta = tokenizer_xml_roberta.batch_encode_plus(
 test_text_xml_roberta,
pad_to_max_length=True,
max_length=max_len
)<split>
import sklearn
from sklearn.svm import SVR
x_train, x_valid, y_train, y_valid = train_test_split(
 train_encoded_xml_roberta['input_ids'], train.label.values,
test_size=0.2, random_state=2020 
)
 x_test = test_encoded_xml_roberta['input_ids']<split>
%%time
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 features = loading_features + fnc_features
overal_score = 0
for target, c, w in [("age", 50, 0.3),("domain1_var1", 5, 0.175),("domain1_var2", 5, 0.175),("domain2_var1", 5, 0.175),("domain2_var2", 5, 0.175)]:
y_oof = np.zeros(df.shape[0])
 y_test = np.zeros(( test_df.shape[0], NUM_FOLDS))
for f,(train_ind, val_ind)in enumerate(kf.split(df, df)) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
train_df = train_df[train_df[target].notnull() ]
model = SVR(C=c, cache_size=3000.0)
 model.fit(train_df[features], train_df[target])
 y_oof[val_ind] = model.predict(val_df[features])
 y_test[:, f] = model.predict(test_df[features])
 df["pred_{}".format(target)] = y_oof
test_df[target] = y_test.mean(axis=1)
 score = metric(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values)
 overal_score += w*score
print(target, np.round(score, 4))
print()
print("Overal score:", np.round(overal_score, 4))
auto = tf.data.experimental.AUTOTUNE
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(batch_size)
.prefetch(auto)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(batch_size)
.cache() 
.prefetch(auto)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(batch_size)
 )<categorify>
sub_df4 = pd.melt(test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]], id_vars=["Id"], value_name="Predicted")
 sub_df4["Id"] = sub_df4["Id"].astype("str")+ "_" + sub_df4["variable"].astype("str")
 sub_df4 = sub_df4.drop("variable", axis=1 ).sort_values("Id")
with strategy.scope() :
transformer_encoder = TFAutoModel.from_pretrained(model_xml_roberta)
 input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 sequence_output = transformer_encoder(input_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(3, activation='softmax' )(cls_token)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()<train_model>
sub_df4.to_csv("submission50.csv", index=False, float_format='%.6f' )
n_steps = len(x_train)// batch_size
train_history_xml_roberta = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=n_epochs
)<load_pretrained>
%%time
NUM_FOLDS = 10
kf = KFold(n_splits=NUM_FOLDS, shuffle=True, random_state=0)
 features = loading_features + fnc_features
overal_score = 0
for target, c, w in [("age", 80, 0.3),("domain1_var1", 8, 0.175),("domain1_var2", 8, 0.175),("domain2_var1", 8, 0.175),("domain2_var2", 8, 0.175)]:
y_oof = np.zeros(df.shape[0])
 y_test = np.zeros(( test_df.shape[0], NUM_FOLDS))
for f,(train_ind, val_ind)in enumerate(kf.split(df, df)) :
train_df, val_df = df.iloc[train_ind], df.iloc[val_ind]
train_df = train_df[train_df[target].notnull() ]
model = SVR(C=c, cache_size=3000.0)
 model.fit(train_df[features], train_df[target])
 y_oof[val_ind] = model.predict(val_df[features])
 y_test[:, f] = model.predict(test_df[features])
 df["pred_{}".format(target)] = y_oof
test_df[target] = y_test.mean(axis=1)
 score = metric(df[df[target].notnull() ][target].values, df[df[target].notnull() ]["pred_{}".format(target)].values)
 overal_score += w*score
print(target, np.round(score, 4))
print()
print("Overal score:", np.round(overal_score, 4))
response = requests.get('https://miro.medium.com/max/398/1*BfvKeP4ykqi4J4C5g4EZzg.png')
 image_bytes = io.BytesIO(response.content)
 img = PIL.Image.open(image_bytes)
 img<create_dataframe>
sub_df4 = pd.melt(test_df[["Id", "age", "domain1_var1", "domain1_var2", "domain2_var1", "domain2_var2"]], id_vars=["Id"], value_name="Predicted")
 sub_df4["Id"] = sub_df4["Id"].astype("str")+ "_" + sub_df4["variable"].astype("str")
 sub_df4 = sub_df4.drop("variable", axis=1 ).sort_values("Id")
<create_dataframe><EOS> 
sub_df4.to_csv("submission80.csv", index=False, float_format='%.6f' )