Kaggle competition: TReNDS Neuroimaging
<SOS> metric: WMAE Kaggle data source: trends-neuroimaging<init_hyperparams>
%matplotlib inline
AugParams = {
'd1' : 100,
'd2': 160,
'rotate' : 45,
'ratio' : 0.4
}<prepare_x_and_y>
train_scores = pd.read_csv('/kaggle/input/trends-assessment-prediction/train_scores.csv' ).sort_values(by='Id')
 loadings = pd.read_csv('/kaggle/input/trends-assessment-prediction/loading.csv')
 fnc = pd.read_csv('/kaggle/input/trends-assessment-prediction/fnc.csv')
 sample = pd.read_csv('/kaggle/input/trends-assessment-prediction/sample_submission.csv')
 reveal = pd.read_csv('/kaggle/input/trends-assessment-prediction/reveal_ID_site2.csv')
 icn_nums = pd.read_csv('/kaggle/input/trends-assessment-prediction/ICN_numbers.csv')
def transform2(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform2(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, label):
mask = GridMask(IMAGE_SIZE[0],
IMAGE_SIZE[1],
AugParams['d1'],
AugParams['d2'],
AugParams['rotate'],
AugParams['ratio'])
 if IMAGE_SIZE[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 return tf.cast(image * tf.cast(mask, tf.float32), tf.float32), label<categorify>
import re
from tqdm import tqdm
def apply_all_aug(img, label, no_grid = False):
if tf.random.uniform([],0,1)>.5:
if tf.random.uniform([],0,1)>.5:
img, label = transform(img, label)
 else:
img = dropout(img, PROBABILITY = 1)
 else:
if not no_grid:
if tf.random.uniform([],0,1)>.5:
img, label = apply_grid_mask(img, label)
 else:
img, label = img, label
else:
if tf.random.uniform([],0,1)>.5:
img, label = transform(img, label)
 else:
img, label = img, label
return img, label<categorify>
r = re.compile('\d+')
 col_dict = {}
for col in fnc.columns:
ind = r.findall(col)
 if ind:
col_dict[col] = [int(i)for i in ind]
def onehot(image,label):
CLASSES = len(classes)
 return image,tf.one_hot(label,CLASSES )<categorify>
def get_matrix(df_row, return_idx=False):
matrix = np.zeros(( 100, 100))
for col in df_row.index[1:]:
i, j = col_dict[col]
matrix[i, j] = df_row[col]
matrix += matrix.T
idx = np.array([ 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 18, 20, 21, 23,
27, 32, 33, 37, 38, 40, 43, 45, 48, 51, 53, 54, 55, 56, 61, 62, 63,
66, 67, 68, 69, 70, 71, 72, 77, 79, 80, 81, 83, 84, 88, 93, 94, 96,
98, 99])
 if return_idx:
return matrix[:, idx][idx, :], idx
return matrix[:, idx][idx, :]
def mixup(image, label, PROBABILITY = 1.0, DIM = IMAGE_SIZE[0]):
CLASSES = 104
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 a = tf.random.uniform([],0,1)*P
img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2<categorify>
_, idx = get_matrix(fnc.iloc[0], return_idx=True)
 degrees = pd.DataFrame(degrees, columns=idx)
 degrees['Id'] = fnc['Id']
def cutmix(image, label, PROBABILITY = 1.0, DIM = IMAGE_SIZE[0]):
CLASSES = 104
imgs = []; labs = []
for j in range(AUG_BATCH):
P = tf.cast(tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([],0,AUG_BATCH),tf.int32)
 x = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 y = tf.cast(tf.random.uniform([],0,DIM),tf.int32)
 b = tf.random.uniform([],0,1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 if len(label.shape)==1:
lab1 = tf.one_hot(label[j],CLASSES)
 lab2 = tf.one_hot(label[k],CLASSES)
 else:
lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))
return image2,label2<categorify>
len(glob('/kaggle/input/trends-assessment-prediction/fMRI_train/*.mat')) , len(glob('/kaggle/input/trends-assessment-prediction/fMRI_test/*.mat'))
def cut_and_mix(image, label, DIM = IMAGE_SIZE[0]):
CLASSES = len(classes)
 SWITCH = 1/2
CUTMIX_PROB = 2/3
MIXUP_PROB = 2/3
image2, label2 = cutmix(image, label, CUTMIX_PROB)
 image3, label3 = mixup(image, label, MIXUP_PROB)
 imgs = []; labs = []
for j in range(BATCH_SIZE):
P = tf.cast(tf.random.uniform([],0,1)<=SWITCH, tf.float32)
 imgs.append(P*image2[j,]+(1-P)*image3[j,])
 labs.append(P*label2[j,]+(1-P)*label3[j,])
 image4 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
label4 = tf.reshape(tf.stack(labs),(BATCH_SIZE,CLASSES))
return image4,label4<install_modules>
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import Ridge, Lasso
from sklearn.svm import SVR
from sklearn.model_selection import GridSearchCV, KFold, cross_val_predict
!pip install -q efficientnet 
<choose_model_class>
train_ids = sorted(loadings[loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_ids = sorted(loadings[~loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_prediction = pd.DataFrame(test_ids, columns=['Id'], dtype=str)
 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 fnc_columns = fnc.columns[1:]
degrees_columns = degrees.columns[:-1]
data = pd.merge(loadings, train_scores, on='Id')
 data = pd.merge(data, fnc, on='Id')
 data = pd.merge(data, degrees, on='Id')
 X_test = pd.merge(loadings[loadings.Id.isin(test_ids)], fnc, on='Id')
 X_test = pd.merge(X_test, degrees, on='Id' ).drop('Id', axis=1 )
def get_DenseNet201() :
CLASSES = len(classes)
 with strategy.scope() :
dnet = DenseNet201(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 dnet.trainable = True
model = tf.keras.Sequential([
dnet,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_Xception() :
CLASSES = len(classes)
 with strategy.scope() :
xception = Xception(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 xception.trainable = True
model = tf.keras.Sequential([
xception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_InceptionV3() :
CLASSES = len(classes)
 with strategy.scope() :
inception = InceptionV3(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception.trainable = True
model = tf.keras.Sequential([
inception,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_InceptionResNetV2() :
CLASSES = len(classes)
 with strategy.scope() :
inception_res = InceptionResNetV2(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 inception_res.trainable = True
model = tf.keras.Sequential([
inception_res,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_EfficientNetB4(one_hot = False):
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB4(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 if one_hot:
model.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['categorical_accuracy'])
 else:
model.compile(optimizer='adam', loss = 'sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
 return model
def get_EfficientNetB5(one_hot = False):
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB5(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax', dtype = 'float32')
 ])
 if one_hot:
model.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['categorical_accuracy'])
 else:
model.compile(optimizer='adam', loss = 'sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
 return model
def get_EfficientNetB6() :
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB6(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'imagenet',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax',dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model
def get_EfficientNetB7() :
CLASSES = len(classes)
 with strategy.scope() :
efficient = efn.EfficientNetB7(
 input_shape =(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),
weights = 'noisy-student',
include_top = False 
)
 efficient.trainable = True
model = tf.keras.Sequential([
efficient,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(CLASSES, activation = 'softmax', dtype = 'float32')
 ])
 model.compile(
 optimizer='adam',
loss = 'categorical_crossentropy',
metrics=['categorical_accuracy'] 
)
 return model<init_hyperparams>
def MAPE(y_true, y_pred, **kwargs):

 return np.sum(np.abs(y_true - y_pred)) / y_true.sum()
mape_scorer = make_scorer(MAPE, greater_is_better=False )
histories = []
preds = np.zeros(( count_data_items(TEST_FILENAMES),104))
kfold = KFold(FOLDS, shuffle = True, random_state = SEED)
 for f,(train_index, val_index)in enumerate(kfold.split(TRAINING_FILENAMES)) :
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
train_ds = get_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[train_index]['TRAINING_FILENAMES']),
course_drop = False, mat_aug = False, grid_mask = False, all_aug = False, cutmixup = True,
labeled = True, return_image_names = True, repeat = True, shuffle = True)
 val_ds = get_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES} ).loc[val_index]['TRAINING_FILENAMES']),
course_drop = False, mat_aug = False, grid_mask = False, all_aug = False, cutmixup = False,
one_hot = True, labeled = True, return_image_names = False, repeat = False, shuffle = True)
 print('Getting model...'); print(''); print('Training model...')
 model = get_EfficientNetB5(one_hot = True)
 history = model.fit(train_ds, validation_data = val_ds, callbacks = [lr_callback],
verbose = VERBOSE, steps_per_epoch = STEPS_PER_EPOCH, epochs = EPOCHS 
)
 histories.append(history)
 print('Predicting test without TTA')
 ds_test = get_dataset(TEST_FILENAMES, course_drop = False, mat_aug = False, grid_mask = False, all_aug = False, cutmixup = False,
labeled = False, return_image_names = False, repeat = False, shuffle = False)
 pred = model.predict(ds_test,verbose = VERBOSE)
 preds += pred * 1 / FOLDS
del model; z = gc.collect()
if FIRST_FOLD_ONLY:
break<save_to_csv>
best_models = {}
total_score = []
for col in target_columns:
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
gs.fit(X_train, y_train)
 best_models[col] = gs.best_estimator_
y_pred = cross_val_predict(gs.best_estimator_, X_train, y_train, cv=cv, n_jobs=-1)
 total_score.append(MAPE(y_train, y_pred))
print(col, MAPE(y_train, y_pred))
total_score = np.array(total_score)
 print(f'Total score: {np.sum(total_score*[.3,.175,.175,.175,.175])}' )
test_ds = get_dataset(TEST_FILENAMES, labeled = False, repeat = False, return_image_names = True)
 test_images_ds = test_ds.map(lambda image, idnum: image)
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 submission = pd.DataFrame()
submission['id'] = test_ids
submission['label'] = preds.argmax(axis = 1)
 submission.to_csv('submission.csv', index = False)
 submission.head(10 )<feature_engineering>
def get_pred(col, model):
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
y_pred = cross_val_predict(model, X_train, y_train, cv=cv, n_jobs=-1)
 return y_pred
os.environ["WANDB_API_KEY"] = "0"<load_from_csv>
X_test[fnc_columns] /= 500
for col in target_columns:
test_prediction[col] = best_models[col].predict(X_test )
df_train = pd.read_csv('.. /input/contradictory-my-dear-watson/train.csv')
 df_test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv' )<count_missing_values>
df_train.isnull().sum()<count_missing_values>
def make_sub(test_prediction):

 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 _columns =(0,1,2,3,4)
 tst = test_prediction.rename(columns=dict(zip(target_columns, _columns)))
 tst = tst.melt(id_vars='Id',
value_vars=_columns,
value_name='Predicted')
 tst['target_type'] = tst.variable.map(dict(zip(_columns, target_columns)))
 tst['Id_'] = tst[['Id', 'target_type']].apply(lambda x: '_'.join(( str(x[0]), str(x[1]))), axis=1)
 return tst.sort_values(by=['Id', 'variable'])\ 
.drop(['Id', 'variable', 'target_type'],axis=1)\ 
.rename(columns={'Id_':'Id'})\ 
.reset_index(drop=True)\
[['Id', 'Predicted']]
df_test.isnull().sum()<categorify>
sub = make_sub(test_prediction)
 sub.head()
def encode_sentence(s):
tokens = list(tokenizer.tokenize(s))
tokens.append('[SEP]')
 return tokenizer.convert_tokens_to_ids(tokens )<categorify>
sub.to_csv('ridge_mape_500.csv', index=False )
def bert_encode(premise, hypothesis, tokenizer):
sentence1 = tf.ragged.constant([encode_sentence(s)for s in np.array(premise)])
 sentence2 = tf.ragged.constant([encode_sentence(s)for s in np.array(hypothesis)])
 cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*(sentence1.shape[0])
 input_word_ids = tf.concat([cls, sentence1, sentence2], axis=-1)
 input_masks = tf.ones_like(input_word_ids ).to_tensor()
type_cls = tf.zeros_like(cls)
 type_s1 = tf.zeros_like(sentence1)
 type_s2 = tf.ones_like(sentence2)
 input_type_ids = tf.concat([type_cls, type_s1, type_s2], axis=-1 ).to_tensor()
inputs = {
'input_word_ids': input_word_ids.to_tensor() ,
'input_masks': input_masks,
'input_type_ids': input_type_ids}
return inputs<categorify>
%matplotlib inline
tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
 train_input = bert_encode(df_train.premise.values, df_train.hypothesis.values, tokenizer)
<set_options>
train_scores = pd.read_csv('/kaggle/input/trends-assessment-prediction/train_scores.csv' ).sort_values(by='Id')
 loadings = pd.read_csv('/kaggle/input/trends-assessment-prediction/loading.csv')
 fnc = pd.read_csv('/kaggle/input/trends-assessment-prediction/fnc.csv')
 sample = pd.read_csv('/kaggle/input/trends-assessment-prediction/sample_submission.csv')
 reveal = pd.read_csv('/kaggle/input/trends-assessment-prediction/reveal_ID_site2.csv')
 icn_nums = pd.read_csv('/kaggle/input/trends-assessment-prediction/ICN_numbers.csv')
K.clear_session()<choose_model_class>
import re
from tqdm import tqdm
def build_bert_baseline(max_len=80):
bert_layer = TFBertModel.from_pretrained('bert-base-multilingual-cased')
 input_word_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_word_ids')
 input_masks = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_masks')
 input_type_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_type_ids')
 sequence_output = bert_layer([input_word_ids, input_masks, input_type_ids])[0]
output = tf.keras.layers.Dense(3, activation='softmax' )(sequence_output[:,0,:])
 model = tf.keras.Model(inputs=[input_word_ids,input_masks,input_type_ids],outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=3e-5),
loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 print(model.summary())
 return model<train_model>
r = re.compile('\d+')
 col_dict = {}
for col in fnc.columns:
ind = r.findall(col)
 if ind:
col_dict[col] = [int(i)for i in ind]
model = build_bert_baseline(max_len=80)
 filepath='best_weight.hdf5'
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
<set_options>
def get_matrix(df_row, return_idx=False):
matrix = np.zeros(( 100, 100))
for col in df_row.index[1:]:
i, j = col_dict[col]
matrix[i, j] = df_row[col]
matrix += matrix.T
idx = np.array([ 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 18, 20, 21, 23,
27, 32, 33, 37, 38, 40, 43, 45, 48, 51, 53, 54, 55, 56, 61, 62, 63,
66, 67, 68, 69, 70, 71, 72, 77, 79, 80, 81, 83, 84, 88, 93, 94, 96,
98, 99])
 if return_idx:
return matrix[:, idx][idx, :], idx
return matrix[:, idx][idx, :]
K.clear_session()<load_pretrained>
_, idx = get_matrix(fnc.iloc[0], return_idx=True)
 degrees = pd.DataFrame(degrees, columns=idx)
 degrees['Id'] = fnc['Id']
def build_bert_LSTM_head(max_len=80):
bert_layer = TFBertModel.from_pretrained('bert-base-multilingual-cased')
 input_word_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_word_ids')
 input_masks = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_masks')
 input_type_ids = tf.keras.Input(shape =(max_len,),dtype =tf.int32, name='input_type_ids')
 sequence_output = bert_layer([input_word_ids, input_masks, input_type_ids])[0]
bi_lstm = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True))(sequence_output)
 avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)
 max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)
 concat = tf.keras.layers.concatenate([avg_pool, max_pool])
 dropout = tf.keras.layers.Dropout(0.3 )(concat)
 output = tf.keras.layers.Dense(3, activation="softmax" )(dropout)
 model = tf.keras.Model(inputs=[input_word_ids,input_masks,input_type_ids],outputs=output)
 model.compile(tf.keras.optimizers.Adam(lr=3e-5),
loss='sparse_categorical_crossentropy',metrics=['accuracy'])
 print(model.summary())
 return model<train_model>
len(glob('/kaggle/input/trends-assessment-prediction/fMRI_train/*.mat')) , len(glob('/kaggle/input/trends-assessment-prediction/fMRI_test/*.mat'))
bert_lstm_model = build_bert_LSTM_head(max_len=80)
 filepath='LSTM_best_weight.hdf5'
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True,
mode='min',save_freq = 'epoch')
<set_options>
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import Ridge, Lasso
from sklearn.svm import SVR
from sklearn.model_selection import GridSearchCV, KFold, cross_val_predict
K.clear_session()<set_options>
train_ids = sorted(loadings[loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_ids = sorted(loadings[~loadings['Id'].isin(train_scores.Id)]['Id'].values)
 test_prediction = pd.DataFrame(test_ids, columns=['Id'], dtype=str)
 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 fnc_columns = fnc.columns[1:]
degrees_columns = degrees.columns[:-1]
data = pd.merge(loadings, train_scores, on='Id')
 data = pd.merge(data, fnc, on='Id')
 data = pd.merge(data, degrees, on='Id')
 X_test = pd.merge(loadings[loadings.Id.isin(test_ids)], fnc, on='Id')
 X_test = pd.merge(X_test, degrees, on='Id' ).drop('Id', axis=1 )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<categorify>
def MAPE(y_true, y_pred, **kwargs):

 return np.sum(np.abs(y_true - y_pred)) / y_true.sum()
mape_scorer = make_scorer(MAPE, greater_is_better=False )
def roberta_encode(text,tokenizer,max_len=80):
encoded_text = tokenizer.batch_encode_plus(text,
pad_to_max_length=True,
max_length=max_len)
 return encoded_text<load_pretrained>
best_models = {}
total_score = []
for col in target_columns:
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
gs.fit(X_train, y_train)
 best_models[col] = gs.best_estimator_
y_pred = cross_val_predict(gs.best_estimator_, X_train, y_train, cv=cv, n_jobs=-1)
 total_score.append(MAPE(y_train, y_pred))
print(col, MAPE(y_train, y_pred))
total_score = np.array(total_score)
 print(f'Total score: {np.sum(total_score*[.3,.175,.175,.175,.175])}' )
tokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')
 train_text = df_train[['premise', 'hypothesis']].values.tolist()
encoded_train_text = roberta_encode(train_text,tokenizer,max_len=80 )<split>
def get_pred(col, model):
X_train = data.dropna(subset=[col], axis=0 ).drop(list(target_columns), axis=1 ).drop('Id', axis=1)
 X_train[fnc_columns] /= 500
y_train = data.dropna(subset=[col], axis=0)[col]
y_pred = cross_val_predict(model, X_train, y_train, cv=cv, n_jobs=-1)
 return y_pred
x_train, x_valid, y_train, y_valid = train_test_split(
 encoded_train_text['input_ids'],
df_train.label.values,
test_size=0.2, random_state=98 )<define_variables>
X_test[fnc_columns] /= 500
for col in target_columns:
test_prediction[col] = best_models[col].predict(X_test )
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
AUTO = tf.data.experimental.AUTOTUNE<create_dataframe>
def create_tf_dataset(X, y,val,batch_size= BATCH_SIZE,auto=AUTO):
dataset = tf.data.Dataset.from_tensor_slices(( X,y)).shuffle(98)
 if not val:
dataset = dataset.repeat().batch(batch_size ).prefetch(auto)
 else:
dataset = dataset.batch(batch_size ).prefetch(auto)
 return dataset<create_dataframe>
def make_sub(test_prediction):

 target_columns =('age', 'domain1_var1', 'domain1_var2','domain2_var1','domain2_var2')
 _columns =(0,1,2,3,4)
 tst = test_prediction.rename(columns=dict(zip(target_columns, _columns)))
 tst = tst.melt(id_vars='Id',
value_vars=_columns,
value_name='Predicted')
 tst['target_type'] = tst.variable.map(dict(zip(_columns, target_columns)))
 tst['Id_'] = tst[['Id', 'target_type']].apply(lambda x: '_'.join(( str(x[0]), str(x[1]))), axis=1)
 return tst.sort_values(by=['Id', 'variable'])\ 
.drop(['Id', 'variable', 'target_type'],axis=1)\ 
.rename(columns={'Id_':'Id'})\ 
.reset_index(drop=True)\
[['Id', 'Predicted']]
tf_train_dataset = create_tf_dataset(x_train,y_train,val=False)
 tf_val_dataset = create_tf_dataset(x_valid,y_valid,val=True )<train_model>
sub = make_sub(test_prediction)
 sub.head()
<choose_model_class><EOS> 
sub.to_csv('ridge_mape_500.csv', index=False )