Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<define_variables>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
kfold = KFold(5, shuffle = True, random_state = 42)
 scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 for i,(tr_idx, va_idx)in enumerate(kfold.split(X_node, As)) :
print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("./base_ae")
 model = get_model(base, config)
 if pretrain_dir is not None:
d = f"./model{i}"
print(f"--- load from {d} ---")
 model.load_weights(d)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va, As_va], [y_va]),
epochs = epochs,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"./model{i}")
 p = model.predict([X_node_va, As_va])
 scores.append(mcrmse(y_va, p))
print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
if one_fold:
break
pd.to_pickle(preds, "oof.pkl" )<predict_on_test>
def extract_title(name):
name = name.split()
for w in name:
if '.' in w:
return w
return None
all_titles = pd.concat([train, test], sort=False ).Name.apply(lambda name: extract_title(name))
all_titles.unique()
p_pub = 0
p_pri = 0
for i in range(5):
model.load_weights(f"./model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
if one_fold:
p_pub *= 5
p_pri *= 5
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
train['title'] = train.Name.apply(lambda name: extract_title(name))
test['title'] = test.Name.apply(lambda name: extract_title(name))
train.groupby('title' ).Survived.agg(['count', 'mean'] )
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.to_csv("submission.csv", index = False)
 preds_df.head()<set_options>
train.groupby('title' ).Survived.agg(['count', 'mean'] )
%matplotlib inline
warnings.filterwarnings('ignore' )<load_from_csv>
title_encoder = LabelEncoder().fit(train.title)
 train.title = title_encoder.transform(train.title)
 test.title = title_encoder.transform(test.title )
train = pd.read_json(".. /input/stanford-covid-vaccine/train.json",lines=True)
 test = pd.read_json(".. /input/stanford-covid-vaccine/test.json",lines=True)
 sub = pd.read_csv(".. /input/stanford-covid-vaccine/sample_submission.csv")
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]<load_from_csv>
sex_encoder = LabelEncoder().fit(train.Sex)
 train.Sex = sex_encoder.transform(train.Sex)
 test.Sex = sex_encoder.transform(test.Sex )
aug_df = pd.read_csv('.. /input/covid19-mrna-augmentation-data-and-features/aug_data1.csv')
 aug_df = aug_df.drop_duplicates(subset=['id', 'structure'])
 def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train ).reset_index(drop=True)
 test_pub = aug_data(test_pub ).reset_index(drop=True)
 test_pri = aug_data(test_pri ).reset_index(drop=True )<data_type_conversions>
age_translator = pd.concat([train, test], sort=False ).groupby('title' ).Age.median().to_dict()
train['completeAge'] = train.apply(lambda x: age_translator[x.title], axis=1)
 train.loc[train.Age.notnull() , 'completeAge'] = train.loc[train.Age.notnull() , 'Age']
test['completeAge'] = test.apply(lambda x: age_translator[x.title], axis=1)
 test.loc[test.Age.notna() , 'completeAge'] = test.loc[test.Age.notna() , 'Age']
As = []
for id in tqdm(train["id"]):
a = np.load(f".. /input/stanford-covid-vaccine/bpps/{id}.npy" ).astype(np.float16)
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f".. /input/stanford-covid-vaccine/bpps/{id}.npy" ).astype(np.float16)
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f".. /input/stanford-covid-vaccine/bpps/{id}.npy" ).astype(np.float16)
 As_pri.append(a)
 As_pri = np.array(As_pri)
 As_cf = []
for id in tqdm(train["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_contrafold_gamma1/bpps_contrafold_gamma1/bpps_contrafold_gamma1/{id}.npy" ).astype(np.float16)
 As_cf.append(a)
 As_cf = np.array(As_cf)
 As_pub_cf = []
for id in tqdm(test_pub["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_contrafold_gamma1/bpps_contrafold_gamma1/bpps_contrafold_gamma1/{id}.npy" ).astype(np.float16)
 As_pub_cf.append(a)
 As_pub_cf = np.array(As_pub_cf)
 As_pri_cf = []
for id in tqdm(test_pri["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_contrafold_gamma1/bpps_contrafold_gamma1/bpps_contrafold_gamma1/{id}.npy" ).astype(np.float16)
 As_pri_cf.append(a)
 As_pri_cf = np.array(As_pri_cf)
 As_rs = []
for id in tqdm(train["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/{id}.npy" ).astype(np.float16)
 As_rs.append(a)
 As_rs = np.array(As_rs)
 As_pub_rs = []
for id in tqdm(test_pub["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/{id}.npy" ).astype(np.float16)
 As_pub_rs.append(a)
 As_pub_rs = np.array(As_pub_rs)
 As_pri_rs = []
for id in tqdm(test_pri["id"]):
a = np.load(f".. /input/covid19-mrna-augmentation-data-and-features/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/bpps_rnasoft_gamma1/{id}.npy" ).astype(np.float16)
 As_pri_rs.append(a)
 As_pri_rs = np.array(As_pri_rs )<categorify>
train['familysize'] = train.Parch + train.SibSp + 1
test['familysize'] = test.Parch + test.SibSp + 1
train.drop(['Parch', 'SibSp'], inplace=True, axis=1)
 train['hasfamily'] = 0
train.loc[train.familysize == 1, 'hasfamily'] = 1
test['hasfamily'] = 0
test.loc[test.familysize == 1, 'hasfamily'] = 1
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for i in range(seq_length):
if structure[i] == "(":
cue.append(i)
 elif structure[i] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[i])][start, i] = 1
a_structures[(sequence[i], sequence[start])][i, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 return Ss.astype(np.float16)
 Ss = get_structure_adj(train)
 Ss_pub = get_structure_adj(test_pub)
 Ss_pri = get_structure_adj(test_pri )<prepare_x_and_y>
train['ticketno'] = train.Ticket.apply(lambda x: x.split() [-1])
 train.ticketno.replace('LINE', "0", inplace=True)
 train.ticketno = train['ticketno'].astype(np.int)
 test['ticketno'] = test.Ticket.apply(lambda x: x.split() [-1])
 test.ticketno = test['ticketno'].astype(np.int )
def get_distance_matrix(As):
idx = np.arange(As.shape[1])
 Ds = []
for i in range(len(idx)) :
d = np.abs(idx[i] - idx)
 Ds.append(d)
 Ds = np.array(Ds)+ 1
Ds = Ds / Ds.shape[1]
Ds = Ds[None, :,:]
Ds = np.repeat(Ds, len(As), axis = 0)
 Dss = []
for i in [1]:
Dss.append(Ds ** i)
 Ds = np.stack(Dss, axis = 3)
 return Ds.astype(np.float16)
 Ds = get_distance_matrix(As)
 Ds_pub = get_distance_matrix(As_pub)
 Ds_pri = get_distance_matrix(As_pri )<data_type_conversions>
train['decklevel'] = train.Cabin.dropna().apply(lambda x: str(x)[0])
 test['decklevel'] = test.Cabin.dropna().apply(lambda x: str(x)[0])
 decklevel_encoder = LabelEncoder().fit(train.decklevel.dropna())
 train.loc[train.decklevel.notna() , 'decklevel'] = decklevel_encoder.transform(train.decklevel.dropna())
 test.loc[test.decklevel.notna() , 'decklevel'] = decklevel_encoder.transform(test.decklevel.dropna() )
As = np.concatenate([As[:,:,:,None], As_cf[:,:,:,None], As_rs[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float16)
 del Ss, Ds, As_cf, As_rs
As_pub = np.concatenate([As_pub[:,:,:,None], As_pub_cf[:,:,:,None], As_pub_rs[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float16)
 del Ss_pub, Ds_pub, As_pub_cf, As_pub_rs
As_pri = np.concatenate([As_pri[:,:,:,None], As_pri_cf[:,:,:,None], As_pri_rs[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float16)
 del Ss_pri, Ds_pri, As_pri_cf, As_pri_rs
print(As.shape, As_pub.shape, As_pri.shape )<data_type_conversions>
train.Embarked.fillna('S', inplace=True)
 embarked_encoder = LabelEncoder().fit(train.Embarked)
 train.Embarked = embarked_encoder.transform(train.Embarked)
 test.Embarked = embarked_encoder.transform(test.Embarked )
f, ax = plt.subplots(1, 5, figsize=(15, 3))
for i in range(As.shape[-1]):
ax[i].imshow(As[0, :, :, i].astype(np.float32))
plt.show()<load_pretrained>
test.loc[test.Fare.isna() , 'Fare'] = train.Fare.median()
arnie_train_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/arnie/arnie/train_features.p', 'rb'))
arnie_test_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/arnie/arnie/test_features.p', 'rb'))
capr_train_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/CapR_train.pickle', 'rb'))
capr_test_features = pickle.load(open('.. /input/covid19-mrna-augmentation-data-and-features/CapR_test.pickle', 'rb'))<categorify>
featurelist = ['Pclass', 'Fare', 'title', 'Sex', 'Embarked', 'rich_small_families', 'completeAge', 'hasfamily']
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_pos1 = np.stack(train["sequence"].apply(lambda x : np.arange(len(x)) % 3 == 0)) [:,:,np.newaxis]
X_pos2 = np.stack(train["sequence"].apply(lambda x : np.arange(len(x)) % 3 == 1)) [:,:,np.newaxis]
X_pos3 = np.stack(train["sequence"].apply(lambda x : np.arange(len(x)) % 3 == 2)) [:,:,np.newaxis]
X_certainty = np.zeros(( X_pos1.shape[:2] +(3,)))
 X_loop_capr = np.zeros(( X_pos1.shape[:2] +(6,)))
 X_bpp_sum = np.zeros(( X_pos1.shape[:2] +(1,)))
 X_bpp_nb_zeroes = np.zeros(( X_pos1.shape[:2] +(1,)))
 for k, mol_id in enumerate(train['id'].values):
if mol_id in arnie_train_features:
arr1 = np.array(list(arnie_train_features[mol_id]['mfe']['vienna_2']))
arr2 = np.array(list(arnie_train_features[mol_id]['mfe']['contrafold_2']))
arr3 = np.array(list(arnie_train_features[mol_id]['mfe']['rnastructure']))
else:
arr1 = np.array(list(arnie_test_features[mol_id]['mfe']['vienna_2']))
arr2 = np.array(list(arnie_test_features[mol_id]['mfe']['contrafold_2']))
arr3 = np.array(list(arnie_test_features[mol_id]['mfe']['rnastructure']))
X_certainty[k, :, 0] =(( arr1 == '(')+(arr2 == '(')+(arr3 == '(')).astype(int)/ 3
X_certainty[k, :, 1] =(( arr1 == ')')+(arr2 == ')')+(arr3 == ')')).astype(int)/ 3
X_certainty[k, :, 2] =(( arr1 == '.')+(arr2 == '.')+(arr3 == '.')).astype(int)/ 3
if mol_id in capr_train_features:
X_loop_capr[k] = np.array([list(map(float, i.split() [1:])) for i in capr_train_features[mol_id].split('
')[1:-2]] ).T
else:
X_loop_capr[k] = np.array([list(map(float, i.split() [1:])) for i in capr_test_features[mol_id].split('
')[1:-2]] ).T
bpp = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).astype(np.float16)
 X_bpp_sum[k] = bpp.sum(axis=1)[:, None]
X_bpp_nb_zeroes[k] =(( bpp == 0 ).sum(axis=1)/ X_pos1.shape[1])[:, None]
X_node = np.concatenate([X_node, X_loop, X_pos1, X_pos2, X_pos3, X_certainty, X_loop_capr, X_bpp_sum, X_bpp_nb_zeroes], axis = 2)
 return X_node
X_node = get_input(train ).astype(np.float16)
 X_node_pub = get_input(test_pub ).astype(np.float16)
 X_node_pri = get_input(test_pri ).astype(np.float16 )<drop_column>
parameters = {'n_estimators': [40, 50, 60, 70],
'max_depth': [7, 10, 13],
'max_features': range(3, len(featurelist)) ,
'min_samples_leaf': [1, 2, 3],
'min_samples_split': [7, 10, 12]}
model = RandomForestClassifier(random_state=42)
 grid = GridSearchCV(model, param_grid=parameters, cv=5, n_jobs=-1, scoring='accuracy')
 grid.fit(train[featurelist], train.Survived )
del arnie_train_features, arnie_test_features, capr_train_features, capr_test_features<concatenate>
grid.best_params_
targets = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 print(y.shape )<define_variables>
grid.best_score_
N_NODE_FEATURES = X_node.shape[2]
N_EDGE_FEATURES = As.shape[3]<import_modules>
print(classification_report(grid.predict(train[featurelist]), train['Survived']))
import tensorflow as tf
from tensorflow.keras import layers as L
import tensorflow_addons as tfa
from tensorflow.keras import backend as K
from sklearn.utils import shuffle
from sklearn.model_selection import GroupKFold<define_search_space>
result = test
result.loc[:,'Survived'] = grid.predict(test[featurelist])
 result.head()
<compute_test_metric><EOS> 
result[['PassengerId', 'Survived']].to_csv('submission.csv', header=True, index=False )