Kaggle competition: Mechanisms of Action (MoA) Prediction
predictions = model.predict(test_data )<save_to_csv>
warnings.filterwarnings("ignore" )
output = pd.DataFrame({'PassengerId': test_data.index, 'Survived': predictions})
 output.to_csv('my_submission34.csv', index=False )<feature_engineering>
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler
from tensorflow.keras.losses import BinaryCrossentropy
import tensorflow_addons as tfa
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
from sklearn import preprocessing
from tqdm.notebook import tqdm
import math
<import_modules>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 data = train_features.append(test_features)
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
import numpy as np
import pandas as pd<load_from_csv>
def preprocess(df):
df = df.copy()
df.drop(['cp_type'], axis=1, inplace=True)
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 df = pd.get_dummies(df, columns=['cp_time','cp_dose'])
 del df['sig_id']
return df
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
train_data = pd.read_csv('.. /input/titanic/train.csv')
 test_data = pd.read_csv('.. /input/titanic/test.csv')
 test_ids = test_data['PassengerId']
data = pd.concat([train_data, test_data], axis=0)
 data.head()<drop_column>
scaler = preprocessing.MinMaxScaler()
scaler.fit(train.append(test))
train_trans = scaler.transform(train)
 test_trans = scaler.transform(test)
 train = pd.DataFrame(train_trans, columns=train.columns)
 test = pd.DataFrame(test_trans, columns=test.columns )
data = data.drop(['Cabin', 'Ticket'], axis=1 )<categorify>
somthing_rate = 1e-15
P_MIN = somthing_rate
P_MAX = 1 - P_MIN
def loss_fn(yt, yp):
yp = np.clip(yp, P_MIN, P_MAX)
 return log_loss(yt, yp, labels=[0,1])
data['Title'] = [value.split(', ')[1].split('.')[0] for value in data['Name'].values]
data.loc[(data['Title'] == 'Lady')|(data['Title'] == 'Mme')|(data['Title'] == 'Ms')|(data['Title'] == 'the Countess')|(data['Title'] == 'Mlle'), 'Title'] = 'Miss'
data.loc[(data['Title'] != 'Mr')&(data['Title'] != 'Mrs')&(data['Title'] != 'Miss')&(data['Title'] != 'Master'), 'Title'] = 'Other'
encoder = LabelEncoder()
data['Surname'] = encoder.fit_transform([value.split(', ')[0] for value in data['Name'].values] )<feature_engineering>
def create_model(num_columns, actv='relu'):
model = tf.keras.Sequential([tf.keras.layers.Input(num_columns)])
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation=actv)))
 if actv == 'elu':
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.AlphaDropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, kernel_initializer='lecun_normal', activation='selu')))
 else:
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.2))
model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation=actv)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid")))
 model.compile(optimizer=tfa.optimizers.AdamW(lr = 1e-3, weight_decay = 1e-5, clipvalue = 756),
loss=BinaryCrossentropy(label_smoothing=somthing_rate), 
)
 return model
data['Alone'] = np.zeros(data.shape[0], dtype=np.int)
 data.loc[(data['SibSp'] == 0)&(data['Parch'] == 0), 'Alone'] = 1
data['FamilyCount'] = data['SibSp'] + data['Parch']
data.head()<feature_engineering>
def metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(loss_fn(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float)))
 return np.mean(metrics )
np.random.seed(seed=157)
 mean_ages_title = data[['Title', 'Age']].loc[data['Age'].notna() ].groupby('Title' ).mean()
age_nan_titles = data['Title'].loc[data['Age'].isnull() ].unique()
for i in range(len(age_nan_titles)) :
if age_nan_titles[i] == 'Mr' or age_nan_titles[i] == 'Mrs' or age_nan_titles[i] == 'Miss':
data.loc[(data['Age'].isnull())&(data['Title'] == age_nan_titles[i]), 'Age'] = [np.ceil(mean_ages_title.loc[age_nan_titles[i]].values[0])+ np.random.randint(-6, 6)
 for _ in data['Age'].loc[(data['Age'].isnull())&(data['Title'] == age_nan_titles[i])]]
else:
data.loc[(data['Age'].isnull())&(data['Title'] == age_nan_titles[i]), 'Age'] = [np.ceil(mean_ages_title.loc[age_nan_titles[i]].values[0])
 for _ in data['Age'].loc[(data['Age'].isnull())&(data['Title'] == age_nan_titles[i])]]<feature_engineering>
N_STARTS = 14
S_STARTS = int(N_STARTS/2)
 res_relu = train_targets.copy()
res_elu = train_targets.copy()
res_relu.loc[:, train_targets.columns] = 0
res_elu.loc[:, train_targets.columns] = 0
ss_relu = ss.copy()
ss_elu = ss.copy()
ss_relu.loc[:, train_targets.columns] = 0
ss_elu.loc[:, train_targets.columns] = 0
ss_dict = {}
historys = dict()
tf.random.set_seed(42)
 for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=7, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f"======{train_targets.values[tr].shape}========{train_targets.values[te].shape}=====")
 if seed < S_STARTS:
print(f'Seed: {seed} => Fold: {n} ==>(RELU MODEL)')
 model = create_model(len(top_feats), actv='relu')
 else:
print(f'Seed: {seed} => Fold: {n} ==>(ELU MODEL)')
 model = create_model(len(top_feats), actv='elu')
 checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.2, min_lr=1e-6, patience=4, verbose=1, mode='auto')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 1, save_best_only = True,
save_weights_only = True, mode = 'auto')
 early = EarlyStopping(monitor="val_loss", mode="min", restore_best_weights=True, patience= 10, verbose = 1)
 history = model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=60, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt, early], verbose=2 
)
 historys[f'history_{seed+1}'] = history
print("Model History Saved.")
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 if seed < S_STARTS:
ss_relu.loc[:, train_targets.columns] += test_predict
res_relu.loc[te, train_targets.columns] += val_predict
else:
ss_elu.loc[:, train_targets.columns] += test_predict
res_elu.loc[te, train_targets.columns] += val_predict
print(f'OOF Metric For SEED {seed} => FOLD {n} : {metric(train_targets.loc[te, train_targets.columns], pd.DataFrame(val_predict, columns=train_targets.columns)) }')
 print('+-' * 10)
 ss_relu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_relu.loc[:, train_targets.columns] /= S_STARTS
ss_elu.loc[:, train_targets.columns] /=(( n+1)* S_STARTS)
 res_elu.loc[:, train_targets.columns] /= S_STARTS
_, age_categories = pd.cut(data['Age'], 5, retbins=True)
 age_categories = [np.floor(value)for value in age_categories]
data['AgeRange'] = np.zeros(len(data), dtype=np.int)
 data.loc[data['Age'] <= age_categories[1], 'AgeRange'] = 0
data.loc[(data['Age'] > age_categories[1])&(data['Age'] <= age_categories[2]), 'AgeRange'] = 1
data.loc[(data['Age'] > age_categories[2])&(data['Age'] <= age_categories[3]), 'AgeRange'] = 2
data.loc[(data['Age'] > age_categories[3])&(data['Age'] <= age_categories[4]), 'AgeRange'] = 3
data.loc[data['Age'] > age_categories[4], 'AgeRange'] = 4<feature_engineering>
print(f'OOF Metric(relu): {metric(train_targets, res_relu)}')
 print(f'OOF Metric(elu): {metric(train_targets, res_elu)}' )
<categorify><EOS> 
ss_relu.to_csv('submission_relu.csv', index=False)
 ss_elu.to_csv('submission_elu.csv', index=False )