Kaggle competition: Mechanisms of Action (MoA) Prediction
def preprocessing_1(df_train, df_test):

 y_train = df_train['target'].copy()
df = pd.concat([df_train.drop('target', axis=1), df_test])
 df = df.fillna(value={'bin_0': df.bin_0.mode() [0], 'bin_1':df.bin_1.mode() [0], 'bin_2':df.bin_2.mode() [0], 'bin_3':df.bin_3.mode() [0],
'bin_4':df.bin_4.mode() [0], 'nom_0':df.nom_0.mode() [0], 'nom_1':'nan', 'nom_2':'nan', 'nom_3':'nan', 'nom_4':'nan',
'nom_5':'nan', 'nom_6':'nan', 'nom_7':'nan', 'nom_8':'nan', 'nom_9':'nan', 'ord_0':df.ord_0.median() , 'ord_5':'0',
'day':8, 'month':13})
 df['ord_5'] = df['ord_5'].astype(str ).str[0]
replace_letters = {j:i for i,j in enumerate(list(string.ascii_uppercase + string.ascii_lowercase)) }
replace_letters['0'] = np.nan
df = df.replace({'bin_3':{'F':0, 'T':1}, 'bin_4':{'N':0, 'Y':1},'ord_1': {'Novice':1, 'Contributor':2, 'Expert':3, 'Master':4, 'Grandmaster':5},
'ord_2':{'Freezing': 1, 'Cold':2, 'Warm':3, 'Hot':4, 'Boiling Hot': 5, 'Lava Hot':6},'ord_3': replace_letters, 'ord_4':replace_letters,
'ord_5':replace_letters})
 df = df.fillna(value={'ord_1':df['ord_1'].median() , 'ord_2':df['ord_2'].median() , 'ord_3':df['ord_3'].median() , 'ord_4':df['ord_4'].median() , 'ord_5':df['ord_5'].median() })
 one_hot_cols = ['nom_0','nom_1','nom_2','nom_3','nom_4']
X_cat = df[one_hot_cols].astype(str ).copy()
one_hot_enc = OneHotEncoder()
X_cat_1hot = one_hot_enc.fit_transform(X_cat)
 X_cat_1hot = pd.DataFrame(X_cat_1hot.toarray() , index=X_cat.index, columns = one_hot_enc.get_feature_names())
 X_cat_1hot.drop(['x'+str(i)+'_' + X_cat['nom_' + str(i)].mode() [0] for i in range(5)], axis=1, inplace=True)
 X = pd.concat([df.drop(one_hot_cols, axis=1), X_cat_1hot], axis=1)
 X['sin_day'] = np.sin(2*np.pi*X.day/8)
 X['cos_day'] = np.cos(2*np.pi*X.day/8)
 X['sin_month'] = np.sin(2*np.pi*X.month/13)
 X['cos_month'] = np.cos(2*np.pi*X.month/13)
 X.drop(['day', 'month'], axis=1, inplace=True)
 X_train = X.iloc[:600000]
X_test = X.iloc[600000:1000000]
target_enc_cols = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
X_train_encoded = X_train.copy()
for index_fit, index_transform in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X_train, y_train):
target_enc = TargetEncoder(cols = target_enc_cols, smoothing=0.5)
 target_enc.fit(X_train.iloc[index_fit,:], y_train.iloc[index_fit])
 X_train_encoded.loc[index_transform,:] = target_enc.transform(X_train.iloc[index_transform, :])
 target_enc = TargetEncoder(cols = target_enc_cols, smoothing=0.5)
 target_enc.fit(X_train, y_train)
 X_test_encoded = target_enc.transform(X_test)
 X_train_encoded.loc[:,target_enc_cols] = X_train_encoded[target_enc_cols].astype(float)
 X_train_encoded.loc[:, X_train_encoded.columns.drop(target_enc_cols ).to_list() ] = X_train_encoded.loc[:, X_train_encoded.columns.drop(target_enc_cols ).to_list() ].astype(float)
 X_test_encoded.loc[:,'ord_3'] = X_test_encoded.loc[:,'ord_3'].astype(float)
 X_test_encoded.loc[:,'ord_4'] = X_test_encoded.loc[:,'ord_4'].astype(float)
 return(X_train_encoded, y_train, X_test_encoded )<prepare_x_and_y>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id'],axis=1 )
start = datetime.now()
X, y, X_test = preprocessing_1(df, test)
 end = datetime.now()
print('Duration: ', end-start )<save_to_csv>
cat = 'cp_dose'
le = preprocessing.LabelEncoder()
le.fit(train_features[cat])
 train_features[cat] = le.transform(train_features[cat])
 test_features[cat] = le.transform(test_features[cat] )
X.to_csv('X_1st_strategy.csv', index=False)
 y.to_csv('y.csv', index=False, header=True)
 X_test.to_csv('X_test_1st_strategy.csv', index=False )<compute_test_metric>
scaler = preprocessing.QuantileTransformer(output_distribution='normal')
 data_train = scaler.fit_transform(train_features.iloc[:,2:])
 data_test = scaler.transform(test_features.drop('cp_type',axis=1 ).iloc[:,2:])
 scaler = preprocessing.StandardScaler()
data_train = np.concatenate(( scaler.fit_transform(train_features.iloc[:,:2]),data_train),axis=1)
 data_test = np.concatenate(( scaler.transform(test_features.drop('cp_type',axis=1 ).iloc[:,:2]),data_test),axis=1 )
def display_scores(train_scores, val_scores, y_validations, y_predictions):
print('Training scores: ', train_scores)
 print('Mean training score: ', np.mean(train_scores))
print('Standard deviation of the training scores: ', np.std(train_scores), '
')
 print('Validation scores: ', val_scores)
 print('Mean validation score: ', np.mean(val_scores))
print('Standard deviation of the validation scores: ', np.std(val_scores), '
')
 y_valid = np.concatenate(tuple(y_validations))
y_pred = np.concatenate(tuple(y_predictions))
plot_precision_recall_vs_threshold(y_valid, y_pred)
 print('
')
 fpr, tpr, threshold = roc_curve(y_valid, y_pred)
 plot_roc_curve(fpr,tpr )<split>
n_features = data_train.shape[1]
n_labels = labels_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]
start = datetime.now()
train_scores = []
val_scores = []
y_validations = []
y_predictions = []
for index_train, index_val in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X, y):
X_train = X.iloc[index_train]
y_train = y.iloc[index_train]
X_valid = X.iloc[index_val]
y_valid = y.iloc[index_val]
weights = sum(y_train.values==0)/sum(y_train.values==1)
 xgb_gbrf = xgb.XGBClassifier(n_estimators=3000, random_state=0, objective='binary:logistic', scale_pos_weight=weights,
learning_rate=0.15, max_depth=2, subsample=0.7, min_child_weight=500,
colsample_bytree = 0.2, reg_lambda = 3.5, reg_alpha=1.5, num_parallel_tree= 5)
 xgb_gbrf.fit(X_train, y_train, eval_set= [(X_train, y_train),(X_valid, y_valid)], eval_metric=['auc'],
early_stopping_rounds=100, verbose=False)
 y_pred = xgb_gbrf.predict_proba(X_train)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 train_scores.append(train_score)
 y_pred = xgb_gbrf.predict_proba(X_valid)
 val_score = roc_auc_score(y_valid, y_pred[:,1])
 val_scores.append(val_score)
 y_validations.append(y_valid)
 y_predictions.append(y_pred[:,1])
 display_scores(train_scores, val_scores, y_validations, y_predictions)
 runtime = datetime.now() - start
print('
Runtime ', runtime )<save_to_csv>
cs = train_features.columns.str.contains('c-')
 cells_train = data_train[:,cs]
cells_test = data_test[:,cs]
encoding_dim = 256
activation = 'swish'
dropout_noise = 0.15
dropout = 0.1
class Autoencoder(Model):
def __init__(self, n_inputs, encoding_dim):
super(Autoencoder, self ).__init__()
self.encoder = Sequential([
layers.BatchNormalization() ,
layers.Dropout(dropout_noise),
layers.Dense(0.5*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(0.75*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(encoding_dim,activation=activation),
])
 self.decoder = Sequential([
layers.Dense(0.75*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(0.5*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(n_inputs)
 ])
 def call(self, x):
encoded = self.encoder(x)
 decoded = self.decoder(encoded)
 return decoded
cells_autoencoder = Autoencoder(cs.sum() ,encoding_dim)
 cells_autoencoder.compile(optimizer=optimizers.Adam(learning_rate=1E-5), loss='mse')
 ae_reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1E-5)
 ae_early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=1E-5, patience=16, restore_best_weights=True)
 def scheduler(epoch,lr):
if epoch%32<17:
lr +=(0.3*1E-2)/16
else:
lr -=(0.3*1E-2)/16
return lr
lr_scheduler = callbacks.LearningRateScheduler(scheduler)
 hist = cells_autoencoder.fit(cells_train,cells_train,batch_size=128, verbose=0, validation_data =(cells_test,cells_test), epochs=256, shuffle=True,
callbacks=[ae_early_stopping,lr_scheduler])
 cells_autoencoder.compile(optimizer=optimizers.Adam(learning_rate=5*1E-5), loss='mse')
 cells_autoencoder.fit(cells_train,cells_train,batch_size=128, verbose=0, validation_data =(cells_test,cells_test), epochs=256, shuffle=True,
callbacks=[ae_early_stopping,ae_reduce_lr])
 cells_autoencoder.save('CellsAE' )
gbrf_feature_importances = pd.DataFrame({'Predictors': X.columns, 'Normalized gains': xgb_gbrf.feature_importances_})
 gbrf_feature_importances.sort_values('Normalized gains',ascending=False, inplace=True)
 gbrf_feature_importances.reset_index(inplace=True, drop=True)
 gbrf_feature_importances.to_csv('fi_gbrf_1st_strategy' )<choose_model_class>
gs = train_features.columns.str.contains('g-')
 genes_train = data_train[:,gs]
genes_test = data_test[:,gs]
encoding_dim = 1600
activation = 'swish'
dropout_noise = 0.15
dropout = 0.1
class Autoencoder(Model):
def __init__(self, n_inputs, encoding_dim):
super(Autoencoder, self ).__init__()
self.encoder = Sequential([
layers.BatchNormalization() ,
layers.Dropout(dropout_noise),
layers.Dense(0.5*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(0.75*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(encoding_dim,activation=activation),
])
 self.decoder = Sequential([
layers.Dense(0.75*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(0.5*encoding_dim,activation=activation),
layers.BatchNormalization() ,
layers.Dropout(dropout),
layers.Dense(n_inputs)
 ])
 def call(self, x):
encoded = self.encoder(x)
 decoded = self.decoder(encoded)
 return decoded
genes_autoencoder = Autoencoder(gs.sum() ,encoding_dim)
 genes_autoencoder.compile(optimizer=optimizers.Adam(learning_rate=1E-5), loss='mse')
 ae_reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1E-5)
 ae_early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=1E-5, patience=16, restore_best_weights=True)
 hist = genes_autoencoder.fit(genes_train,genes_train,batch_size=128, verbose=0, validation_data =(genes_test,genes_test), epochs=256, shuffle=True,
callbacks=[ae_early_stopping,lr_scheduler])
 genes_autoencoder.compile(optimizer=optimizers.Adam(learning_rate=5*1E-5), loss='mse')
 genes_autoencoder.fit(genes_train,genes_train,batch_size=128, verbose=0, validation_data =(genes_test,genes_test), epochs=256, shuffle=True,
callbacks=[ae_early_stopping])
 genes_autoencoder.save('GenesAE' )
weight = sum(y.values==0)/sum(y.values==1)
 xgb_gbrf = xgb.XGBClassifier(n_estimators=818, random_state=0, objective='binary:logistic', scale_pos_weight=weight,
learning_rate=0.15, max_depth=2, subsample=0.7, min_child_weight=500, colsample_bytree = 0.2,
reg_lambda = 3.5, reg_alpha=1.5, num_parallel_tree= 5 )<train_model>
ae_cells_train = cells_autoencoder.encoder(cells_train ).numpy()
ae_cells_test = cells_autoencoder.encoder(cells_test ).numpy()
autoencoder_error = mean_squared_error(cells_test,cells_autoencoder.decoder(ae_cells_test ).numpy())
 print("Cells reconstruction error is " + str(autoencoder_error))
xgb_gbrf.fit(X, y, eval_set= [(X, y)], eval_metric=['auc'], verbose=False )<predict_on_test>
pca = PCA(n_components=64)
 pca.fit(cells_train)
 pca_error = mean_squared_error(cells_test,pca.inverse_transform(pca.transform(cells_test)))
 print('PCA Reconstruction Error for Cells is ' + str(pca_error))
y_test = xgb_gbrf.predict_proba(X_test)[:,1]
output = pd.DataFrame({'id':sample_submission.index, 'target':y_test})
 output<save_to_csv>
ae_genes_train = genes_autoencoder.encoder(genes_train ).numpy()
ae_genes_test = genes_autoencoder.encoder(genes_test ).numpy()
autoencoder_error = mean_squared_error(genes_test,genes_autoencoder.decoder(ae_genes_test ).numpy())
 print("Genes reconstruction error is " + str(autoencoder_error))
output.to_csv('output_gbrf_1st_strategy.csv', index=False )<categorify>
pca = PCA(n_components=512)
 pca.fit(genes_train)
 pca_error = mean_squared_error(genes_test,pca.inverse_transform(pca.transform(genes_test)))
 print('PCA Reconstruction Error for Genes is ' + str(pca_error))
df_total = pd.concat([df.drop('target', axis=1), test])
 one_hot_cols = ['day','month']
X_cat = df_total[one_hot_cols].astype(str ).copy()
one_hot_enc = OneHotEncoder()
X_cat_1hot = one_hot_enc.fit_transform(X_cat)
 X_cat_1hot = pd.DataFrame(X_cat_1hot.toarray() , index=X_cat.index,
columns = pd.Series(one_hot_enc.get_feature_names() ).str.replace('x0', 'day' ).str.replace('x1', 'month'))
X_cat_1hot.drop(['day_' + X_cat['day'].mode() [0], 'month_' + X_cat['month'].mode() [0]], axis=1, inplace=True)
 X_lr_total = pd.concat([X, X_test])
 X_lr_total = pd.concat([X_lr_total.drop(['sin_day', 'cos_day', 'sin_month', 'cos_month'], axis=1), X_cat_1hot], axis=1)
 X_lr_total['bin_2_nan'] = df_total.bin_2.isna().astype(int)
 X_lr_total['bin_4_nan'] = df_total.bin_4.isna().astype(int)
 X_lr = X_lr_total.iloc[:600000]
X_test_lr = X_lr_total.iloc[600000:1000000]<split>
data_train = np.concatenate(( data_train[:,~(cs+gs)],ae_genes_train,ae_cells_train),axis=1)
 data_test = np.concatenate(( data_test[:,~(cs+gs)],ae_genes_test,ae_cells_test),axis=1 )
start = datetime.now()
X_train, X_valid, y_train, y_valid = train_test_split(X_lr, y, test_size=0.2, stratify=y, random_state=42)
 weight = sum(y_train.values==0)/sum(y_train.values==1)
 gbrf = xgb.XGBClassifier(n_estimators=3000, random_state=0, objective='binary:logistic', scale_pos_weight=weight,
learning_rate=0.10, max_depth=4, subsample=0.7, min_child_weight=500,
colsample_bytree = 0.2, reg_lambda = 3.5, reg_alpha=1.5, num_parallel_tree= 5)
 gbrf.fit(X_train, y_train, eval_set= [(X_train, y_train),(X_valid, y_valid)], eval_metric=['auc'],
early_stopping_rounds=100, verbose=False)
 y_pred = gbrf.predict_proba(X_train)
 print("Train AUROC = ", roc_auc_score(y_train, y_pred[:,1]))
y_pred = gbrf.predict_proba(X_valid)
 print("Validation AUROC = ", roc_auc_score(y_valid, y_pred[:,1]))
runtime = datetime.now() - start
print('
Runtime ', runtime )<save_to_csv>
n_labels = train_targets_scored.shape[1]
n_features = data_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
n_seeds = 3
seeds = [34,9,18]
n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
bias = tf.keras.initializers.Constant(np.log(labels_train.mean(axis=0)))
 for seed in seeds:
fold = 0
mskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in mskf.split(data_train,labels_train):
X_train = data_train[train]
X_test = data_train[test]
y_train = labels_train[train]
y_test = labels_train[test]
model = Sequential()
model.add(layers.Dropout(0.3))
model.add(layers.Dense(1536))
model.add(layers.Activation('elu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.7))
model.add(layers.Dense(1024))
model.add(layers.Activation('elu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(512))
model.add(layers.Activation('elu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(n_labels,activation='sigmoid',bias_initializer=bias))
model.compile(optimizer=optimizers.Adam(learning_rate=1E-5), loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=['binary_crossentropy',logloss])
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.3, patience=5, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=24, mode='min',restore_best_weights=True)
 def scheduler(epoch,lr):
if epoch%16<9:
lr += np.exp(-int(epoch/16)) *(0.5*1E-2)/16
else:
lr -= np.exp(-int(epoch/16)) *(0.5*1E-2)/16
return lr
lr_scheduler = callbacks.LearningRateScheduler(scheduler)
 hist = model.fit(X_train,y_train, batch_size=128, epochs=192,verbose=0,validation_data =(X_test,y_test),callbacks=[lr_scheduler, early_stopping])
 model.compile(optimizer=optimizers.Adam(learning_rate=1E-5), loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=['binary_crossentropy',logloss])
 model.fit(X_train,y_train, batch_size=128, epochs=192,verbose=0,validation_data =(X_test,y_test),callbacks=[early_stopping])
 hists.append(hist)
 model.save('AutoEncoded_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict(X_test)
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1
<load_from_csv><EOS> 
p_min = 0.0005
p_max = 0.9995
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )