Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<save_to_csv>
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.models import Sequential
import tensorflow.keras.layers as L
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
from tensorflow.keras.models import load_model
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
import random
sub = xgb_preds * 0.1 + y_pred_xgb * 0.65 + lgb_pred * 0.25
sub[sub > 1] = 1
test = pd.read_csv('.. /input/test.csv')
 test['winPlacePercPred'] = sub
aux = test.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
aux.columns = ['matchId','groupId','winPlacePerc']
test = test.merge(aux, how='left', on=['matchId','groupId'])
 subm = test[['Id','winPlacePerc']]
subm.to_csv("Stacked_1.csv", index=False )<save_to_csv>
train_data = pd.read_csv('.. /input/lishmy/train_data_exc.csv')
 test_data = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv' )
LR = LinearRegression()
LR.fit(df_train, Y)
 X_train, X_val, y_train,y_val = train_test_split(df_train,Y,test_size=0.3, random_state=42)
 print('Accuracy on training：
',LR.score(X_train, y_train))
print('Accuracy on validation：
',LR.score(X_val, y_val))
print('LinearRegression Accuracy：
',LR.score(df_train, Y))
pred = LR.predict(df_test)
 pred = pd.DataFrame({'Id':sum_id, 'winPlacePerc':pred})
 pred.to_csv('pred_Linear.csv',index=None )<set_options>
genes = [col for col in train_data if col[0:2]=='g-']
cells = [col for col in train_data if col[0:2]=='c-']
!pip install ultimate
gc.enable() 
<define_variables>
temp = pd.read_csv('.. /input/lish-moa/train_features.csv')
 features = temp.drop(['sig_id'],axis=1 ).columns
temp = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 targets = temp.drop('sig_id',axis=1 ).columns
del temp
INPUT_DIR = ".. /input/"<load_from_csv>
from sklearn.preprocessing import StandardScaler
def feature_engineering(is_train=True):
if is_train:
print("processing train_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test_V2.csv")
 df = pd.read_csv(INPUT_DIR + 'test_V2.csv')
 state('totalDistance')
 s = timer()
df['totalDistance'] = df['rideDistance'] + df["walkDistance"] + df["swimDistance"]
e = timer()
state('totalDistance', False, e - s)
 state('rankPoints')
 s = timer()
df['rankPoints'] = np.where(df['rankPoints'] <= 0 ,0 , df['rankPoints'])
 e = timer()
state('rankPoints', False, e-s)
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchDuration")
 features.remove("matchType")
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y 
<normalization>
class MoaDataPrepare:
def __init__(self,train,test):
self.train_data = train.copy()
self.test_data = test.copy()
self.train_data["cp_time"]= self.train_data["cp_time"].map({24:'24hr',48:'48hr',72:'72hr'})
 self.train_data["cp_dose"]= self.train_data["cp_dose"].map({'D1':0,'D2':1})
 self.test_data["cp_time"]= self.test_data["cp_time"].map({24:'24hr',48:'48hr',72:'72hr'})
 self.test_data["cp_dose"]= self.test_data["cp_dose"].map({'D1':0,'D2':1})
 cell_data = self.train_data.loc[:,cells]
cell_data_test = self.test_data.loc[:,cells]
pca=PCA(0.98)
 tran = pca.fit_transform(cell_data,train_data.loc[:,targets])
 conv_cell_data = pd.DataFrame(tran,columns=['pca-'+str(i)for i in range(tran.shape[1])])
 tran = pca.transform(cell_data_test)
 conv_cell_test = pd.DataFrame(tran,columns=['pca-'+str(i)for i in range(tran.shape[1])])
 self.train_data = self.train_prepare()
self.test_data = self.test_prepare()
self.train_data = pd.concat([self.train_data.iloc[:,:3+len(genes)],conv_cell_data,self.train_data.iloc[:,3+len(genes)+len(cells):]],axis=1)
 self.test_data = pd.concat([self.test_data.iloc[:,:3+len(genes)],conv_cell_test],axis=1)
 sc = StandardScaler()
self.train_data.loc[:,genes] = sc.fit_transform(self.train_data.loc[:,genes])
 self.test_data.loc[:,genes] = sc.fit_transform(self.test_data.loc[:,genes])
 self.features = self.train_data.columns[:3+len(genes)+conv_cell_data.shape[1]]
self.targets = targets
def train_prepare(self):
self.train_data = self.train_data.drop(['sig_id','cp_type'],axis=1)
 cp_time = pd.get_dummies(self.train_data.cp_time,drop_first = True)
 self.train_data = pd.concat([cp_time,self.train_data.drop(['cp_time'],axis=1)],axis=1)
 return self.train_data
def test_prepare(self):
self.test_data = self.test_data.drop(['sig_id','cp_type'],axis=1)
 cp_time = pd.get_dummies(self.test_data.cp_time,drop_first = True)
 self.test_data = pd.concat([cp_time,self.test_data.drop(['cp_time'],axis=1)],axis=1)
 return self.test_data
def getFold(self,fold):
X_train = self.train_data[self.train_data.kfold!=fold].loc[:,self.features]
X_val = self.train_data[self.train_data.kfold==fold].loc[:,self.features]
Y_train = self.train_data[self.train_data.kfold!=fold].loc[:,self.targets]
Y_val = self.train_data[self.train_data.kfold==fold].loc[:,self.targets]
return X_train,X_val,Y_train,Y_val
def get_train(self):
return self.train_data.loc[:,self.get_test().columns]
def get_test(self):
return self.test_data
def __del__(self):
del self.train_data
del self.test_data
%%time
x_train, y = feature_engineering(True)
 scaler = preprocessing.MinMaxScaler(feature_range=(-1, 1), copy=False ).fit(x_train )<train_model>
class ModelBuilder:
def __init__(self,random_state=666):
self.random_state=random_state
def SimpleNeuralNet(self,shape=None,learning_rate=0.001,output_shape=206):

 model = tf.keras.models.Sequential([
L.InputLayer(input_shape=shape),
L.BatchNormalization() ,
L.Dropout(0.5),
tfa.layers.WeightNormalization(L.Dense(512,kernel_initializer="he_normal")) ,
L.BatchNormalization() ,
L.Activation(tf.nn.leaky_relu),
L.Dropout(0.5),
tfa.layers.WeightNormalization(L.Dense(256,kernel_initializer="he_normal")) ,
L.BatchNormalization() ,
L.Activation(tf.nn.leaky_relu),
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(output_shape,activation="sigmoid",kernel_initializer="he_normal"))
])
 model.compile(optimizer= tf.keras.optimizers.Adam(lr=learning_rate),loss="binary_crossentropy",metrics=["binary_crossentropy"])
 return model
def transfer_weight(self,model_source,model_dest):
for i in range(len(model_source.layers[:-1])) :
model_dest.layers[i].set_weights(model_source.layers[i].get_weights())
 return model_dest
def metric(self,train,predict):
metrics=[]
for col in range(train.shape[1]):
metrics.append(log_loss(train.iloc[:,col],predict[:,col],labels=[0,1]))
return np.mean(metrics )
print("x_train", x_train.shape, x_train.max() , x_train.min())
 scaler.transform(x_train)
 print("x_train", x_train.shape, x_train.max() , x_train.min() )<choose_model_class>
data_prepare = MoaDataPrepare(train_data,test_data )
%%time
NN_model = Sequential()
NN_model.add(Dense(x_train.shape[1], input_dim = x_train.shape[1], activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(136, activation='relu'))
NN_model.add(Dense(1, activation='linear'))
NN_model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['mean_absolute_error'])
 NN_model.summary() 
<train_model>
model_builder = ModelBuilder(random_state=606 )
%%time
NN_model.fit(x=x_train, y=y, batch_size=1000,
epochs=30, verbose=1, callbacks=callbacks_list,
validation_split=0.15, validation_data=None, shuffle=True,
class_weight=None, sample_weight=None, initial_epoch=0,
steps_per_epoch=None, validation_steps=None)
 del x_train, y
gc.collect()<normalization>
transfer_data = data_prepare.get_train()
transfer_data.loc[:,'sig_id'] = train_data.sig_id
transfer_data = transfer_data.merge(train_targets_nonscored,how='inner',on='sig_id')
 transfer_data = transfer_data.drop('sig_id',axis=1)
 Y_transfer = transfer_data.loc[:,train_targets_nonscored.columns[1:]]
transfer_data = transfer_data.loc[:,data_prepare.get_train().columns]
x_test, _ = feature_engineering(False)
 scaler.transform(x_test)
 print("x_test", x_test.shape, x_test.max() , x_test.min())
 np.clip(x_test, out=x_test, a_min=-1, a_max=1)
 print("x_test", x_test.shape, x_test.max() , x_test.min() )<predict_on_test>
x_train,x_test,y_train,y_test = train_test_split(transfer_data,Y_transfer,test_size=0.2,random_state = 101 )
%%time
pred = NN_model.predict(x_test)
 del x_test
gc.collect()<prepare_output>
model = model_builder.SimpleNeuralNet(shape =(transfer_data.shape[1],),output_shape = 402 )
pred = pred.reshape(-1)
 pred =(pred + 1)/ 2<load_from_csv>
save_weight = tf.keras.callbacks.ModelCheckpoint('model.learned.hdf5',save_best_only=True,save_weights_only=True,monitor = 'val_loss',mode='min')
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 early = EarlyStopping(monitor='val_loss',patience=5,mode='min' )
df_test = pd.read_csv(INPUT_DIR + 'test_V2.csv' )<feature_engineering>
model.fit(x_train,y_train,
epochs=30,
batch_size=128,
validation_data=(x_test,y_test),
callbacks=[early,save_weight,reduce_lr_loss]
)
%%time
print("fix winPlacePerc")
 for i in range(len(df_test)) :
winPlacePerc = pred[i]
maxPlace = int(df_test.iloc[i]['maxPlace'])
 if maxPlace == 0:
winPlacePerc = 0.0
elif maxPlace == 1:
winPlacePerc = 1.0
else:
gap = 1.0 /(maxPlace - 1)
 winPlacePerc = round(winPlacePerc / gap)* gap
if winPlacePerc < 0: winPlacePerc = 0.0
if winPlacePerc > 1: winPlacePerc = 1.0
pred[i] = winPlacePerc<feature_engineering>
model.load_weights('model.learned.hdf5' )
df_test['winPlacePerc'] = pred<save_to_csv>
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss.loc[:,'sig_id'] = test_data['sig_id'].values
ss.iloc[:,1:]=0
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<load_from_csv>
histories=[]
scores = []
for fold in range(5):
x_train,x_val,y_train,y_val = data_prepare.getFold(fold)
 print(f'Fold: {fold}
')
 tf.keras.backend.clear_session()
print('training with transfered weights')
 model_fin = model_builder.SimpleNeuralNet(shape=(x_train.shape[1],),learning_rate=0.001,output_shape=206)
 model_fin = model_builder.transfer_weight(model,model_fin)
 for layer in model_fin.layers:
layer.trainable=True
checkpoint_path = f'best_model_{fold}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
mode = 'min')
 early = EarlyStopping(monitor='val_loss',patience=5,mode='min')
 history=model_fin.fit(x_train.values,
y_train.values,
validation_data=(x_val.values,y_val.values),
epochs=50, batch_size=128,
callbacks=[early,reduce_lr_loss,checkpt] 
)
 print('')
 histories.append(history)
 model_fin= tf.keras.models.load_model(checkpoint_path, custom_objects={'leaky_relu': tf.nn.leaky_relu})
 score= model_builder.metric(y_val,model_fin.predict(x_val ).astype(float))
scores.append(score)
 print(f'Validation metric: {score}')
 test_predict = model_fin.predict(data_prepare.get_test().values)
 ss.loc[:, y_train.columns] += test_predict
print('')
train_df = pd.read_csv('.. /input/train_V2.csv')
 test_df = pd.read_csv('.. /input/test_V2.csv' )<set_options>
ss.loc[:, train_targets_scored.columns[1:]] /= 5
%matplotlib inline
warnings.filterwarnings("ignore" )<categorify>
ss.loc[test_data[test_data.cp_type=="ctl_vehicle"].index,train_targets_scored.drop('sig_id',axis=1 ).columns]=0
train_matchType = pd.get_dummies(train_df['matchType'])
 test_matchType = pd.get_dummies(test_df['matchType'])
 train_df2 = train_df.drop(columns=['matchType'])
 test_df2 = test_df.drop(columns=['matchType'])
 train_df2 = pd.concat([train_matchType, train_df2], axis = 1)
 test_df2 = pd.concat([test_matchType, test_df2], axis = 1 )<filter>
ss.to_csv('./submission.csv',index=False )