Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<prepare_x_and_y>
warnings.filterwarnings('ignore' )
def feature_engineering(is_train=True):
if is_train:
print("processing train.csv")
 df = train
else:
print("processing test.csv")
 df = test
target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove('matchType')
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y
x_train, y = feature_engineering(True)
<feature_engineering>
from sklearn.preprocessing import QuantileTransformer
test,z=feature_engineering(False )<normalization>
import os
import gc
import pickle
import datetime
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
from tqdm.notebook import tqdm
from time import time
scaler=MinMaxScaler(feature_range=(-1,1),copy=False ).fit(x_train)
 x_train=scaler.transform(x_train )<normalization>
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
from tensorflow.keras import layers,regularizers,Sequential,backend,callbacks,optimizers,metrics,losses
x_test=scaler.transform(test )<import_modules>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
from xgboost.sklearn import XGBRegressor
from sklearn.model_selection import cross_val_score,KFold
import numpy as np
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Lasso,LinearRegression
from sklearn import preprocessing
from sklearn.model_selection import RandomizedSearchCV,train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone<compute_train_metric>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
n_folds=2
def mbs(model):
kf=KFold(n_folds,shuffle=True,random_state=42 ).get_n_splits(x_train)
 score=-cross_val_score(model,x_train,y,scoring='neg_mean_absolute_error',cv=kf)
 return score 
<choose_model_class>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
lasso=Lasso(alpha=.005,random_state=1 )<define_search_space>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
params={'learning_rate':[.02,.03,.1],'min_child_weight':[4,6,8],'max_depth':[8,10],"subsample":[.6,.4],"n_estimators":[300,500]} 
<choose_model_class>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
xgb=XGBRegressor(learning_rate=.03,min_child_weight=4,max_depth=10,subsample=.4,n_estimators=500,n_jobs=-1)
<choose_model_class>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
GBoost = GradientBoostingRegressor(n_estimators=600, learning_rate=0.05,
max_depth=4, max_features='auto',
min_samples_leaf=15, min_samples_split=5,
loss='huber', random_state =5 )<train_model>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
class Average_models(BaseEstimator,RegressorMixin,TransformerMixin):
def __init__(self,models):
self.models=models
def fit(self,X,y):
self.models_=[clone(x)for x in self.models]
for model in self.models_:
model.fit(X,y)
 return self
def predict(self,X):
predictions= np.column_stack([model.predict(X)for model in self.models_])
 return np.mean(predictions,axis=1 )<find_best_model_class>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
training,valid,ytraining,yvalid=train_test_split(x_train,y)
 xgb.fit(training,ytraining)
 lasso.fit(training,ytraining)
 pred1=xgb.predict(valid)
 pred2=lasso.predict(valid)
 test_pred1=xgb.predict(test)
 test_pred2=lasso.predict(test)
 stacked_predictions=np.column_stack(( pred1,pred2))
stacked_test_prediction=np.column_stack(( test_pred1,test_pred2))
meta_model=LinearRegression()
meta_model.fit(stacked_predictions,yvalid)
 final=meta_model.predict(stacked_test_prediction)
<save_to_csv>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
df=pd.DataFrame({'id':test_id,'winPlacePerc':final},columns=['id','winPlacePerc'])
 csv=df[['id','winPlacePerc']].to_csv('submission_stack3.csv',index=False )<load_from_csv>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
warnings.filterwarnings('ignore')
 train = pd.read_csv(".. /input/train.csv")
 train.head()<load_from_csv>
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
ss_rf = ss_svc.copy()
ss = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']
test = pd.read_csv(".. /input/test.csv")
 test.head()<feature_engineering>
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
train["distance"] = train["rideDistance"]+train["walkDistance"]+train["swimDistance"]
train["skill"] = train["headshotKills"]+train["roadKills"]
train.head()<feature_engineering>
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
test["distance"] = test["rideDistance"]+test["walkDistance"]+test["swimDistance"]
test["skill"] = test["headshotKills"]+test["roadKills"]
test["distance"].head()<define_variables>
def create_model1(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.0015),metrics=logloss 
)
 return model
predictors = [ "kills",
"maxPlace",
"numGroups",
"distance",
"boosts",
"heals",
"revives",
"killStreaks",
"weaponsAcquired",
"winPoints",
"skill",
"assists",
"damageDealt",
"DBNOs",
"killPlace",
"killPoints",
"vehicleDestroys",
"longestKill"
]
x_train = train[predictors]
x_train.head()<prepare_x_and_y>
def create_model2(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.0015),metrics=logloss 
)
 return model
y_train = train["winPlacePerc"]
y_train.head()<train_model>
top_feats = range(1039)
 print(len(top_feats))
regressor = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 20, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.8)
 regressor.fit(x_train, y_train )<compute_train_metric>
scaler = StandardScaler()
X = scaler.fit_transform(train.values)
 x_tt = scaler.transform(test_features.values )
scores = cross_val_score(regressor, x_train, y_train, cv=10)
 print(scores )<compute_test_metric>
train = train.astype('float')
 test = test.astype('float' )
accuracy = scores.mean()
print(accuracy )<predict_on_test>
p_min = 0.0015
p_max = 0.9985
y_predict = regressor.predict(x_test)
 print(y_predict )<merge>
N_STARTS = [8,9]
tf.random.set_seed(42)
 ss_NN = ss_svc.copy()
res_NN = train_targets.copy()
ss_NN.loc[:, train_targets.columns] = 0
res_NN.loc[:, train_targets.columns] = 0
for seed in N_STARTS:
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=12, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f'Fold {n}')
 if seed%2 == 0:
model = create_model1(len(top_feats))
else:
model = create_model2(len(top_feats))
checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min')
 model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=35, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt], verbose=2, 
)
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss_NN.loc[:, train_targets.columns] += test_predict
res_NN.loc[te, train_targets.columns] += val_predict
print('')
 ss_NN.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res_NN.loc[:, train_targets.columns] /= N_STARTS
y_predict[y_predict > 1] = 1
test['winPlacePercPredictions'] = y_predict
aux = test.groupby(['matchId','groupId'])['winPlacePercPredictions'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
aux.columns = ['matchId','groupId','winPlacePerc']
test = test.merge(aux, how='left', on=['matchId','groupId'])
 submission = test[['Id','winPlacePerc']]<save_to_csv>
print(f'NN OOF Metric: {log_loss_metric(train_targets, res_NN)}')
 res_NN.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_NN.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )
submission.to_csv("kill_them_all.csv", index=False )<drop_column>
ss_NN.to_csv('submission.csv', index = False )