Kaggle competition: Mechanisms of Action (MoA) Prediction
count0 = df0.time.value_counts() /len(df0)
 count0 = count0.sort_index()
count1 = df1.time.value_counts() /len(df1)
 count1 = count1.sort_index()
count_test = df_test.time.value_counts(normalize=True )<concatenate>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split, KFold
from sklearn.feature_selection import VarianceThreshold
from keras import Sequential
from keras.backend import clear_session
from keras.layers import Dense, Dropout, BatchNormalization, Input
from keras.optimizers import Adam
from keras.losses import BinaryCrossentropy
from tensorflow_addons.layers import WeightNormalization
from tensorflow.keras import callbacks
full_data = pd.concat([raw_train, raw_test], sort=False ).drop(columns='target')
 full_data.shape<feature_engineering>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_non_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 train_targets.head()
cate_columns = full_data.select_dtypes(include=['object'] ).columns
for col in cate_columns:
train_values = set(raw_train[col].unique())
 test_values = set(raw_test[col].unique())
 xor_values = test_values - train_values
if xor_values:
print(f'Replace {len(xor_values)} in {col} column')
 print('They are: ', xor_values)
 print()
full_data.loc[full_data[col].isin(xor_values), col] = 'xor'<categorify>
train_features = train_features.set_index('sig_id')
 train_targets = train_targets.set_index('sig_id')
 train_non_scored = train_non_scored.set_index('sig_id' )
map_ord1 = {'Novice':1,
'Contributor':2,
'Expert':4,
'Master':5,
'Grandmaster':6}
full_data.ord_1 = full_data.ord_1.map(map_ord1 )<categorify>
g_features = [x for x in train_features.columns if x.startswith('g-')]
c_features = [x for x in train_features.columns if x.startswith('c-')]
other_features = [x for x in train_features.columns if x not in g_features+c_features]
map_ord2 = {'Freezing':1,
'Cold':2,
'Warm':3,
'Hot':4,
'Boiling Hot':5,
'Lava Hot':6}
full_data.ord_2 = full_data.ord_2.map(map_ord2 )<categorify>
train_features['cp_type'] = train_features['cp_type'].map({
'trt_cp' : 0,
'ctl_vehicle' : 1})
 train_features['cp_dose'] = train_features['cp_dose'].map({
'D1' : 0,
'D2' : 1} )
full_data['ord_3_by_ord'] = full_data.ord_3.map(ord, na_action='ignore')
 map_ord3 = {key:value for value,key in enumerate(sorted(full_data.ord_3.dropna().unique())) }
full_data.ord_3 = full_data.ord_3.map(map_ord3 )<categorify>
X = train_features
y = train_targets
y_non_scored = train_non_scored
X = pd.get_dummies(X, columns = ['cp_time'] )
full_data['ord_4_by_ord'] = full_data.ord_4.map(ord, na_action='ignore')
 map_ord4 = {key:value for value,key in enumerate(sorted(full_data.ord_4.dropna().unique())) }
full_data.ord_4 = full_data.ord_4.map(map_ord4 )<categorify>
X['g_std'] = X[g_features].std(axis = 1)
 X['g_var'] = X[g_features].var(axis = 1)
 X['g_skew'] = X[g_features].skew(axis = 1)
 X['g_kurt'] = X[g_features].kurtosis(axis = 1 )
full_data['ord_5_1'] = full_data.ord_5.map(lambda string: ord(string[0]), na_action='ignore')
 full_data['ord_5_2'] = full_data.ord_5.map(lambda string: ord(string[1]), na_action='ignore')
 map_ord5 = {key:value for value,key in enumerate(sorted(full_data.ord_5.dropna().unique())) }
full_data.ord_5 = full_data.ord_5.map(map_ord5 )<drop_column>
X['c_std'] = X[c_features].std(axis = 1)
 X['c_var'] = X[c_features].var(axis = 1)
 X['c_skew'] = X[c_features].skew(axis = 1)
 X['c_kurt'] = X[c_features].kurtosis(axis = 1 )
num_columns = full_data.select_dtypes(exclude=['object'] ).columns.drop(['bin_0', 'bin_1', 'bin_2'])
 cate_columns = full_data.columns.drop(num_columns )<data_type_conversions>
stat_features = ['g_std', 'g_var', 'g_skew', 'g_kurt', 'c_std', 'c_var', 'c_skew', 'c_kurt']
missing_num_columns = [col for col in num_columns if any(full_data[col].isnull())]
for col in missing_num_columns:
full_data[col+'_is_missing'] = full_data[col].isnull().astype(int )<feature_engineering>
selector = VarianceThreshold(0.85)
 selector_cols = g_features + c_features + stat_features
selector.fit(X[selector_cols] )
time_cols = ['day', 'month']
for col in time_cols:
full_data[col+'_sin'] = np.sin(2*np.pi*full_data[col]/7)
 full_data[col+'_cos'] = np.cos(2*np.pi*full_data[col]/12)
 full_data = full_data.drop(columns=time_cols )<categorify>
drop = [col for col, support in zip(selector_cols, selector.get_support())if not support]
X = X.drop(columns = drop )
print(f"One-Hot encoding {len(OH_cols)} columns")
 OH_full = pd.get_dummies(
 full_data,
columns=OH_cols,
drop_first=True,
dummy_na=True,
sparse=True,
).sparse.to_coo()<create_dataframe>
scale_cols = g_features_train + c_features_train + stat_features_train
scaler = StandardScaler()
X[scale_cols] = scaler.fit_transform(X[scale_cols] )
imputer = SimpleImputer(strategy='mean')
 retain_full = pd.DataFrame(imputer.fit_transform(full_data[retain_cols]), columns=retain_cols)
 retain_full = retain_full/retain_full.max()<categorify>
scale_cols = g_features_train + c_features_train + stat_features_train
scaler = StandardScaler()
X[scale_cols] = scaler.fit_transform(X[scale_cols])
 X.shape
encoded_full = scipy.sparse.hstack([OH_full, retain_full, retain_full**2] ).tocsr()
print(encoded_full.shape)
 encoded_train = encoded_full[:len(raw_train)]
encoded_test = encoded_full[len(raw_train):]<choose_model_class>
n_components = 100
pca = PCA(n_components = n_components)
 pca_features = pca.fit_transform(X[g_features_train + c_features_train])
 pca_cols = ['pca_'+str(i)for i in range(n_components)]
X[pca_cols] = pca_features
model = LogisticRegression(C=0.03, max_iter=300 )<train_on_grid>
auto_input_cols = g_features_train + c_features_train
len(auto_input_cols )
<predict_on_test>
auto = tf.keras.Sequential([
tf.keras.layers.Dense(400, activation = 'relu', input_shape =(len(auto_input_cols),)) ,
tf.keras.layers.Dense(100, activation = 'relu'),
tf.keras.layers.Dense(400, activation = 'relu'),
tf.keras.layers.Dense(len(auto_input_cols))
])
 auto.compile(optimizer = tf.keras.optimizers.Adam() ,
loss = tf.keras.losses.MeanSquaredError())
 auto.summary()
%%time
model = LogisticRegression(C=0.03, max_iter=300)
 model.fit(encoded_train, raw_train.target)
 test_pred = model.predict_proba(encoded_test)[:, 1]<save_to_csv>
auto.fit(X[auto_input_cols], X[auto_input_cols],
epochs = 80,
batch_size = 128 )
%%time
submiss = pd.DataFrame({"id": raw_test.index, "target": test_pred})
 submiss.to_csv('Phan_Viet_Hoang.csv', index=False )<install_modules>
auto_features = tf.keras.backend.function([auto.input], [auto.layers[1].output] )([X[auto_input_cols].to_numpy() , 1])[0]
!pip install --no-warn-conflicts -q deepctr<set_options>
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test = test.set_index('sig_id')
 test['cp_type'] = test['cp_type'].map({
'trt_cp' : 0,
'ctl_vehicle' : 1})
 test['cp_dose'] = test['cp_dose'].map({
'D1' : 0,
'D2' : 1})
 X_test = pd.get_dummies(test, columns = ['cp_time'])
 X_test['g_std'] = X_test[g_features].std(axis = 1)
 X_test['g_var'] = X_test[g_features].var(axis = 1)
 X_test['g_skew'] = X_test[g_features].skew(axis = 1)
 X_test['g_kurt'] = X_test[g_features].kurtosis(axis = 1)
 X_test['c_std'] = X_test[c_features].std(axis = 1)
 X_test['c_var'] = X_test[c_features].var(axis = 1)
 X_test['c_skew'] = X_test[c_features].skew(axis = 1)
 X_test['c_kurt'] = X_test[c_features].kurtosis(axis = 1)
 X_test = X_test.drop(columns = drop)
 X_test[scale_cols] = scaler.transform(X_test[scale_cols])
 pca_features = pca.transform(X_test[g_features_train + c_features_train])
 X_test[pca_cols] = pca_features
auto_features = tf.keras.backend.function([auto.input], [auto.layers[1].output] )([X_test[auto_input_cols].to_numpy() , 1])[0]
X_test[auto_cols] = auto_features
X_test.shape
warnings.simplefilter('ignore' )<load_from_csv>
def build_model(input_shape, output_shape, l1 = 1e-7, l2 = 1e-7, r = 1):

 reg = tf.keras.regularizers.L1L2(l1 = l1, l2 = l2)
 model = Sequential()
model.add(Input(input_shape))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(WeightNormalization(Dense(int(2048*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(WeightNormalization(Dense(int(1024*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(WeightNormalization(Dense(int(512*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(WeightNormalization(Dense(int(395*r), activation = 'selu',
kernel_initializer = 'lecun_normal',
kernel_regularizer = reg)))
 model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(WeightNormalization(Dense(output_shape, activation = 'sigmoid',
kernel_regularizer = reg)))
 return model
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv' )<feature_engineering>
def custom_metric(y_true, y_pred):

 eps = tf.constant(1e-7, dtype = tf.float32)
 y_pred = tf.math.maximum(tf.math.minimum(y_pred, 1-eps),eps)
 log_loss = -y_true * tf.math.log(y_pred)-(1 - y_true)* tf.math.log(1 - y_pred)
 return tf.reduce_mean(log_loss )
test["target"] = -1<concatenate>
reduce_lr = callbacks.ReduceLROnPlateau(patience = 5, mode = 'min', monitor = 'val_custom_metric', factor=0.1, epsilon=1e-4 )
data = pd.concat([train, test] ).reset_index(drop=True )<feature_engineering>
from skopt import gp_minimize
from skopt.space.space import Real, Integer
from skopt.utils import use_named_args
data['null'] = data.isna().sum(axis=1 )<feature_engineering>
dim_l1 = Real(low = 1e-7, high = 3e-7, name = 'l1')
 dim_l2 = Real(low = 9e-9, high = 3e-8, name = 'l2')
 dim_r = Real(low = 0.85, high = 1.15, name = 'r')
 dimensions = [dim_l1, dim_l2, dim_r]
sparse_features = [feat for feat in train.columns if feat not in ['id','target']]
data[sparse_features] = data[sparse_features].fillna('-1', )<categorify>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, shuffle = True )
for feat in sparse_features:
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat].fillna("-1" ).astype(str ).values )<prepare_x_and_y>
@use_named_args(dimensions = dimensions)
 def obj_fun(l1, l2, r):
clear_session()
model = build_model(X_train.shape[1], y_train.shape[1], l1 = l1, l2 = l2, r = r)
 model.compile(optimizer = Adam(2e-3),
loss = BinaryCrossentropy(label_smoothing = 0.001),
metrics = [custom_metric])
 history = model.fit(X_train, y_train,
batch_size = 128,
epochs = 45,
validation_data =(X_val, y_val),
callbacks = [reduce_lr],
verbose = 0)
 return history.history['val_custom_metric'][-1]
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True )<count_unique_values>
l1 = 1.3e-07
l2 = 2.27e-08
r = 0.946078
fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique())for feat in sparse_features]
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns )<compute_test_metric>
y_preds = []
val_log_losses = []
verbose = 0
batch_size = 128
runs = 5
splits = 7
epochs = 45
for run in range(runs):
kf = KFold(n_splits = splits, shuffle = True)
 for fold,(train_idx, val_idx)in enumerate(kf.split(X, y)) :
print('Run {} Fold {}'.format(run, fold))
X_train, X_val = X.to_numpy() [train_idx], X.to_numpy() [val_idx]
y_train, y_val = y.to_numpy() [train_idx], y.to_numpy() [val_idx]
y_non_scored_train, y_non_scored_val = y_non_scored.to_numpy() [train_idx], y_non_scored.to_numpy() [val_idx]
clear_session()
model = build_model(X_train.shape[1], y_train.shape[1], l1 = l1, l2 = l2, r = r)
 model.compile(optimizer = Adam(2e-3),
loss = BinaryCrossentropy(label_smoothing = 0.001),
metrics = [custom_metric])
 history = model.fit(X_train, y_train,
batch_size = batch_size,
epochs = epochs,
validation_data =(X_val, y_val),
callbacks = [reduce_lr],
verbose = verbose)
 val_log_losses.append(history.history['val_custom_metric'][-1])
 y_preds.append(model.predict(X_test))
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )<choose_model_class>
np.mean(val_log_losses )
class CyclicLR(keras.callbacks.Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1 /(2.**(x - 1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma **(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1 + self.clr_iterations /(2 * self.step_size))
x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(
 self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr())
<define_variables>
predictions = np.mean(np.array(y_preds), axis = 0 )
target = ['target']
N_Splits = 20
Epochs = 10
SEED = 2020<prepare_x_and_y>
submission = pd.DataFrame(predictions, columns = train_targets.columns)
 submission['sig_id'] = test.index
submission = submission[['sig_id']+list(train_targets.columns)]
submission.loc[list(test.cp_type == 1), train_targets.columns] = 0
print(submission.shape)
 submission.head()
oof_pred_deepfm = np.zeros(( len(train),))
y_pred_deepfm = np.zeros(( len(test),))
skf = StratifiedKFold(n_splits=N_Splits, shuffle=True, random_state=SEED)
 for fold,(tr_ind, val_ind)in enumerate(skf.split(train, train[target])) :
X_train, X_val = train[sparse_features].iloc[tr_ind], train[sparse_features].iloc[val_ind]
y_train, y_val = train[target].iloc[tr_ind], train[target].iloc[val_ind]
train_model_input = {name:X_train[name] for name in feature_names}
val_model_input = {name:X_val[name] for name in feature_names}
test_model_input = {name:test[name] for name in feature_names}
model = NFFM(linear_feature_columns, dnn_feature_columns)
 model.compile("adam", "binary_crossentropy", metrics=[auc],)
 es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.0001, patience=2, verbose=1, mode='max', baseline=None, restore_best_weights=True)
 sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=0)
 clr = CyclicLR(base_lr=0.00001 / 100, max_lr = 0.0001,
step_size= int(1.0*(test.shape[0])/1024), mode='exp_range',
gamma=1., scale_fn=None, scale_mode='cycle')
 history = model.fit(train_model_input, y_train,
validation_data=(val_model_input, y_val),
batch_size=512, epochs=Epochs, verbose=1,
callbacks=[es, sb, clr],)
 model.load_weights('./nn_model.w8')
 val_pred = model.predict(val_model_input, batch_size=512)
 print(f"validation AUC fold {fold+1} : {round(roc_auc_score(y_val, val_pred), 5)}")
 oof_pred_deepfm[val_ind] = val_pred.ravel()
y_pred_deepfm += model.predict(test_model_input, batch_size=512 ).ravel() /(N_Splits)
 K.clear_session() 
<compute_test_metric>
submission.to_csv('submission.csv', index = False )