Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<count_values>
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
y.value_counts()<categorify>
train=pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_target = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv" )
le = LabelEncoder()
y = le.fit_transform(y)
 print(le.classes_ )<categorify>
test = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
def create_update_time_of_day_features(df_, dic_labels):

 try:
print('
Creating or updating period feature 
...early morning from 0H to 6H 
...morning from 6H to 12H 
...afternoon from 12H to 18H 
...evening from 18H to 24H')
 conditions = [(df_[dic_labels['datetime']].dt.hour >= 0)&(df_[dic_labels['datetime']].dt.hour < 6), 
(df_[dic_labels['datetime']].dt.hour >= 6)&(df_[dic_labels['datetime']].dt.hour < 12), 
(df_[dic_labels['datetime']].dt.hour >= 12)&(df_[dic_labels['datetime']].dt.hour < 18), 
(df_[dic_labels['datetime']].dt.hour >= 18)&(df_[dic_labels['datetime']].dt.hour < 24)]
choices = ['early morning', 'morning', 'afternoon', 'evening']
df_['PeriodOfDay'] = np.select(conditions, choices, 'undefined')
 print('...the period of day feature was created')
 except Exception as e:
raise e<data_type_conversions>
ctlVehicle_idx = train["cp_type"] != "ctl_vehicle"
train = train.loc[ctlVehicle_idx].reset_index(drop=True)
 train = train.drop("cp_type", axis=1)
 train_target = train_target.loc[ctlVehicle_idx].reset_index(drop=True )
df_train['Dates'] = pd.to_datetime(df_train['Dates'])
 df_test['Dates'] = pd.to_datetime(df_test['Dates'])
 create_update_time_of_day_features(df_train,dic_labels={'datetime' : 'Dates'})
 create_update_time_of_day_features(df_test,dic_labels={'datetime' : 'Dates'} )<feature_engineering>
gcols = [g for g in train.columns if "g-" in g]
ccols = [c for c in train.columns if "c-" in c]
cpcols = [cp for cp in train.columns if "cp_" in cp]
df_train['year'] = df_train['Dates'].dt.year
df_train['month'] = df_train['Dates'].dt.month
df_train['day'] = df_train['Dates'].dt.day
df_train['hour'] = df_train['Dates'].dt.hour
df_train['minute'] = df_train['Dates'].dt.minute
df_train["n_days"] =(df_train['Dates'] - df_train['Dates'].min() ).apply(lambda x: x.days)
 df_train.drop("Dates", axis=1, inplace=True )<feature_engineering>
ctlVehicle_test = test["cp_type"] == "ctl_vehicle"
test = test.drop("cp_type", axis=1)
 train["cp_time"]=train["cp_time"].map({24: 1, 48: 2, 72: 3})
 train["cp_dose"] = train["cp_dose"].map({"D1": 1, "D2": 2})
 test["cp_time"]= test["cp_time"].map({24: 1, 48: 2, 72: 3})
 test["cp_dose"]=test["cp_dose"].map({"D1": 1, "D2": 2})
 train
df_test['year'] = df_test['Dates'].dt.year
df_test['month'] = df_test['Dates'].dt.month
df_test['day'] = df_test['Dates'].dt.day
df_test['hour'] = df_test['Dates'].dt.hour
df_test['minute'] = df_test['Dates'].dt.minute
df_test['special_time'] = df_train['minute'].isin([0, 30] ).astype(int)
 df_test["n_days"] =(df_test['Dates'] - df_test['Dates'].min() ).apply(lambda x: x.days)
 df_test.drop("Dates", axis=1, inplace=True )<count_values>
for col in train.iloc[:, 3:].columns:
percent = train[col].quantile([0.01, 0.99] ).values
train[col] = np.clip(train[col], percent[0], percent[1])
 for col in test.iloc[:, 3:].columns:
percent = test[col].quantile([0.01, 0.99] ).values
test[col] = np.clip(test[col], percent[0], percent[1] )
df_train["DayOfWeek"].value_counts()<count_values>
g_pca = PCA(n_components=0.99)
 c_pca = PCA(n_components=0.99)
 train_test_g_concat = pd.concat([train[gcols], test[gcols]], axis=0)
 train_test_c_concat = pd.concat([train[ccols], test[ccols]], axis=0)
 g_pca.fit(train_test_g_concat)
 c_pca.fit(train_test_c_concat)
 train_gtrans = pd.DataFrame(g_pca.transform(train[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=train.index)
 test_gtrans = pd.DataFrame(g_pca.transform(test[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=test.index)
 train_ctrans = pd.DataFrame(c_pca.transform(train[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=train.index)
 test_ctrans = pd.DataFrame(c_pca.transform(test[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=test.index)
 g_pca.n_components_, c_pca.n_components_
df_train["DayOfWeek"].value_counts()<drop_column>
train = pd.concat([train_gtrans, train_ctrans, train[cpcols]], axis=1)
 test = pd.concat([test_gtrans, test_ctrans, test[cpcols]], axis=1)
 train
df_train.drop(["Category", "Descript", "Resolution", "Address"], axis=1, inplace=True)
 df_test.drop("Address", axis=1, inplace=True )<categorify>
train_target = train_target.drop("sig_id", axis=1)
 train_target
categorical_features = ["DayOfWeek", "PdDistrict", "PeriodOfDay"]
ct = ColumnTransformer(transformers=[("categorical_features", OrdinalEncoder() , categorical_features)],
remainder="passthrough")
 df_train = ct.fit_transform(df_train )<categorify>
def create_shallow_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(128, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=adam)
 return model
categorical_features = ["DayOfWeek", "PdDistrict","PeriodOfDay"]
ct = ColumnTransformer(transformers=[("categorical_features", OrdinalEncoder() , categorical_features)],
remainder="passthrough")
 df_test = ct.fit_transform(df_test )<import_modules>
def create_mid_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_adamw)
 return model
from sklearn.model_selection import train_test_split<import_modules>
def create_deep_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="selu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="elu")) ,
L.BatchNormalization() ,
L.Dropout(0.1),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_radam)
 return model
from sklearn.model_selection import train_test_split<split>
predictions = []
kf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 tf.random.set_seed(42)
 for fold_id,(train_idx, valid_idx)in enumerate(kf.split(train, train_target)) :
model1 = create_shallow_model()
model2 = create_mid_model()
model3 = create_deep_model()
history1 = model1.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model1_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 1, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history1.history["loss"]), min(history1.history["val_loss"])))
 history2 = model2.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model2_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 2, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history2.history["loss"]), min(history2.history["val_loss"])))
 history3 = model3.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model3_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 3, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history3.history["loss"]), min(history3.history["val_loss"])))
 model1.load_weights("model1_fold" + str(fold_id)+ ".h5")
 model2.load_weights("model2_fold" + str(fold_id)+ ".h5")
 model3.load_weights("model3_fold" + str(fold_id)+ ".h5")
 pred1 = model1.predict(test)
 pred2 = model2.predict(test)
 pred3 = model3.predict(test)
 predictions.append(np.average([pred1, pred2, pred3], weights=[0.15, 0.7, 0.15], axis=0))
X_train, X_test, y_train, y_test = train_test_split(df_train, y, test_size=0.25, random_state=42,stratify=y )<import_modules>
pred = np.mean(predictions, axis=0)
 pred = np.clip(pred, 0.001, 0.999)
 pred.shape
from sklearn.preprocessing import StandardScaler<normalization>
sub = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 sub.loc[:, 1:] = pred
sub.loc[ctlVehicle_test, sub.columns != "sig_id"] = 0
std = StandardScaler()
X_train_std = std.fit_transform(X_train)
 X_test_std = std.transform(X_test)
 X_predict_std = StandardScaler().fit_transform(df_test )<choose_model_class>
sub.to_csv("submission.csv", index=False )