Kaggle competition: Mechanisms of Action (MoA) Prediction
test2_rows = [row for row in submission['id'] if 'evaluation' in row]
test2 = submission[submission['id'].isin(test2_rows)]
test2.columns = ['id', 'd_1942', 'd_1943', 'd_1944', 'd_1945', 'd_1946', 'd_1947', 'd_1948', 'd_1949', 'd_1950', 'd_1951', 'd_1952', 'd_1953', 'd_1954', 'd_1955', 'd_1956', 'd_1957', 'd_1958', 'd_1959',
'd_1960', 'd_1961', 'd_1962', 'd_1963', 'd_1964', 'd_1965', 'd_1966', 'd_1967', 'd_1968', 'd_1969']
product = sales_train_validation[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()
test2 = test2.merge(product, how = 'left', on = 'id')
 test2 = pd.melt(test2,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation['part'] = 'train'
test2['part'] = 'test'<concatenate>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook
data = pd.concat([sales_train_validation,test2], axis = 0)
 del sales_train_validation,test2
gc.collect()<merge>
train=pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train
data.reset_index(drop=True,inplace=True)
 data = data.loc[30000000:]
calendar.drop(['weekday', 'wday', 'month', 'year'], inplace = True, axis = 1)
 data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])
 data.drop(['d', 'day'], inplace = True, axis = 1 )<merge>
train_target = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv")
data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')
 del calendar,sell_prices,product
gc.collect()<categorify>
ctlVehicle_idx = train["cp_type"] != "ctl_vehicle"
train = train.loc[ctlVehicle_idx].reset_index(drop=True)
 train = train.drop("cp_type", axis=1)
 train_target = train_target.loc[ctlVehicle_idx].reset_index(drop=True )
nan_features = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in nan_features:
data[feature].fillna('unknown', inplace = True)
 cat = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']
for feature in cat:
encoder = LabelEncoder()
data[feature] = encoder.fit_transform(data[feature] )<statistical_test>
gcols = [g for g in train.columns if "g-" in g]
ccols = [c for c in train.columns if "c-" in c]
cpcols = [cp for cp in train.columns if "cp_" in cp]
data = reduce_mem_usage(data )<feature_engineering>
test = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 ctlVehicle_test = test["cp_type"] == "ctl_vehicle"
test = test.drop("cp_type", axis=1)
 train["cp_time"]=train["cp_time"].map({24: 1, 48: 2, 72: 3})
 train["cp_dose"] = train["cp_dose"].map({"D1": 1, "D2": 2})
 test["cp_time"]= test["cp_time"].map({24: 1, 48: 2, 72: 3})
 test["cp_dose"]=test["cp_dose"].map({"D1": 1, "D2": 2})
 train
data['lag_price_t1'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1))
data['price_change_t1'] =(data['lag_price_t1'] - data['sell_price'])/(data['lag_price_t1'])
 data['rolling_price_max_t365'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1 ).rolling(365 ).max())
 data['price_change_t365'] =(data['rolling_price_max_t365'] - data['sell_price'])/(data['rolling_price_max_t365'])
 data['rolling_price_std_t7'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(7 ).std())
 data['rolling_price_std_t30'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(30 ).std())
 data.drop(['rolling_price_max_t365', 'lag_price_t1'], inplace = True, axis = 1 )<feature_engineering>
for col in train.iloc[:, 3:].columns:
percent = train[col].quantile([0.01, 0.99] ).values
train[col] = np.clip(train[col], percent[0], percent[1])
 for col in test.iloc[:, 3:].columns:
percent = test[col].quantile([0.01, 0.99] ).values
test[col] = np.clip(test[col], percent[0], percent[1] )
data['date'] = pd.to_datetime(data['date'])
 data['year'] = data['date'].dt.year
data['month'] = data['date'].dt.month
data['week'] = data['date'].dt.week
data['day'] = data['date'].dt.day
data['dayofweek'] = data['date'].dt.dayofweek<feature_engineering>
g_pca = PCA(n_components=0.99)
 c_pca = PCA(n_components=0.99)
 train_test_g_concat = pd.concat([train[gcols], test[gcols]], axis=0)
 train_test_c_concat = pd.concat([train[ccols], test[ccols]], axis=0)
 g_pca.fit(train_test_g_concat)
 c_pca.fit(train_test_c_concat)
 train_gtrans = pd.DataFrame(g_pca.transform(train[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=train.index)
 test_gtrans = pd.DataFrame(g_pca.transform(test[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=test.index)
 train_ctrans = pd.DataFrame(c_pca.transform(train[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=train.index)
 test_ctrans = pd.DataFrame(c_pca.transform(test[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=test.index)
 g_pca.n_components_, c_pca.n_components_
def weekend(arg):
if arg==5 or arg==6:
return 1
else:
return 0
data['isweekend'] = data['dayofweek'].apply(weekend )<feature_engineering>
train = pd.concat([train_gtrans, train_ctrans, train[cpcols]], axis=1)
 test = pd.concat([test_gtrans, test_ctrans, test[cpcols]], axis=1)
 train
data['revenue'] = data['demand'] * data['sell_price']
data['lag_revenue_t1'] = data.groupby(['id'])['revenue'].transform(lambda x: x.shift(28))
data['rolling_revenue_std_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).std())
 data['rolling_revenue_mean_t28'] = data.groupby(['id'])['lag_revenue_t1'].transform(lambda x: x.rolling(28 ).mean())
 data.drop(['revenue'],axis=1,inplace=True )<define_variables>
train_target = train_target.drop("sig_id", axis=1)
 train_target
features = ['item_id', 'cat_id', 'state_id', 'year', 'month', 'week', 'day', 'dayofweek', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2',
'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90',
'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30', 'rolling_skew_t30', 'rolling_kurt_t30',
'isweekend','lag_revenue_t1','rolling_revenue_std_t28','rolling_revenue_mean_t28'
]
data = reduce_mem_usage(data )<prepare_x_and_y>
def create_shallow_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(128, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=adam)
 return model
def create_mid_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_adamw)
 return model
def create_deep_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(512, activation="selu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(512, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.42),
tfa.layers.WeightNormalization(L.Dense(256, activation="elu")) ,
L.BatchNormalization() ,
L.Dropout(0.1),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 0.0001)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 lookahead_adamw = tfa.optimizers.Lookahead(adamw)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-6), optimizer=lookahead_radam)
 return model
x_train = data[data['part'] == 'train']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-04-24')&(data['part'] == 'train')]
y_val = x_val['demand']
test = data[data['date'] > '2016-04-24']
test.loc[test['part']=='train','id'] = test.loc[test['part']=='train','id'].str.replace('_evaluation','_validation')
 x_train.drop(['demand','part',],inplace=True,axis=1)
 x_val.drop(['demand','part',],inplace=True,axis=1)
 test.drop(['demand','part',],inplace=True,axis=1 )<set_options>
predictions = []
kf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 tf.random.set_seed(42)
 for fold_id,(train_idx, valid_idx)in enumerate(kf.split(train, train_target)) :
model1 = create_shallow_model()
model2 = create_mid_model()
model3 = create_deep_model()
history1 = model1.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model1_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 1, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history1.history["loss"]), min(history1.history["val_loss"])))
 history2 = model2.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model2_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 2, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history2.history["loss"]), min(history2.history["val_loss"])))
 history3 = model3.fit(train.iloc[train_idx], train_target.iloc[train_idx], batch_size=32,
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=100,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model3_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Model 3, Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history3.history["loss"]), min(history3.history["val_loss"])))
 model1.load_weights("model1_fold" + str(fold_id)+ ".h5")
 model2.load_weights("model2_fold" + str(fold_id)+ ".h5")
 model3.load_weights("model3_fold" + str(fold_id)+ ".h5")
 pred1 = model1.predict(test)
 pred2 = model2.predict(test)
 pred3 = model3.predict(test)
 predictions.append(np.average([pred1, pred2, pred3], weights=[0.15, 0.7, 0.15], axis=0))
del data
gc.collect()<init_hyperparams>
pred = np.mean(predictions, axis=0)
 pred = np.clip(pred, 0.001, 0.999)
 pred.shape
params = {
'boosting_type': 'gbdt',
'metric': 'rmse',
'objective': 'regression',
'n_jobs': -1,
'seed': 236,
'learning_rate': 0.1}<train_model>
sub = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 sub.loc[:, 1:] = pred
sub.loc[ctlVehicle_test, sub.columns != "sig_id"] = 0
sub
<prepare_x_and_y><EOS> 
sub.to_csv("submission.csv", index=False )