Kaggle competition: Mechanisms of Action (MoA) Prediction
param_grid_logreg = {'penalty':['elasticnet'],
'C':[0.03],
'l1_ratio':[0.0],
'solver':['saga']}<choose_model_class>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn
import seaborn as sns
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression, Perceptron, RidgeCV, Lasso
from sklearn.decomposition import PCA
from sklearn.metrics import log_loss
from sklearn.model_selection import train_test_split, KFold
from sklearn.feature_selection import VarianceThreshold
import torch
from torch import optim
from torch import nn
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F
grid_logreg = GridSearchCV(LogisticRegression() , param_grid_logreg,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )<train_model>
train_X = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_y = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train_y_ns = pd.read_csv("/kaggle/input/lish-moa/train_targets_nonscored.csv")
 test_X = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 print(test_X.shape )
grid_logreg.fit(X_train_fe, y_train )<define_search_space>
train_ctl_vehicle_idx = train_X[train_X['cp_type']=='ctl_vehicle'].index
test_ctl_vehicle_idx = test_X[test_X['cp_type'] == 'ctl_vehicle'].index
train_X = train_X.drop(train_ctl_vehicle_idx ).reset_index(drop=True)
 train_y = train_y.drop(train_ctl_vehicle_idx ).reset_index(drop=True)
 test_X = test_X.drop(test_ctl_vehicle_idx ).reset_index(drop=True)
 train_X, test_X = train_X.drop('cp_type', axis=1), test_X.drop('cp_type', axis=1 )
param_grid_knn = {'n_neighbors':[20],
'weights':['uniform'],
'algorithm':['ball_tree']}<choose_model_class>
g_features = [col for col in train_X.columns if col.startswith('g-')]
c_features = [col for col in train_X.columns if col.startswith('c-')]
grid_knn = GridSearchCV(KNeighborsClassifier() , param_grid_knn,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )<train_model>
def drop_cp(df):
return df[df['cp_type'] == 'trt_cp'].reset_index(drop=True)
grid_knn.fit(X_train_fe, y_train )<define_search_space>
test_ids = test_X.sig_id
param_grid_svc = {'C':[0.5],
'kernel':['rbf'],
'gamma':[0.1]}<choose_model_class>
def normal_split(train_X, train_y):
X_n_columns = len(train_X.columns)
 y_n_columns = len(train_y.columns)
 train_X_y = pd.merge(train_X, train_y ,on='sig_id')
 train_X = train_X_y.iloc[:, :X_n_columns]
train_y = train_X_y.iloc[:, X_n_columns:]
return train_X, train_y
train_X, train_y = normal_split(train_X, train_y)
 train_y.shape
grid_svc = GridSearchCV(SVC() , param_grid_svc, cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )<train_model>
def preprocess(dataFrames, cat_features, con_features):
new_dfs = []
dataFrames_cat = [ df[cat_features] for df in dataFrames]
dataFrames_con = [ df[con_features] for df in dataFrames]
dataFrames_con = filter_variance(dataFrames_con)
 print("Variance filtered")
 print("High correlated features dropped")
 dataFrames_con = add_PCA(dataFrames_con, 30, 10)
 print("PCA added")
 dataFrames_con = add_new(dataFrames_con)
 print("Additional featires added")
 dataFrames_cat = map_categorical(dataFrames_cat)
 print("Categorical features transformed")
 for df_cat, df_con in zip(dataFrames_cat, dataFrames_con):
new_dfs.append(pd.concat(( df_cat, df_con), axis=1))
return new_dfs
grid_svc.fit(X_train_fe, y_train )<define_search_space>
param_grid_random = {'n_estimators':[200, 500],
'max_depth':[5, 9],
'max_samples':[0.5, 0.7],
'max_features':[0.5, 0.7],
'min_samples_split':[2, 5, 8]}<choose_model_class>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 class MoADataset(Dataset):
def __init__(self, X, y=None):
if y is not None:
assert X.shape[0] == y.shape[0]
self.X = torch.tensor(X.to_numpy() , dtype=torch.float)
 self.y = None
if y is not None:
self.y = torch.tensor(y.to_numpy() , dtype=torch.float)
 def __getitem__(self, idx):
tensor_X = self.X[idx]
if self.y is not None:
tensor_y = self.y[idx]
return tensor_X, tensor_y
else:
return tensor_X
def __len__(self):
return len(self.X )
grid_random = GridSearchCV(RandomForestClassifier() , param_grid_random,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=4 )<train_model>
dataset = MoADataset(train_X, train_y)
 train_dataloader = DataLoader(dataset, batch_size=10 )
grid_random.fit(X_train_fe, y_train )<define_search_space>
class MoANet(nn.Module):
def __init__(self, input_size, output_size):
super().__init__()
self.input_size = input_size
self.output_size = output_size
self.hidden_size = 256
self.wn = nn.utils.weight_norm
self.mlp = nn.Sequential(
 self.wn(nn.Linear(input_size, self.hidden_size)) ,
nn.Dropout(0.3),
nn.ELU() ,
self.wn(nn.Linear(self.hidden_size, self.hidden_size)) ,
nn.Dropout(0.3),
nn.ELU() ,
self.wn(nn.Linear(self.hidden_size, self.output_size)))
 def forward(self, x):
x = self.mlp(x)
 return x
net = MoANet(train_X.shape[1], train_y.shape[1] )
param_grid_gradient = {'max_depth':[3],
'n_estimators':[300, 400, 500],
'learning_rate':[0.035, 0.055],
'subsample':[0.4, 0.6],
'max_features':[0.4, 0.6],
'min_samples_split':[2, 5, 8, 12]
}<choose_model_class>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x =F.leaky_relu(self.dense2(x), 1e-3)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
grid_gradient = GridSearchCV(GradientBoostingClassifier() , param_grid_gradient,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )<train_model>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for X, y in dataloader:
optimizer.zero_grad()
X, y = X.to(device), y.to(device)
 outputs = model(X)
 loss = loss_fn(outputs, y)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for X, y in dataloader:
X, y = X.to(device), y.to(device)
 outputs = model(X)
 loss = loss_fn(outputs, y)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for X in dataloader:
X = X.to(device)
 with torch.no_grad() :
outputs = model(X)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
grid_gradient.fit(X_train_fe, y_train )<define_search_space>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
HIDDEN_SIZE=2048
param_grid_xgb = {'n_estimators':[300, 450],
'learning_rate':[0.02, 0.03],
'max_depth':[6],
'subsample':[0.5, 0.7],
'colsample_bylevel':[0.5, 0.7],
'reg_lambda':[1, 5, 15, ]
}<choose_model_class>
def run_training(fold_number, train_X, train_y, test_X, train_indexes, val_indexes, seed):
x_train = train_X.iloc[train_indexes].reset_index(drop=True)
 y_train = train_y.iloc[train_indexes].reset_index(drop=True)
 x_val = train_X.iloc[val_indexes].reset_index(drop=True)
 y_val = train_y.iloc[val_indexes].reset_index(drop=True)
 train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_val, y_val)
 trainloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=train_X.shape[-1],
num_targets=train_y.shape[-1],
hidden_size=HIDDEN_SIZE, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e1,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( train_X.shape[0], train_y.shape[-1]))
best_loss = np.inf
print(f"FOLD: {fold_number}")
 for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
early_step =0
best_loss = valid_loss
oof[val_indexes] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold_number}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
testdataset = MoADataset(test_X)
 testloader = DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=train_X.shape[-1],
num_targets=train_y.shape[-1],
hidden_size=HIDDEN_SIZE, 
)
 model.load_state_dict(torch.load(f"FOLD{fold_number}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_X), train_y.shape[-1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
grid_xgb = GridSearchCV(XGBClassifier() , param_grid_xgb,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=2, random_state=42),
scoring='accuracy', verbose=2, n_jobs=-1 )<train_model>
def run_k_fold(NFOLDS, train_X, train_y, test_X, seed):
oof = np.zeros(( train_y.shape))
predictions = np.zeros(( test_X.shape[0], train_y.shape[-1]))
kf = KFold(n_splits=NFOLDS, shuffle=True, random_state=seed)
 for fold,(train_indexes, val_indexes)in enumerate(kf.split(train_X)) :
oof_, pred_ = run_training(fold, train_X, train_y, test_X, train_indexes, val_indexes, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
grid_xgb.fit(X_train_fe, y_train )<choose_model_class>
SEED = [1973, 1998]
oof = np.zeros(( train_X.shape[0], train_y.shape[-1]))
predictions = np.zeros(( test_X.shape[0], train_y.shape[-1]))
for seed in SEED:
print(f"SEED: {seed}")
 oof_, predictions_ = run_k_fold(NFOLDS, train_X, train_y, test_X, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
logreg = LogisticRegression(**{'C': 0.03, 'l1_ratio': 0, 'penalty': 'elasticnet', 'solver': 'saga'})
 svc = SVC(**{'C': 0.5, 'gamma': 0.1, 'kernel': 'rbf'})
 knn = KNeighborsClassifier(**{'algorithm': 'ball_tree', 'n_neighbors': 20, 'weights': 'uniform'})
 rfc = RandomForestClassifier(**{'max_depth': 5, 'max_features': 0.5, 'max_samples': 0.7,
'min_samples_split': 5, 'n_estimators': 500})
 gradient = GradientBoostingClassifier(**{'learning_rate': 0.055, 'max_depth': 3, 'max_features': 0.4,
'min_samples_split': 2, 'n_estimators': 400, 'subsample': 0.4})
 xgb = XGBClassifier(**{'colsample_bylevel': 0.5, 'learning_rate': 0.03, 'max_depth': 6,
'n_estimators': 300, 'reg_lambda': 5, 'subsample': 0.7})
 estimators = [('logreg', logreg),('knn', knn),('svc', svc),('rfc', rfc),('gradient', gradient), 
('xgb', xgb)]
stack = StackingClassifier(estimators=estimators,
cv=10, n_jobs=-1 )<train_model>
train_prediction_df= pd.DataFrame(data=oof, columns=train_y.columns)
 test_prediction_df = pd.DataFrame(data=predictions, columns=train_y.columns )
stack.fit(X_train_fe, y_train )<predict_on_test>
train_ctl_vehicle_idx
test_ctl_vehicle_idx
tts = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 tstf = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 print(tts.shape)
 dropped_sig_id_train = pd.DataFrame(tts.iloc[:,0].drop(train_ctl_vehicle_idx, axis=0 ).reset_index(drop=True), columns=['sig_id'])
 train_prediction_df = pd.concat([dropped_sig_id_train, train_prediction_df], axis=1)
 valid_results = tts.drop(columns=train_y.columns ).merge(train_prediction_df, on='sig_id', how='left' ).fillna(0)
 y_true = tts[train_y.columns].values
y_pred = valid_results[train_y.columns].values
score = 0
for i in range(y_true.shape[1]):
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_
score /= train_y.shape[1]
print("CV log_loss: ", score)
y_preds = stack.predict(X_test_fe )<create_dataframe>
dropped_sig_id_test = pd.DataFrame(tstf.iloc[:,0].drop(test_ctl_vehicle_idx, axis=0 ).reset_index(drop=True), columns=['sig_id'])
 test_prediction_df = pd.concat([dropped_sig_id_test, test_prediction_df], axis=1)
 test_results = tstf.drop(columns=tstf.columns[1:] ).merge(test_prediction_df, on='sig_id', how='left' ).fillna(0)
 test_results.shape
submission = pd.DataFrame({'PassengerId':test.index,
'Survived':y_preds} )<save_to_csv>
test_results.to_csv("submission.csv", index=None )
submission.to_csv('submission.csv', index=False )<save_to_csv>
submission.to_csv('submission.csv', index=False )<load_from_csv>
pd.read_csv('submission.csv' )<load_from_csv>
def one_to_all_reg(reg_model, train_X, train_y):
models = {}
for idx, target_name in enumerate(train_y.columns):
print("Train
targets = train_y[target_name]
reg_model.fit(train_X, targets)
 models[target_name] = reg_model
return models
def predict_ensemble(test_X, models, targets_names):
df_prediction = pd.DataFrame()
for target_name in targets_names:
df_prediction[target_name] = models[target_name].predict(test_X)
 return df_prediction
def create_equal_ordering(pred_df, true_df):
assert pred_df.shape == true_df.shape
numpy_pred = np.zeros(pred_df.shape)
 numpy_true = np.zeros(true_df.shape)
 for idx, target_name in enumerate(true_df.columns):
numpy_pred[:, idx] = pred_df[target_name].to_numpy()
numpy_true[:, idx] = true_df[target_name].to_numpy()
return numpy_pred, numpy_true
<load_from_csv><EOS>