Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<prepare_x_and_y>
%%HTML
<style type="text/css">
div.h1 {
background-color:
color: white;
padding: 8px;
padding-right: 300px;
font-size: 35px;
max-width: 1500px;
margin: auto;
margin-top: 50px;
}
div.h2 {
background-color:
color: white;
padding: 8px;
padding-right: 300px;
font-size: 35px;
max-width: 1500px;
margin: auto;
margin-top: 50px;
}
</style>
cols=["target","ID_code"]
X = train.drop(cols,axis=1)
 y = train["target"]<drop_column>
warnings.filterwarnings('ignore' )
X_test = test.drop("ID_code",axis=1 )<init_hyperparams>
from sklearn.preprocessing import QuantileTransformer
params = {'objective' : "binary",
'boost':"gbdt",
'metric':"auc",
'boost_from_average':"false",
'num_threads':8,
'learning_rate' : 0.01,
'num_leaves' : 13,
'max_depth':-1,
'tree_learner' : "serial",
'feature_fraction' : 0.05,
'bagging_freq' : 5,
'bagging_fraction' : 0.4,
'min_data_in_leaf' : 80,
'min_sum_hessian_in_leaf' : 10.0,
'verbosity' : 1}<train_model>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
%%time
y_pred_lgb = np.zeros(len(X_test))
num_round = 1000000
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 lgb_model = lgb.train(params,train_data,num_round,
valid_sets = [train_data, valid_data],verbose_eval=1000,early_stopping_rounds = 3500)
 y_pred_lgb += lgb_model.predict(X_test, num_iteration=lgb_model.best_iteration)/5<save_to_csv>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
submission_lgb = pd.DataFrame({
"ID_code": test["ID_code"],
"target": y_pred_lgb
})
 submission_lgb.to_csv('submission_lgb.csv', index=False )<set_options>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
warnings.filterwarnings('ignore')
 init_notebook_mode() 
<set_options>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
%time
gc.enable()
seed = 127
np.random.seed(seed )<load_from_csv>
train_targets_scored.sum() [1:].sort_values()
%time
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<feature_engineering>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
def missingcount(count, samples):

 return(int(count)/samples)* 100
def missingPercent(df):

 count_df = pd.DataFrame(df.isnull().sum(axis=0 ).reset_index())
 count_df.columns = ['FEATURE', 'MISSING_COUNT']
count_df["MISSING_%"] = count_df["MISSING_COUNT"].map(lambda x: missingcount(x, df.shape[0]))
return count_df.set_index('FEATURE' ).T<count_missing_values>
n_comp = 15
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
missingPercent(train_df )<count_unique_values>
var_thresh = VarianceThreshold(threshold=0.5)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features
labels=[str(x)for x in list(train_df.target.unique())]
values=[(len(train_df[train_df.target == 0])/len(train_df)) *100,(len(train_df[train_df.target > 0])/len(train_df)) *100]
sample_percentage_per_class(labels,values )<choose_model_class>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
rf = RandomForestClassifier(max_depth=10, verbose=3, n_estimators=20)
 parameters = {'n_estimators': [120, 100], 'max_depth':[3,5,]}
grid = GridSearchCV(rf, parameters, cv=3, n_jobs=-1, verbose=3, scoring=make_scorer(roc_auc_score))<train_model>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
df_train_FS=train_df
grid.fit(df_train_FS.drop(["target",'ID_code'], axis=1 ).values, df_train_FS.target.values )<find_best_score>
grid.best_score_<prepare_x_and_y>
X=df_train_FS.drop(["target",'ID_code'], axis=1)
 Y=df_train_FS.target<split>
x_train,x_test,y_train, y_test = train_test_split(X, Y, random_state=41)
 rfmodel=RandomForestClassifier(max_depth=10, n_estimators=20,verbose=0 ).fit(x_train, y_train )<train_model>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
perm = PermutationImportance(rfmodel, random_state=41, scoring=make_scorer(roc_auc_score)).fit(x_test,y_test )<load_pretrained>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=5)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
eli5.show_weights(perm, feature_names = x_test.columns.tolist() , top=80 )<prepare_x_and_y>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
y = train_df.target
train_df= train_df.iloc[:, 2:]<create_dataframe>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
ids = test_df.ID_code.to_frame()
test_df.drop('ID_code', inplace=True, axis=1 )<feature_engineering>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.5)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.5)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
%%time
idx = features = train_df.columns.values[2:202]
for df in [test_df, train_df]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )<normalization>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
sc = StandardScaler(seed, with_std=True)
 sc.fit(train_df)
 train_df= sc.transform(train_df)
 test_df = sc.transform(test_df )<init_hyperparams>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
params = {
'tree_learner': 'serial',
'objective': 'binary',
'learning_rate': 0.00742,
'num_leaves': 3,
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':True,
'boost': 'gbrt',
'feature_fraction': 0.04,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 85,
'min_sum_hessian_in_leaf': 0.80,
'num_threads': 16,
'verbosity': 1
}<split>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024
num_round = 250000
folds = StratifiedKFold(n_splits=7, shuffle=True, random_state=seed)
 lstCV=folds.split(train_df, y )<define_variables>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
train_pred = np.zeros(len(train_df))
test_pred = np.zeros(len(test_df))<predict_on_test>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
%time
for fold_,(trn_idx, val_idx)in enumerate(lstCV):
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df[trn_idx], label=y[trn_idx])
 val_data = lgb.Dataset(train_df[val_idx], label=y[val_idx])
 clf = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3500)
 train_pred[val_idx] = clf.predict(train_df[val_idx], num_iteration=clf.best_iteration)
 test_pred += clf.predict(test_df, num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(y, train_pred)) )<save_to_csv>
SEED = [0, 1, 2, 3 ,4, 5]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
sub_df = pd.DataFrame({"ID_code": ids["ID_code"]})
 sub_df["target"] = test_pred
sub_df.to_csv("submission.csv", index=False )<load_from_csv>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
sub_2 = pd.read_csv(".. /input/santa-kag1/lgb.csv",names=["ID_code","target"],skiprows=[0],header=None)
 sub_4 = pd.read_csv(".. /input/santa-kag1/submission_lgb.csv",names=["ID_code","target3"],skiprows=[0],header=None)
 sub_5 = pd.read_csv(".. /input/santa-kag1/submission.csv",names=["ID_code","target4"],skiprows=[0],header=None)
 sub_6 = pd.read_csv(".. /input/santa-kag1/submission_xgb1.csv",names=["ID_code","target5"],skiprows=[0],header=None)
 sub_7 = pd.read_csv(".. /input/santa-kag1/stcp2.csv",names=["ID_code","target6"],skiprows=[0],header=None)
 sub_8 = pd.read_csv(".. /input/santa-kag1/toy_sub.csv",names=["ID_code","target7"],skiprows=[0],header=None)
 sub_9 = pd.read_csv(".. /input/santa-kag1/LightGBM Minimize Leaves with GaussianNB.csv",names=["ID_code","target8"],skiprows=[0],header=None)
 sub_10 = pd.read_csv(".. /input/santa-kag1/lgbsubmission1111.csv",names=["ID_code","target9"],skiprows=[0],header=None)
 sub_11 = pd.read_csv(".. /input/santa-kag1/submission111.csv",names=["ID_code","target10"],skiprows=[0],header=None)
 sub_13 = pd.read_csv(".. /input/santa-kag1/submission_lgb_cat.csv",names=["ID_code","target12"],skiprows=[0],header=None)
 sub_14 = pd.read_csv(".. /input/santa-kag1/Customer_Transaction_rank_predictions.csv",names=["ID_code","target13"],skiprows=[0],header=None)
 sub_15 = pd.read_csv(".. /input/santa-kag1/submission_lgb1.csv",names=["ID_code","target14"],skiprows=[0],header=None)
 sub_16 = pd.read_csv(".. /input/santa-kag1/submission1.csv",names=["ID_code","target15"],skiprows=[0],header=None )<save_to_csv>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )