Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<categorify>
pd.set_option('max_columns', None)
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
def initialize_from_past_model(model,past_model_file):
pretrained_dict = torch.load(past_model_file)
 model_dict = model.state_dict()
pretrained_dict['dense3.bias']=pretrained_dict['dense3.bias'][:206]
pretrained_dict['dense3.weight_g']=pretrained_dict['dense3.weight_g'][:206]
pretrained_dict['dense3.weight_v']=pretrained_dict['dense3.weight_v'][:206]
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
model_dict.update(pretrained_dict)
 model.load_state_dict(pretrained_dict)
<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.shape
def run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose=False,**kwargs):
seed_everything(seed)
 train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],
hidden_size=hidden_size,**kwargs 
)
 model.to(DEVICE)
 initialize_from_past_model(model,f".. /input/pytorchauxtargets5f1/FOLD{fold}_original_torch_moa_5_folds_AUX.pth")
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)
 loss_val = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_{exp_name}.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<define_variables>
train.isnull().any().any()
def run_k_fold(folds,target_cols,test,NFOLDS, seed,verbose=False,**kwargs):
train = preprocess_data(folds)
 test_ = preprocess_data(test)
 oof = train[target_cols].copy()
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols.to_list() +['kfold'])]
X_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
X_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
X_test = test_[feature_cols].values
oof_, pred_ = run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose,**kwargs)
 oof[train['kfold'] == fold] = oof_
predictions += pred_ / NFOLDS
return oof, predictions<init_hyperparams>
train['target'].value_counts()
params ={'drop_rate1':0.5,'drop_rate2':0.2,'drop_rate3':0.2}<load_from_csv>
corrs = train.corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
corrs = corrs[corrs['level_0'] != corrs['level_1']]
corrs.tail(10 )
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
folds = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 oof_, predictions_ = run_k_fold(folds,target_cols,test,NFOLDS, seed,verbose=True,**params)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 test[target_cols] = predictions 
<compute_test_metric>
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 n_fold = 20
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42)
 repeated_folds = RepeatedStratifiedKFold(n_splits=20, n_repeats=20, random_state=42)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
y_true = train[target_cols].values
y_pred = oof
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print("CV log_loss: ", score)
<save_to_csv>
def train_model(X, X_test, y, params, folds=folds, model_type='lgb', plot_feature_importance=False, averaging='usual', model=None):
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
X_train, X_valid = X[train_index], X[valid_index]
y_train, y_valid = y[train_index], y[valid_index]
if model_type == 'lgb':
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,
train_data,
num_boost_round=2000,
valid_sets = [train_data, valid_data],
verbose_eval=500,
early_stopping_rounds = 200)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X_tr.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X_tr.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=500, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_tr.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=20000, eval_metric='AUC', **params)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = roc_auc_score(y_valid, y_pred_valid)
 y_pred = model.predict_proba(X_test)[:, 1]
if model_type == 'glm':
model = sm.GLM(y_train, X_train, family=sm.families.Binomial())
 model_results = model.fit()
model_results.predict(X_test)
 y_pred_valid = model_results.predict(X_valid ).reshape(-1,)
 score = roc_auc_score(y_valid, y_pred_valid)
 y_pred = model_results.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 scores.append(roc_auc_score(y_valid, y_pred_valid))
if averaging == 'usual':
prediction += y_pred
elif averaging == 'rank':
prediction += pd.Series(y_pred ).rank().values
if model_type == 'lgb':
fold_importance = pd.DataFrame()
fold_importance["feature"] = X.columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_fold
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 if model_type == 'lgb':
feature_importance["importance"] /= n_fold
if plot_feature_importance:
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
return oof, prediction, feature_importance
return oof, prediction, scores
else:
return oof, prediction, scores
sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission.set_index('sig_id',inplace=True)
 test_features.set_index('sig_id',inplace=True)
 test_features = test_features.loc[sample_submission.index]
sub = sample_submission.drop(columns=target_cols ).merge(test[target_cols], on='sig_id', how='left' ).fillna(0)
 sub.loc[test_features['cp_type']=='ctl_vehicle', target_cols] =0
sub.to_csv('./submission.csv', index=True )<set_options>
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 warnings.filterwarnings('ignore' )<define_variables>
cat_params = {'learning_rate': 0.02,
'depth': 5,
'l2_leaf_reg': 10,
'bootstrap_type': 'Bernoulli',
'od_type': 'Iter',
'od_wait': 50,
'random_seed': 11,
'allow_writing_files': False}
oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=cat_params, model_type='cat' )
FOLDS = 10
EPOCHS = 80
BATCH_SIZE = 124
LR = 0.001
VERBOSE = 0
SEEDS1 = [1, 2, 3, 4, 5, 6, 7]
SEEDS2 = [8, 9, 10, 11, 12, 13, 14]
SEEDS3 = [15, 16, 17, 18, 19, 20, 21]
SEEDS4 = [22, 23, 24, 25, 26, 27, 28]
SEEDS5 = [29, 30, 31, 32, 33, 34, 35]
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed )<categorify>
model = CatBoostClassifier(iterations=400, eval_metric='AUC', **cat_params)
 model.fit(X_train, y_train, cat_features=[], use_best_model=True, verbose=200 )
def mapping_and_filter(train, train_targets, test):
cp_type = {'trt_cp': 0, 'ctl_vehicle': 1}
cp_dose = {'D1': 0, 'D2': 1}
for df in [train, test]:
df['cp_type'] = df['cp_type'].map(cp_type)
 df['cp_dose'] = df['cp_dose'].map(cp_dose)
 train_targets = train_targets[train['cp_type'] == 0].reset_index(drop = True)
 train = train[train['cp_type'] == 0].reset_index(drop = True)
 train_targets.drop(['sig_id'], inplace = True, axis = 1)
 return train, train_targets, test
def scaling(train, test):
features = train.columns[2:]
scaler = RobustScaler()
scaler.fit(pd.concat([train[features], test[features]], axis = 0))
train[features] = scaler.transform(train[features])
 test[features] = scaler.transform(test[features])
 return train, test, features
def fe_pca(train, test, n_components_g = 70, n_components_c = 10, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_pca(train, test, features, kind = 'g', n_components = n_components_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 pca = PCA(n_components = n_components, random_state = SEED)
 data = pca.fit_transform(data)
 columns = [f'pca_{kind}{i + 1}' for i in range(n_components)]
data = pd.DataFrame(data, columns = columns)
 train_ = data.iloc[:train.shape[0]]
test_ = data.iloc[train.shape[0]:].reset_index(drop = True)
 train = pd.concat([train, train_], axis = 1)
 test = pd.concat([test, test_], axis = 1)
 return train, test
train, test = create_pca(train, test, features_g, kind = 'g', n_components = n_components_g)
 train, test = create_pca(train, test, features_c, kind = 'c', n_components = n_components_c)
 return train, test
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in [train, test]:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
def c_squared(train, test):
features_c = list(train.columns[776:876])
 for df in [train, test]:
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
def mean_log_loss(y_true, y_pred):
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 metrics = []
for target in range(206):
metrics.append(log_loss(y_true[:, target], y_pred[:, target]))
return np.mean(metrics)
 def create_model_rs(shape1, shape2):
input_1 = tf.keras.layers.Input(shape =(shape1))
input_2 = tf.keras.layers.Input(shape =(shape2))
head_1 = tf.keras.layers.BatchNormalization()(input_1)
 head_1 = tf.keras.layers.Dropout(0.2 )(head_1)
 head_1 = tf.keras.layers.Dense(512, activation = "elu" )(head_1)
 head_1 = tf.keras.layers.BatchNormalization()(head_1)
 input_3 = tf.keras.layers.Dense(256, activation = "elu" )(head_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.layers.BatchNormalization()(input_3_concat)
 head_2 = tf.keras.layers.Dropout(0.3 )(head_2)
 head_2 = tf.keras.layers.Dense(512, "relu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 head_2 = tf.keras.layers.Dense(512, "elu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 head_2 = tf.keras.layers.Dense(256, "relu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 input_4 = tf.keras.layers.Dense(256, "elu" )(head_2)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.layers.BatchNormalization()(input_4_avg)
 head_3 = tf.keras.layers.Dense(256, kernel_initializer = 'lecun_normal', activation = 'selu' )(head_3)
 head_3 = tf.keras.layers.BatchNormalization()(head_3)
 head_3 = tf.keras.layers.Dense(206, kernel_initializer = 'lecun_normal', activation = 'selu' )(head_3)
 head_3 = tf.keras.layers.BatchNormalization()(head_3)
 output = tf.keras.layers.Dense(206, activation = "sigmoid" )(head_3)
 model = tf.keras.models.Model(inputs = [input_1, input_2], outputs = output)
 opt = tf.optimizers.Adam(learning_rate = LR)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_5l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2560, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2048, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1524, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(780, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(206, activation = 'sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0020),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_4l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2048, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1524, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(206, activation = 'sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0020),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_3l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4914099166744246 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1159, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.18817607797795838 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(960, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.12542057776853896 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1811, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.20175242230280122 )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.Lookahead(opt, sync_period = 10)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_2l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.2688628097505064 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1292, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4598218403250696 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(983, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4703144018483698 )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.Lookahead(opt, sync_period = 10)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def train_and_evaluate(train, test, train_targets, features, start_predictors, SEED = 123, MODEL = '3l'):
seed_everything(SEED)
 oof_pred = np.zeros(( train.shape[0], 206))
test_pred = np.zeros(( test.shape[0], 206))
for fold,(trn_ind, val_ind)in enumerate(MultilabelStratifiedKFold(n_splits = FOLDS,
random_state = SEED,
shuffle = True)\ 
.split(train_targets, train_targets)) :
K.clear_session()
if MODEL == '5l':
model = create_model_5l(len(features))
elif MODEL == '4l':
model = create_model_4l(len(features))
elif MODEL == '3l':
model = create_model_3l(len(features))
elif MODEL == '2l':
model = create_model_2l(len(features))
elif MODEL == "rs":
model = create_model_rs(len(features), len(start_predictors))
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_binary_crossentropy',
mode = 'min',
patience = 10,
restore_best_weights = False,
verbose = VERBOSE)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_binary_crossentropy',
mode = 'min',
factor = 0.3,
patience = 3,
verbose = VERBOSE)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(f'{MODEL}_{fold}_{SEED}.h5',
monitor = 'val_binary_crossentropy',
verbose = VERBOSE,
save_best_only = True,
save_weights_only = True)
 x_train, x_val = train[features].values[trn_ind], train[features].values[val_ind]
y_train, y_val = train_targets.values[trn_ind], train_targets.values[val_ind]
if MODEL == "rs":
x_train_, x_val_ = train[start_predictors].values[trn_ind], train[start_predictors].values[val_ind]
model.fit([x_train, x_train_], y_train,
validation_data =([x_val, x_val_], y_val),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr, checkpoint],
verbose = VERBOSE)
 model.load_weights(f'{MODEL}_{fold}_{SEED}.h5')
 oof_pred[val_ind] = model.predict([x_val, x_val_])
 test_pred += model.predict([test[features].values, test[start_predictors].values])/ FOLDS
else:
model.fit(x_train, y_train,
validation_data =(x_val, y_val),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr, checkpoint],
verbose = VERBOSE)
 model.load_weights(f'{MODEL}_{fold}_{SEED}.h5')
 oof_pred[val_ind] = model.predict(x_val)
 test_pred += model.predict(test[features].values)/ FOLDS
oof_score = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds mean log loss score is {oof_score}')
 return test_pred, oof_pred
def inference(train, test, train_targets, features, start_predictors, SEED = 123, MODEL = '3l', PATH = '.. /input/moa-3layer'):
seed_everything(SEED)
 oof_pred = np.zeros(( train.shape[0], 206))
test_pred = np.zeros(( test.shape[0], 206))
for fold,(trn_ind, val_ind)in enumerate(MultilabelStratifiedKFold(n_splits = FOLDS,
random_state = SEED,
shuffle = True)\ 
.split(train_targets, train_targets)) :
K.clear_session()
if MODEL == '5l':
model = create_model_5l(len(features))
elif MODEL == '4l':
model = create_model_4l(len(features))
elif MODEL == '3l':
model = create_model_3l(len(features))
elif MODEL == '2l':
model = create_model_2l(len(features))
elif MODEL == "rs":
model = create_model_rs(len(features), len(start_predictors))
x_train, x_val = train[features].values[trn_ind], train[features].values[val_ind]
y_train, y_val = train_targets.values[trn_ind], train_targets.values[val_ind]
model.load_weights(f'{PATH}/{MODEL}_{fold}_{SEED}.h5')
 if MODEL == "rs":
x_train_, x_val_ = train[start_predictors].values[trn_ind], train[start_predictors].values[val_ind]
oof_pred[val_ind] = model.predict([x_val, x_val_])
 test_pred += model.predict([test[features].values, test[start_predictors].values])/ FOLDS
else:
oof_pred[val_ind] = model.predict(x_val)
 test_pred += model.predict(test[features].values)/ FOLDS
oof_score = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds mean log loss score is {oof_score}')
 return test_pred, oof_pred
def run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = [123], MODEL = '3l', PATH = '.. /input/moa-3layer'):
test_pred = []
oof_pred = []
for SEED in SEEDS:
print(f'Using model {MODEL} with seed {SEED} for inference')
 print(f'Trained with {len(features)} features')
 test_pred_, oof_pred_ = inference(train, test, train_targets, features, start_predictors, SEED = SEED, MODEL = MODEL, PATH = PATH)
 test_pred.append(test_pred_)
 oof_pred.append(oof_pred_)
 print('-'*50)
 print('
')
 test_pred = np.average(test_pred, axis = 0)
 oof_pred = np.average(oof_pred, axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds log loss for our seed blend model is {seed_log_loss}')
 return test_pred, oof_pred
def submission(test_pred):
sample_submission.loc[:, train_targets.columns] = test_pred
sample_submission.loc[test['cp_type'] == 1, train_targets.columns] = 0
sample_submission.to_csv('submission.csv', index = False)
 return sample_submission
start_predictors = ["g-0", "g-7", "g-8", "g-10", "g-13", "g-17", "g-20", "g-22", "g-24", "g-26", "g-28", "g-29", "g-30", "g-31", "g-32", "g-34", "g-35", "g-36", "g-37", "g-38",
"g-39","g-41", "g-46", "g-48", "g-50", "g-51", "g-52", "g-55", "g-58", "g-59", "g-61", "g-62", "g-63", "g-65", "g-66", "g-67", "g-68", "g-70", "g-72", "g-74",
"g-75", "g-79", "g-83", "g-84", "g-85", "g-86", "g-90", "g-91", "g-94", "g-95", "g-96", "g-97", "g-98", "g-100", "g-102", "g-105", "g-106", "g-112", "g-113",
"g-114", "g-116", "g-121", "g-123", "g-126", "g-128", "g-131", "g-132", "g-134", "g-135", "g-138", "g-139", "g-140", "g-142", "g-144", "g-145", "g-146",
"g-147", "g-148", "g-152", "g-155", "g-157", "g-158", "g-160", "g-163", "g-164", "g-165", "g-170", "g-173", "g-174", "g-175", "g-177", "g-178", "g-181",
"g-183", "g-185", "g-186", "g-189", "g-192", "g-194", "g-195", "g-196", "g-197", "g-199", "g-201", "g-202", "g-206", "g-208", "g-210", "g-213", "g-214",
"g-215", "g-220", "g-226", "g-228", "g-229", "g-235", "g-238", "g-241", "g-242", "g-243", "g-244", "g-245", "g-248", "g-250", "g-251", "g-254", "g-257",
"g-259", "g-261", "g-266", "g-270", "g-271", "g-272", "g-275", "g-278", "g-282", "g-287", "g-288", "g-289", "g-291", "g-293", "g-294", "g-297", "g-298",
"g-301", "g-303", "g-304", "g-306", "g-308", "g-309", "g-310", "g-311", "g-314", "g-315", "g-316", "g-317", "g-320", "g-321", "g-322", "g-327", "g-328",
"g-329", "g-332", "g-334", "g-335", "g-336", "g-337", "g-339", "g-342", "g-344", "g-349", "g-350", "g-351", "g-353", "g-354", "g-355", "g-357", "g-359",
"g-360", "g-364", "g-365", "g-366", "g-367", "g-368", "g-369", "g-374", "g-375", "g-377", "g-379", "g-385", "g-386", "g-390", "g-392", "g-393", "g-400",
"g-402", "g-406", "g-407", "g-409", "g-410", "g-411", "g-414", "g-417", "g-418", "g-421", "g-423", "g-424", "g-427", "g-429", "g-431", "g-432", "g-433",
"g-434", "g-437", "g-439", "g-440", "g-443", "g-449", "g-458", "g-459", "g-460", "g-461", "g-464", "g-467", "g-468", "g-470", "g-473", "g-477", "g-478",
"g-479", "g-484", "g-485", "g-486", "g-488", "g-489", "g-491", "g-494", "g-496", "g-498", "g-500", "g-503", "g-504", "g-506", "g-508", "g-509", "g-512",
"g-522", "g-529", "g-531", "g-534", "g-539", "g-541", "g-546", "g-551", "g-553", "g-554", "g-559", "g-561", "g-562", "g-565", "g-568", "g-569", "g-574",
"g-577", "g-578", "g-586", "g-588", "g-590", "g-594", "g-595", "g-596", "g-597", "g-599", "g-600", "g-603", "g-607", "g-615", "g-618", "g-619", "g-620",
"g-625", "g-628", "g-629", "g-632", "g-634", "g-635", "g-636", "g-638", "g-639", "g-641", "g-643", "g-644", "g-645", "g-646", "g-647", "g-648", "g-663",
"g-664", "g-665", "g-668", "g-669", "g-670", "g-671", "g-672", "g-673", "g-674", "g-677", "g-678", "g-680", "g-683", "g-689", "g-691", "g-693", "g-695",
"g-701", "g-702", "g-703", "g-704", "g-705", "g-706", "g-708", "g-711", "g-712", "g-720", "g-721", "g-723", "g-724", "g-726", "g-728", "g-731", "g-733",
"g-738", "g-739", "g-742", "g-743", "g-744", "g-745", "g-749", "g-750", "g-752", "g-760", "g-761", "g-764", "g-766", "g-768", "g-770", "g-771", "c-0",
"c-1", "c-2", "c-3", "c-4", "c-5", "c-6", "c-7", "c-8", "c-9", "c-10", "c-11", "c-12", "c-13", "c-14", "c-15", "c-16", "c-17", "c-18", "c-19", "c-20",
"c-21", "c-22", "c-23", "c-24", "c-25", "c-26", "c-27", "c-28", "c-29", "c-30", "c-31", "c-32", "c-33", "c-34", "c-35", "c-36", "c-37", "c-38", "c-39",
"c-40", "c-41", "c-42", "c-43", "c-44", "c-45", "c-46", "c-47", "c-48", "c-49", "c-50", "c-51", "c-52", "c-53", "c-54", "c-55", "c-56", "c-57", "c-58",
"c-59", "c-60", "c-61", "c-62", "c-63", "c-64", "c-65", "c-66", "c-67", "c-68", "c-69", "c-70", "c-71", "c-72", "c-73", "c-74", "c-75", "c-76", "c-77",
"c-78", "c-79", "c-80", "c-81", "c-82", "c-83", "c-84", "c-85", "c-86", "c-87", "c-88", "c-89", "c-90", "c-91", "c-92", "c-93", "c-94", "c-95", "c-96",
"c-97", "c-98", "c-99"]
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, train_targets, test = mapping_and_filter(train, train_targets, test)
 train, test = fe_stats(train, test)
 train, test = c_squared(train, test)
 train, test = fe_pca(train, test, n_components_g = 70, n_components_c = 10, SEED = 123)
 train, test, features = scaling(train, test)
 test_pred_5l, oof_pred_5l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS1, MODEL = '5l', PATH = '.. /input/moa-5layer')
 test_pred_4l, oof_pred_4l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS2, MODEL = '4l', PATH = '.. /input/moa-4layer')
 test_pred_3l, oof_pred_3l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS3, MODEL = '3l', PATH = '.. /input/moa-3layer')
 test_pred_2l, oof_pred_2l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS4, MODEL = '2l', PATH = '.. /input/moa-2layer')
 test_pred_rs, oof_pred_rs = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS5, MODEL = 'rs', PATH = '.. /input/moa-rs')
 oof_pred = np.average([oof_pred_5l, oof_pred_4l, oof_pred_3l, oof_pred_2l], axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our final out of folds log loss for our classic dnn blend is {seed_log_loss}')
 test_pred = np.average([test_pred_5l, test_pred_4l, test_pred_3l, test_pred_2l], axis = 0)
 oof_pred = np.average([oof_pred, oof_pred_rs], axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our final out of folds log loss for our classic dnn + dnn resnet type model is {seed_log_loss}')
 test_pred = np.average([test_pred, test_pred_rs], axis = 0)
 sample_submission = submission(test_pred)
 sample_submission.head()<install_modules>
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr_repeated, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds )
!pip install.. /input/iter-strat<set_options>
eli5.show_weights(model, top=50 )
ModelCheckpoint, EarlyStopping, LearningRateScheduler, ReduceLROnPlateau)
 %matplotlib inline
%config InlineBackend.figure_format = 'svg'<define_variables>
top_features = [i[1:] for i in eli5.formatters.as_dataframe.explain_weights_df(model ).feature if 'BIAS' not in i]
X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 SEED = 1984
seed_everything(SEED )<load_from_csv>
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
x_develop = pd.read_csv('.. /input/lish-moa/train_features.csv')
 y_develop = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 x_test= pd.read_csv('.. /input/lish-moa/test_features.csv')
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 c_cols = x_develop.columns[x_develop.columns.str.startswith('c-')]
g_cols = x_develop.columns[x_develop.columns.str.startswith('g-')]
cont_cols = g_cols.to_list() + c_cols.to_list()
target_cols = y_develop.columns[1:]
N_TARGETS = len(target_cols )<categorify>
perm = PermutationImportance(model, random_state=1 ).fit(X_train, y_train)
 eli5.show_weights(perm, top=50 )
def preprocess_df(df):
if 'sig_id' in df.columns:
df.set_index('sig_id', inplace=True)
 df['cp_type'] =(df['cp_type'] == 'trt_cp' ).astype(int)
 df['cp_dose'] =(df['cp_dose'] == 'D2' ).astype(int)
 df = df.join(pd.get_dummies(df['cp_time'], drop_first=False, prefix='cp_time'))
df = df.drop('cp_time', axis=1)
 return df<create_dataframe>
top_features = [i[1:] for i in eli5.formatters.as_dataframe.explain_weights_df(perm ).feature if 'BIAS' not in i]
X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
x_develop = preprocess_df(x_develop)
 y_develop = y_develop.set_index('sig_id')
 x_test = preprocess_df(x_test )<create_dataframe>
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr1, prediction_lr1, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
n_comp = 28
pca_g = PCA(n_components=n_comp, random_state=SEED)
 x_develop_pca = pca_g.fit_transform(x_develop[g_cols])
 x_develop_pca = pd.DataFrame(x_develop_pca, columns=[f'pca_g-{i}' for i in range(n_comp)], index=x_develop.index)
 x_develop = pd.concat(( x_develop, x_develop_pca), axis=1)
 x_test_pca = pca_g.transform(x_test[g_cols])
 x_test_pca = pd.DataFrame(x_test_pca, columns=[f'pca_g-{i}' for i in range(n_comp)], index=x_test.index)
 x_test = pd.concat(( x_test, x_test_pca), axis=1)
 cont_cols += [f'pca_g-{i}' for i in range(n_comp)]
n_comp = 5
pca_c = PCA(n_components=n_comp, random_state=SEED)
 x_develop_pca = pca_c.fit_transform(x_develop[c_cols])
 x_develop_pca = pd.DataFrame(x_develop_pca, columns=[f'pca_c-{i}' for i in range(n_comp)], index=x_develop.index)
 x_develop = pd.concat(( x_develop, x_develop_pca), axis=1)
 x_test_pca = pca_c.transform(x_test[c_cols])
 x_test_pca = pd.DataFrame(x_test_pca, columns=[f'pca_c-{i}' for i in range(n_comp)], index=x_test.index)
 x_test = pd.concat(( x_test, x_test_pca), axis=1)
 cont_cols += [f'pca_c-{i}' for i in range(n_comp)]<categorify>
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
class VarianceThreshold:
def __init__(self, threshold):
self.threshold = threshold
def fit(self, df, cont_cols):
self.cont_cols = cont_cols
self.var = x_develop[cont_cols].var()
good_cols = self.var[self.var > self.threshold]
self.index = good_cols.index.to_list()
self.dropcols = [x for x in cont_cols if x not in self.var[self.var > self.threshold].index.to_list() ]
self.validcols = [x for x in cont_cols if x in self.var[self.var > self.threshold].index.to_list() ]
def transform(self, df):
return df.drop(self.dropcols, axis=1)
 def fit_transform(self, df, cont_cols):
self.fit(df, cont_cols)
 return self.transform(df), self.validcols<normalization>
sfs1 = SFS(model,
k_features=(10, 15),
forward=True,
floating=False,
verbose=0,
scoring='roc_auc',
cv=folds,
n_jobs=-1)
 sfs1 = sfs1.fit(X_train, y_train )
threshold = 0.6
print('Variance Threshold:', threshold)
 VarThres = VarianceThreshold(threshold)
 x_develop, cont_cols = VarThres.fit_transform(x_develop, cont_cols)
 x_test = VarThres.transform(x_test )<categorify>
top_features = list(sfs1.k_feature_names_)
 X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
qt = QuantileTransformer(output_distribution='normal')
 x_develop[cont_cols] = qt.fit_transform(x_develop[cont_cols])
 x_test[cont_cols] = qt.transform(x_test[cont_cols] )<feature_engineering>
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
def create_folds(df, fold_no, fold_type='mls_kfold', save=False):

 if fold_type == 'kfold':
kf = KFold(n_splits=fold_no, shuffle=True, random_state=SEED)
 elif fold_type == 'mls_kfold':
kf = MultilabelStratifiedKFold(n_splits=fold_no, random_state=SEED)
 df['Fold'] = -1
df.reset_index(inplace=True)
 for fold,(t, v)in enumerate(kf.split(df, df)) :
df.loc[v, 'Fold'] = fold
df.set_index('sig_id', inplace=True)
 if save:
df.to_csv('y_develop.csv' )<load_pretrained>
lr = linear_model.LogisticRegression(solver='liblinear', max_iter=1000)
 parameter_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2'],
'C' : [0.001, 0.01, 0.1, 1.0, 10.0, 100.0],
'solver': ['newton-cg', 'sag', 'lbfgs']
}
grid_search = GridSearchCV(lr, param_grid=parameter_grid, cv=folds, scoring='roc_auc')
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
N_FOLDS = 30
fold_type = 'mls_kfold'
create_folds(y_develop, fold_no=N_FOLDS, fold_type=fold_type, save=True )<choose_model_class>
lr = linear_model.LogisticRegression(solver='liblinear', max_iter=10000)
 parameter_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2', 'l1'],
'C' : [0.001, 0.01, 0.08, 0.1, 0.15, 1.0, 10.0, 100.0],
}
grid_search = GridSearchCV(lr, param_grid=parameter_grid, cv=folds, scoring='roc_auc')
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
class Model() :
def __init__(self, input_shape, output_bias=None):
self.input_shape = input_shape
self.output_bias = output_bias
def create_model1(self):
if self.output_bias is not None:
self.output_bias = tf.keras.initializers.Constant(output_bias)
 inputs = tf.keras.Input(shape=self.input_shape)
 x = L.BatchNormalization()(inputs)
 x = tfa.layers.WeightNormalization(L.Dense(800, activation='swish'))(x)
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.4 )(x)
 x = tfa.layers.WeightNormalization(L.Dense(400, activation='swish'))(x)
 x = L.BatchNormalization()(x)
 x = L.Dropout(0.4 )(x)
 outputs = tfa.layers.WeightNormalization(L.Dense(N_TARGETS,
activation='sigmoid',
bias_initializer=self.output_bias 
)
 )(x)
 model = tf.keras.Model(inputs=inputs, outputs=outputs)
 metrics = [tf.keras.losses.BinaryCrossentropy(name='mean_loss')]
OPTIMIZER = tfa.optimizers.Lookahead(
 tfa.optimizers.AdamW(weight_decay=1e-5),
sync_period=5)
 model.compile(optimizer=OPTIMIZER, loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.0008), metrics=metrics)
 return model<predict_on_test>
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
def calc_pred(x_df, models):
if isinstance(models[0], np.ndarray):
pred_test = np.repeat([models[0]], len(x_df), axis=0)
 else:
for i in models:
if i == 0:
pred_test = models[i].predict(x_df)
 else:
pred_test += models[i].predict(x_df)
 pred_test = pred_test/len(models)
 pred_test = pd.DataFrame(index=x_df.index, columns=target_cols, data=pred_test)
 pred_test.loc[x_df['cp_type'] == 0] = 0
return pred_test
def oof_score(oof: dict):
return np.mean(list(oof.values())) , np.std(list(oof.values()))
def combine_pred(pred):
for i in pred:
if i==0:
r = pred[i]
else:
r = np.append(r, pred[i], axis=0)
 return r
def run_dump_baseline_cv(xtrain=x_develop, ytrain=y_develop, N_FOLDS=N_FOLDS, summary=True, debug=False):
models = {x: '' for x in range(N_FOLDS)}
results = {x: '' for x in range(N_FOLDS)}
oof_bp = {x: [] for x in range(N_FOLDS)}
oof_ap = {x: [] for x in range(N_FOLDS)}
pred_val_fold = {x: [] for x in range(N_FOLDS)}
for foldno in np.sort(y_develop['Fold'].unique()):
x_train_fold = x_develop[y_develop['Fold'] != foldno]
y_train_fold = y_develop[y_develop['Fold'] != foldno].drop('Fold', axis=1)
 x_val_fold = x_develop[y_develop['Fold'] == foldno]
y_val_fold = y_develop[y_develop['Fold'] == foldno].drop('Fold', axis=1)
 train_sample_size = len(y_train_fold)
 val_sample_size = len(y_val_fold)
 print(" ")
 print(f"Fold-%d" %(foldno))
print("Train sample size:", train_sample_size, ", Validation sample size:", val_sample_size)
 y_dev = y_develop.drop('Fold', axis=1)
 models[foldno] = y_dev.mean(axis=0 ).to_numpy()
pred_val_fold[foldno] = np.repeat([models[foldno]], len(y_val_fold), axis=0)
 oof_bp[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score: ', oof_bp[foldno])
 pred_val_fold[foldno][x_val_fold['cp_type'] == 0] == 0
oof_ap[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score with post processing: ', oof_ap[foldno])
 pred_val_fold = combine_pred(pred_val_fold)
 print('
')
 if summary:
print('Summary')
 print('Mean OOF score: %f +/- %f' %(oof_score(oof_bp)))
 print('Mean OOF score after postprocessing: %f +/- %f' %(oof_score(oof_ap)))
 return models, oof_ap, pred_val_fold
def run_cv(xtrain=x_develop, ytrain=y_develop, model=None, N_FOLDS=N_FOLDS, summary=True, debug=False):
histories = {x: '' for x in range(N_FOLDS)}
models = {x: '' for x in range(N_FOLDS)}
results = {x: '' for x in range(N_FOLDS)}
oof_bp = {x: [] for x in range(N_FOLDS)}
oof_ap = {x: [] for x in range(N_FOLDS)}
pred_val_fold = {x: [] for x in range(N_FOLDS)}
for foldno in np.sort(y_develop['Fold'].unique()):
x_train_fold = x_develop[y_develop['Fold'] != foldno]
y_train_fold = y_develop[y_develop['Fold'] != foldno].drop('Fold', axis=1)
 x_val_fold = x_develop[y_develop['Fold'] == foldno]
y_val_fold = y_develop[y_develop['Fold'] == foldno].drop('Fold', axis=1)
 train_sample_size = len(y_train_fold)
 val_sample_size = len(y_val_fold)
 print(" ")
 print(f"Fold-%d" %(foldno))
print("Train sample size:", train_sample_size, ", Validation sample size:", val_sample_size)
 FEATURE_SIZE = x_train_fold.shape[-1]
train_ds = tf.data.Dataset.from_tensor_slices(( x_train_fold, y_train_fold))
train_ds = train_ds.shuffle(1024 ).batch(56)
 val_ds = tf.data.Dataset.from_tensor_slices(( x_val_fold, y_val_fold))
val_ds = val_ds.batch(val_sample_size)
 models[foldno] = model.create_model1()
cb_es = EarlyStopping(monitor='val_mean_loss', patience=5, restore_best_weights=True)
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_mean_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 histories[foldno] = models[foldno].fit(train_ds, validation_data=val_ds, epochs=EPOCHS, callbacks=[cb_es, reduce_lr_loss], verbose=1)
 pred_val_fold[foldno] = models[foldno].predict(x_val_fold)
 oof_bp[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score: ', oof_bp[foldno])
 pred_val_fold[foldno][x_val_fold['cp_type'] == 0] == 0
oof_ap[foldno] = tf.keras.losses.binary_crossentropy(y_val_fold, pred_val_fold[foldno] ).numpy().mean()
print('Out-of-Fold Score with post processing: ', oof_ap[foldno])
 if SAVE_MODEL:
models[foldno].save(f'weights-fold{foldno}.h5')
 pred_val_fold = combine_pred(pred_val_fold)
 print('
')
 if summary:
print('Summary')
 print('Mean OOF score: %f +/- %f' %(oof_score(oof_bp)))
 print('Mean OOF score after postprocessing: %f +/- %f' %(oof_score(oof_ap)))
 return models, histories, oof_ap, pred_val_fold
def submit(res):
sub = res.reset_index()
sub.to_csv('submission.csv', index=False )<define_variables>
gnb = GaussianNB()
oof_gnb, prediction_gnb, scores_gnb = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=gnb )
EPOCHS = 45
SAVE_MODEL = True<compute_train_metric>
abc = AdaBoostClassifier()
parameter_grid = {'n_estimators': [5, 10, 20, 50, 100],
'learning_rate': [0.001, 0.01, 0.1, 1.0, 10.0]
}
grid_search = GridSearchCV(abc, param_grid=parameter_grid, cv=folds, scoring='roc_auc')
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
if sub.shape[0] != 3982:
output_bias = -np.log(y_develop[y_develop.columns[:-1]].mean(axis=0 ).to_numpy())
 models, histories, oof_ap, pred_val_fold = run_cv(x_develop, y_develop,
model=Model(x_develop.shape[1], output_bias),
N_FOLDS=N_FOLDS,
debug=False)
 pred_test = calc_pred(x_test, models)
 submit(pred_test)
 np.save('LBS.npy', pred_val_fold)
 else:
sub.to_csv('submission.csv', index=False )<set_options>
abc = AdaBoostClassifier(**grid_search.best_params_)
 oof_abc, prediction_abc, scores_abc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=abc )
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 device<set_options>
etc = ExtraTreesClassifier()
parameter_grid = {'n_estimators': [10, 50, 100, 1000],
'max_depth': [None, 3, 5, 15]
}
grid_search = GridSearchCV(etc, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
etc = ExtraTreesClassifier(**grid_search.best_params_)
 oof_etc, prediction_etc, scores_etc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=etc )
warnings.filterwarnings('ignore' )<set_options>
rfc = RandomForestClassifier()
parameter_grid = {'n_estimators': [10, 50, 100, 1000],
'max_depth': [None, 3, 5, 15]
}
grid_search = GridSearchCV(rfc, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
rfc = RandomForestClassifier(**grid_search.best_params_)
 oof_rfc, prediction_rfc, scores_rfc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=rfc )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(42 )<load_from_csv>
gpc = GaussianProcessClassifier()
oof_gpc, prediction_gpc, scores_gpc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=gpc )
df_train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 df_train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 df_test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 print('df_train_features Shape',df_train_features.shape)
 print('df_train_targets_scored Shape',df_train_targets_scored.shape)
 print('df_test_features Shape',df_test_features.shape )<merge>
svc = SVC(probability=True, gamma='scale')
 parameter_grid = {'C': [0.001, 0.01, 0.1, 1.0, 10.0],
'kernel': ['linear', 'poly', 'rbf'],
}
grid_search = GridSearchCV(svc, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
svc = SVC(probability=True, gamma='scale', **grid_search.best_params_)
 oof_svc, prediction_svc, scores_svc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=svc )
df_train = pd.merge(df_train_features,df_train_targets_scored,on='sig_id')
 df_train = df_train[df_train['cp_type']!= 'ctl_vehicle'].reset_index(drop=True)
 df_train.shape<categorify>
knc = KNeighborsClassifier()
parameter_grid = {'n_neighbors': [2, 3, 5, 10, 20],
'weights': ['uniform', 'distance'],
'leaf_size': [5, 10, 30]
}
grid_search = GridSearchCV(knc, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
knc = KNeighborsClassifier(**grid_search.best_params_)
 oof_knc, prediction_knc, scores_knc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=knc )
df_train['cp_type'] = df_train['cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df_train['cp_dose'] = df_train['cp_dose'].map({'D1': 3, 'D2': 4})
 df_train['cp_time'] = df_train['cp_time'].map({24: 0, 48: 1, 72: 2})
 df_test_features['cp_type'] = df_test_features['cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df_test_features['cp_dose'] = df_test_features['cp_dose'].map({'D1': 3, 'D2': 4})
 df_test_features['cp_time'] = df_test_features['cp_time'].map({24: 0, 48: 1, 72: 2} )<feature_engineering>
bnb = BernoulliNB()
parameter_grid = {'alpha': [0.0001, 1, 2, 10]
}
grid_search = GridSearchCV(bnb, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
bnb = BernoulliNB(**grid_search.best_params_)
 oof_bnb, prediction_bnb, scores_bnb = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=bnb )
df_train['folds'] = -1
df_train = df_train.sample(frac=1 ).reset_index(drop=True)
 splits = 5
kf = KFold(n_splits=splits,shuffle = False)
 for fold,(train_index, val_idx)in enumerate(kf.split(df_train)) :
df_train.loc[val_idx,'folds'] = int(fold)
 print('Number of Unique folds in dataset',df_train['folds'].unique() )<prepare_x_and_y>
sgd = linear_model.SGDClassifier(eta0=1, max_iter=1000, tol=0.0001)
 parameter_grid = {'loss': ['log', 'modified_huber'],
'penalty': ['l1', 'l2', 'elasticnet'],
'alpha': [0.001, 0.01],
'l1_ratio': [0, 0.15, 0.5, 1.0],
'learning_rate': ['optimal', 'invscaling', 'adaptive']
}
grid_search = GridSearchCV(sgd, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
sgd = linear_model.SGDClassifier(eta0=1, tol=0.0001, **grid_search.best_params_)
 oof_sgd, prediction_sgd, scores_sgd = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=sgd )
class Train_Dataset(Dataset):
def __init__(self, dataframe,features_col,labels):
self.dataframe = dataframe
self.features_col = features_col
self.labels = labels
self.x = self.dataframe[self.features_col].values
self.y = self.dataframe[self.labels].values
def __len__(self):
return self.dataframe.shape[0]
def __getitem__(self, idx):
feat = torch.FloatTensor(self.x[idx])
 lab = torch.FloatTensor(self.y[idx])
 return feat,lab
class Test_Dataset(Dataset):
def __init__(self, dataframe,features_col):
self.dataframe = dataframe
self.features_col = features_col
self.x = self.dataframe[self.features_col].values
def __len__(self):
return self.dataframe.shape[0]
def __getitem__(self, idx):
feat = torch.FloatTensor(self.x[idx])
 return feat<choose_model_class>
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
class MultiHead_Attn(nn.Module):
def __init__(self, embed_dim ,num_head):
super().__init__()
self.embed_dim = embed_dim
self.num_head = num_head
self.head_dim = self.embed_dim // self.num_head
assert self.embed_dim%self.num_head == 0
self.q = nn.Linear(self.embed_dim , self.embed_dim)
 self.k = nn.Linear(self.embed_dim , self.embed_dim)
 self.v = nn.Linear(self.embed_dim , self.embed_dim)
 self.f_linear = nn.Linear(self.embed_dim, self.embed_dim)
 self.dropout = nn.Dropout (.35)
 self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device)
 def forward(self, x):
batch_size = x.shape[0]
src_len = x.shape[1]
w_k = self.k(x)
 w_q = self.q(x)
 w_v = self.v(x)
 w_q = w_q.view(batch_size,-1,self.head_dim)
 w_k = w_k.view(batch_size,-1,self.head_dim)
 w_v = w_v.view(batch_size,-1,self.head_dim)
 energy = torch.matmul(w_k.permute(0,2,1),w_q)
 energy = energy/self.scale
energy = torch.softmax(energy,-1)
 f_energy = torch.matmul(self.dropout(energy), w_v.permute(0,2,1))
f_energy = f_energy.permute(0, 2, 1)
 f_energy = f_energy.reshape(batch_size,-1)
 out = self.f_linear(f_energy)
 return out<normalization>
poly = PolynomialFeatures(2)
 X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 X_train_poly = poly.fit_transform(X_train)
 X_test_poly = poly.transform(X_test )
class PositionwiseFeedforwardLayer(nn.Module):
def __init__(self, embed_dim, pf_dim):
super().__init__()
self.fc_1 = nn.Linear(embed_dim, pf_dim)
 self.fc_2 = nn.Linear(pf_dim, embed_dim)
 self.dropout = nn.Dropout (.35)
 def forward(self, x):
x = self.dropout(torch.relu(self.fc_1(x)))
 x = self.fc_2(x)
 return x<choose_model_class>
sc = []
for i in range(10, 510, 5):
top_corr_cols = list(cor.abs().sort_values().tail(i ).reset_index() ['index'].values)
 X_train_poly1 = X_train_poly[:, top_corr_cols]
X_test_poly1 = X_test_poly[:, top_corr_cols]
oof_lr_poly, prediction_lr_poly, scores = train_model(X_train_poly1, X_test_poly1, y_train, params=None, model_type='sklearn', model=model)
 sc.append(scores )
class TransformerBlock(nn.Module):
def __init__(self, embed_dim,num_heads, pf_dim):
super().__init__()
self.self_attn_layer_norm = nn.LayerNorm(embed_dim)
 self.ff_layer_norm = nn.LayerNorm(embed_dim)
 self.attn = MultiHead_Attn(embed_dim ,num_heads)
 self.positionwise_feedforward = PositionwiseFeedforwardLayer(embed_dim,pf_dim)
 self.dropout = nn.Dropout (.35)
 def forward(self,src):
attn = self.attn(src)
 src = self.self_attn_layer_norm(src + self.dropout(attn))
_src = self.positionwise_feedforward(src)
 src = self.ff_layer_norm(src + self.dropout(_src))
return src<categorify>
top_corr_cols = list(cor.abs().sort_values().tail(300 ).reset_index() ['index'].values)
 X_train_poly1 = X_train_poly[:, top_corr_cols]
X_test_poly1 = X_test_poly[:, top_corr_cols]
class Encorder(nn.Module):
def __init__(self, src_dim , embed_dim ,num_heads , pf_dim , target_dim ,depth):
super(Encorder, self ).__init__()
self.embed_layer = nn.Linear(src_dim , embed_dim)
 self.Transformer_layers = TransformerBlock(embed_dim , num_heads ,pf_dim)
 tblocks = []
for i in range(depth):
tblocks.append(self.Transformer_layers)
 self.tblocks = nn.Sequential(*tblocks)
 self.dropout = nn.Dropout (.35)
 self.final_linear = nn.Sequential(
 nn.utils.weight_norm(nn.Linear(embed_dim , embed_dim //2))
,nn.BatchNorm1d(embed_dim //2)
 ,nn.Dropout (.35)
 ,nn.ReLU()
,nn.Linear(embed_dim //2 , target_dim))
def forward(self , x):
batch_size,seq_len = x.shape
embed = self.embed_layer(x)
 for layer in self.tblocks:
x = self.dropout(self.Transformer_layers(embed))
out = self.final_linear(x)
 return out 
<train_model>
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_poly, prediction_lr_poly, scores = train_model(X_train_poly1, X_test_poly1, y_train, params=None, model_type='sklearn', model=model )
def train(loader, model , optimizer , loss_func ,scheduler):
model.train()
train_running_loss = 0
for index,(feat,label)in enumerate(loader):
x,y = feat.to(device),label.to(device)
 optimizer.zero_grad()
pred = model(x)
 loss = loss_func(pred,y)
 loss.backward()
optimizer.step()
train_running_loss += loss.item()
return train_running_loss/len(loader)
 def valid(loader, model , loss_func):
valid_running_loss = 0
pred_list = list()
with torch.no_grad() :
model.eval()
for val_index,(feat,label)in enumerate(loader):
x,y = feat.to(device),label.to(device)
 pred = model(x)
 pred_list.append(pred)
 loss = loss_func(pred,y)
 valid_running_loss += loss.item()
return torch.cat(pred_list ).sigmoid().detach().cpu().numpy() ,valid_running_loss/len(loader)
 def test(loader,model):
labels = []
with torch.no_grad() :
model.eval()
for index,feat in enumerate(loader):
x = feat.to(device)
 output = model(x)
 labels.append(output.sigmoid().detach().cpu().numpy())
 labels = np.concatenate(labels)
 return labels<train_model>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_poly
submission.head()
def train_run(seed , Batch_size , src_dim ,embed_dim , num_heads , pf_dim ,target_dim , depth , n_epoch , patience , learning_rate):
val_preds = np.zeros(( df_train[y_cols].shape))
test_pred = np.zeros(( len(df_test_features), y_cols.shape[0]))
for fold_num in range(splits):
seed_everything(seed)
 print('seed value', seed)
 print('='*30,'*****','Fold',fold_num+1,'*****','='*30)
 trn_idx = df_train[df_train['folds'] != fold_num].index
val_idx = df_train[df_train['folds'] == fold_num].index
df_trn = df_train.loc[trn_idx].reset_index(drop=True)
 df_val = df_train.loc[val_idx].reset_index(drop=True)
 train_dataset = Train_Dataset(df_trn,x_cols,y_cols)
 train_loader = DataLoader(train_dataset, batch_size=Batch_size, shuffle=True)
 valid_dataset = Train_Dataset(df_val,x_cols,y_cols);
valid_loader = DataLoader(valid_dataset, batch_size=Batch_size, shuffle=False)
 model = Encorder(src_dim , embed_dim ,num_heads , pf_dim , target_dim ,depth ).to(device)
 pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
 print('Number of parameters',pytorch_total_params)
 optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay= 1e-5)
 scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', patience=3, verbose=True, factor=0.2)
 loss_func = nn.BCEWithLogitsLoss()
best_score = np.inf
counter = 0
for epoch in range(n_epoch):
print("Epoch: {}/{}.. ".format(epoch+1, n_epoch))
train_loss = train(train_loader, model , optimizer , loss_func ,scheduler)
 v_out,valid_loss = valid(valid_loader, model , loss_func)
 print(f'\tTrain Loss: {train_loss:.5f}')
 print(f'\t Val.Loss: {valid_loss:.5f}')
 if valid_loss < best_score:
best_score = valid_loss
val_preds[val_idx] = v_out
torch.save(model.state_dict() , f"fold_{fold_num}_seed_{seed}.pth")
 else:
print('patience starts.........')
 counter +=1
print('.. counter.. ',counter)
 if(counter >= patience):
break;
scheduler.step(valid_loss)
 print(f'Inferenceing the test data at epoch {epoch+1} and fold {fold_num+1} and seed {seed}.....')
 model = Encorder(src_dim , embed_dim ,num_heads , pf_dim , target_dim ,depth ).to(device)
 model.load_state_dict(torch.load(f"fold_{fold_num}_seed_{seed}.pth"))
test_dataset = Test_Dataset(df_test_features,x_cols)
 test_loader = DataLoader(test_dataset, batch_size=Batch_size, shuffle=False)
 test_pred += test(test_loader,model)/splits
print(f'Inferenceing completed for fold {fold_num+1}')
 return test_pred , val_preds<define_variables>
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 X_train['300'] = X_train.std(1)
 X_test['300'] = X_test.std(1)
 scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_1, prediction_lr_1, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model )
random_seed = [33 , 66 , 99 , 132]
Batch_size = 32
src_dim = 875
embed_dim = 1024
num_heads = 8
pf_dim = 512
target_dim = 206
depth = 8
n_epoch = 30
patience = 10
learning_rate =.0005
test_pred = np.zeros(( len(df_test_features), y_cols.shape[0]))
val_preds = np.zeros(( df_train[y_cols].shape))
for seed in random_seed:
tp , vp = train_run(seed , Batch_size , src_dim ,embed_dim , num_heads , pf_dim ,target_dim , depth , n_epoch , patience , learning_rate)
 test_pred += tp/len(random_seed)
 val_preds += vp/len(random_seed )<compute_test_metric>
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 X_train['300'] = X_train.std(1)
 X_test['300'] = X_test.std(1)
 scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_1, prediction_lr_1_repeated, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_1_repeated
submission.to_csv('repeated_fold_features.csv', index=False)
 submission.head()
score = 0
for i in range(df_train[y_cols].shape[1]):
_score = log_loss(df_train[y_cols].iloc[:,i], val_preds[:,i])
 score += _score / df_train[y_cols].shape[1]
print(f"oof score: {score}" )<load_from_csv>
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 main_cols = X_train.columns.tolist()
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub[y_cols] = test_pred
sub.loc[df_test_features['cp_type'] == 1 , y_cols ] = 0<save_to_csv>
neigh = NearestNeighbors(5, n_jobs=-1)
 neigh.fit(X_train)
 dists, _ = neigh.kneighbors(X_train, n_neighbors=5)
 mean_dist = dists.mean(axis=1)
 max_dist = dists.max(axis=1)
 min_dist = dists.min(axis=1)
 X_train['300'] = X_train.std(1)
 X_train = np.hstack(( X_train, mean_dist.reshape(-1, 1), max_dist.reshape(-1, 1), min_dist.reshape(-1, 1)))
 test_dists, _ = neigh.kneighbors(X_test, n_neighbors=5)
 test_mean_dist = test_dists.mean(axis=1)
 test_max_dist = test_dists.max(axis=1)
 test_min_dist = test_dists.min(axis=1)
 X_test['300'] = X_test.std(1)
 X_test = np.hstack(( X_test, test_mean_dist.reshape(-1, 1), test_max_dist.reshape(-1, 1), test_min_dist.reshape(-1, 1)) )
sub.to_csv('submission.csv', index=False )<save_to_csv>
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_2, prediction_lr_2, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model)
 submission['target'] = prediction_lr_2
submission.to_csv('nn_features.csv', index=False )
sub.to_csv('submission.csv', index=False )<load_from_csv>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] =(prediction_lr_1 + prediction_lr_2)/ 2
submission.head()
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop('sig_id',axis=1 )<categorify>
X_train = train.drop(['id', 'target'], axis=1)
 X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr_1, prediction_lr_1, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )
cat = 'cp_dose'
le = preprocessing.LabelEncoder()
le.fit(train_features[cat])
 train_features[cat] = le.transform(train_features[cat])
 test_features[cat] = le.transform(test_features[cat] )<normalization>
scores_dict = {'f_classif': [], 'mutual_info_classif': []}
for i in range(5, 100, 5):
s1 = SelectPercentile(f_classif, percentile=i)
 X_train1 = s1.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s1.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['f_classif'].append(np.mean(scores))
s2 = SelectPercentile(mutual_info_classif, percentile=i)
 X_train1 = s2.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s2.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['mutual_info_classif'].append(np.mean(scores))
scaler = preprocessing.MinMaxScaler()
scaler.fit(train_features.append(test_features.drop('cp_type',axis=1)))
 data_train = scaler.transform(train_features)
 data_test = scaler.transform(test_features.drop('cp_type',axis=1))<compute_test_metric>
scores_dict = {'f_classif': [], 'mutual_info_classif': []}
for i in range(10, 301, 10):
s1 = SelectKBest(f_classif, k=i)
 X_train1 = s1.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s1.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['f_classif'].append(np.mean(scores))
s2 = SelectKBest(mutual_info_classif, k=i)
 X_train1 = s2.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s2.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['mutual_info_classif'].append(np.mean(scores))
n_labels = train_targets_scored.shape[1]
n_features = data_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
n_seeds = 6
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
mskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in mskf.split(data_train,labels_train):
X_train = data_train[train]
X_test = data_train[test]
y_train = labels_train[train]
y_test = labels_train[test]
model = Sequential()
model.add(layers.Dense(2048, input_shape=(n_features,)))
 model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(1024))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(512))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(n_labels))
model.add(layers.Activation('sigmoid'))
model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.3, patience=5, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=15, mode='min',restore_best_weights=True)
 hist = model.fit(X_train,y_train, batch_size=128, epochs=192,verbose=0,validation_data =(X_test,y_test),callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('LabelSmoothed_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict(X_test)
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1
tf.print('OOF score is ',oof)
 plt.figure(figsize=(12,8))
hist_train = np.zeros(45)
 hist_val = np.zeros(45)
 for i in range(n_folds):
hist_train += np.array(hists[i].history['loss'][:45])/n_folds
hist_val += np.array(hists[i].history['val_loss'][:45])/n_folds
plt.plot(hist_train)
 plt.plot(hist_val)
 plt.yscale('log')
 plt.yticks(ticks=[1,1E-1,1E-2])
 plt.xlabel('Epochs')
 plt.ylabel('Average Logloss')
 plt.legend(['Training','Validation'] )<save_to_csv>
selector = SelectKBest(f_classif, k=60)
 X_trainK = selector.fit_transform(X_train, y_train.values.astype(int))
X_testK = selector.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_trainK, X_testK, y_train, params=None, model_type='sklearn', model=model )
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )<set_options>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_1
submission.head()
%matplotlib inline 
<load_from_csv>
scores_list = []
for i in range(10, 301, 5):
s = RFE(model, i, step=1)
 X_train1 = s.fit_transform(X_train, y_train.values.astype(int))
X_test1 = s.transform(X_test)
 oof_lr_1, prediction_lr_1, scores = train_model(X_train1, X_test1, y_train, params=None, model_type='sklearn', model=model)
 scores_list.append(np.mean(scores))
file_1 = pd.read_csv('.. /input/m5-uncertainity/submission-0.11719.csv')
 file_2 = pd.read_csv('.. /input/m5-uncertainity/submission-0.11440.csv')
 file_3 = pd.read_csv('.. /input/m5-uncertainity/submission-0.11407.csv')
 file_4 = pd.read_csv('.. /input/m5-uncertainity/submission-0.11238.csv')
 file_5 = pd.read_csv('.. /input/m5-uncertainity/submission-0.11248.csv')
 file_6 = pd.read_csv('.. /input/m5-uncertainity/submission-.11029.csv')
 file = file_1<feature_engineering>
selector = RFE(model, 20, step=1)
 X_trainK = selector.fit_transform(X_train, y_train.values.astype(int))
X_testK = selector.transform(X_test)
 oof_lr_1, prediction_lr_rfe_20, scores = train_model(X_trainK, X_testK, y_train, params=None, model_type='sklearn', model=model )
file['F1'] =(file_3['F1']+file_2['F1']+file_1['F1']+file_4['F1']+file_5['F1']+file_6['F1'])/6
file['F2'] =(file_3['F2']+file_2['F2']+file_1['F2']+file_4['F2']+file_5['F2']+file_6['F2'])/6
file['F3'] =(file_3['F3']+file_2['F3']+file_1['F3']+file_4['F3']+file_5['F3']+file_6['F3'])/6
file['F4'] =(file_3['F4']+file_2['F4']+file_1['F4']+file_4['F4']+file_5['F4']+file_6['F4'])/6
file['F5'] =(file_3['F5']+file_2['F5']+file_1['F5']+file_4['F5']+file_5['F5']+file_6['F5'])/6
file['F6'] =(file_3['F6']+file_2['F6']+file_1['F6']+file_4['F6']+file_5['F6']+file_6['F6'])/6
file['F7'] =(file_3['F7']+file_2['F7']+file_1['F7']+file_4['F7']+file_5['F7']+file_6['F7'])/6
file['F8'] =(file_3['F8']+file_2['F8']+file_1['F8']+file_4['F8']+file_5['F8']+file_6['F8'])/6
file['F9'] =(file_3['F9']+file_2['F9']+file_1['F9']+file_4['F9']+file_5['F9']+file_6['F9'])/6
file['F10'] =(file_3['F10']+file_2['F10']+file_1['F10']+file_4['F10']+file_5['F10']+file_6['F10'])/6
file['F11'] =(file_3['F11']+file_2['F11']+file_1['F11']+file_4['F11']+file_5['F11']+file_6['F11'])/6
file['F12'] =(file_3['F12']+file_2['F12']+file_1['F12']+file_4['F12']+file_5['F12']+file_6['F12'])/6
file['F13'] =(file_3['F13']+file_2['F13']+file_1['F13']+file_4['F13']+file_5['F13']+file_6['F13'])/6
file['F14'] =(file_3['F14']+file_2['F14']+file_1['F14']+file_4['F14']+file_5['F14']+file_6['F14'])/6
file['F15'] =(file_3['F15']+file_2['F15']+file_1['F15']+file_4['F15']+file_5['F15']+file_6['F15'])/6
file['F16'] =(file_3['F16']+file_2['F16']+file_1['F16']+file_4['F16']+file_5['F16']+file_6['F16'])/6
file['F17'] =(file_3['F17']+file_2['F17']+file_1['F17']+file_4['F17']+file_5['F17']+file_6['F17'])/6
file['F18'] =(file_3['F18']+file_2['F18']+file_1['F18']+file_4['F18']+file_5['F18']+file_6['F18'])/6
file['F19'] =(file_3['F19']+file_2['F19']+file_1['F19']+file_4['F19']+file_5['F19']+file_6['F19'])/6
file['F20'] =(file_3['F20']+file_2['F20']+file_1['F20']+file_4['F20']+file_5['F20']+file_6['F20'])/6
file['F21'] =(file_3['F21']+file_2['F21']+file_1['F21']+file_4['F21']+file_5['F21']+file_6['F21'])/6
file['F22'] =(file_3['F22']+file_2['F22']+file_1['F22']+file_4['F22']+file_5['F22']+file_6['F22'])/6
file['F23'] =(file_3['F23']+file_2['F23']+file_1['F23']+file_4['F23']+file_5['F23']+file_6['F23'])/6
file['F24'] =(file_3['F24']+file_2['F24']+file_1['F24']+file_4['F24']+file_5['F24']+file_6['F24'])/6
file['F25'] =(file_3['F25']+file_2['F25']+file_1['F25']+file_4['F25']+file_5['F25']+file_6['F25'])/6
file['F26'] =(file_3['F26']+file_2['F26']+file_1['F26']+file_4['F26']+file_5['F26']+file_6['F26'])/6
file['F27'] =(file_3['F27']+file_2['F27']+file_1['F27']+file_4['F27']+file_5['F27']+file_6['F27'])/6
file['F28'] =(file_3['F28']+file_2['F28']+file_1['F28']+file_4['F28']+file_5['F28']+file_6['F28'])/6<save_to_csv>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr_rfe_20
submission.to_csv('rfe_20.csv', index=False)
 submission.head()
file.to_csv("submission.csv", index=False )<load_from_csv>
selector = SelectKBest(f_classif, k=15)
 X_trainK = selector.fit_transform(X_train, y_train.values.astype(int))
X_testK = selector.transform(X_test )
price=pd.read_csv("/kaggle/input/m5-forecasting-uncertainty/sell_prices.csv")
 calen=pd.read_csv("/kaggle/input/m5-forecasting-uncertainty/calendar.csv" )<categorify>
oof_glm, prediction_glm, scores = train_model(X_trainK, X_testK, y_train, params=None, model_type='glm' )
def preprocess_sales(sales, start=1400, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale","start"],
var_name='d', value_name='demand')
 print("generating order")
 if start is not None:
skip = start
else:
skip = 1
sales["nb"] =sales.index // 42840 + skip
return sales<categorify>
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_glm
submission.to_csv('glm.csv', index=False)
 submission.head()
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar<load_from_csv>
eli5_weights = eli5.formatters.as_dataframe.explain_weights_df(model)
 eli5_weights['weight'] = eli5_weights['weight'].abs()
eli5_weights = eli5_weights.sort_values('weight', ascending=False)
 eli5_weights
def make_dataset(categorize=False ,start=1400, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 del calendar
print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 gc.collect()
return sales<import_modules>
train['mean'] = train.mean(1)
 train['std'] = train.std(1)
 test['mean'] = test.mean(1)
 test['std'] = test.std(1)
 scores_dict = {'simple': [], 'with_std': [], 'with_mean': []}
for i in range(1, eli5_weights.shape[0] + 1):
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:i]
X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['simple'].append(np.mean(scores))
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['with_mean'].append(np.mean(scores))
X_train = train[top_features + ['std']]
X_test = test[top_features + ['std']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model)
 scores_dict['with_std'].append(np.mean(scores))
import time
import gc<create_dataframe>
train['mean'] = train.mean(1)
 train['std'] = train.std(1)
 test['mean'] = test.mean(1)
 test['std'] = test.std(1)
 scores_dict = {'simple': [], 'with_std': [], 'with_mean': []}
for i in range(1, eli5_weights.shape[0] + 1):
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:i]
X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 scores_dict['simple'].append(np.mean(scores))
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 scores_dict['with_mean'].append(np.mean(scores))
X_train = train[top_features + ['std']]
X_test = test[top_features + ['std']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 scores_dict['with_std'].append(np.mean(scores))
%%time
CATEGORIZE = True;
START = 1400; UPPER = 1970;
maps = {}
mods = {}
sales = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )<import_modules>
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:8]
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr
submission.to_csv('submission_top8.csv', index=False)
 submission.head()
<define_variables><EOS> 
top_features = [i[1:] for i in eli5_weights.feature if 'BIAS' not in i][:10]
X_train = train[top_features + ['mean']]
X_test = test[top_features + ['mean']]
scaler = StandardScaler()
X_train[X_train.columns[:-1]] = scaler.fit_transform(X_train[X_train.columns[:-1]])
 X_test[X_train.columns[:-1]] = scaler.transform(X_test[X_train.columns[:-1]])
 oof_lr, prediction_lr, scores = train_model(X_train.values, X_test.values, y_train, params=None, model_type='sklearn', model=model, folds=repeated_folds)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction_lr
submission.to_csv('submission_top10.csv', index=False)
 submission.head()