Kaggle competition: Don't Overfit! II
def run_training(train, valid, feature_cols, target_cols, model_path, seed, param_provided=None):
seed_everything(seed)
 x_train, y_train = train[feature_cols].values, train[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 if valid is not None:
x_valid, y_valid = valid[feature_cols].values, valid[target_cols].values
valid_dataset = MoADataset(x_valid, y_valid)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 if param_provided is not None:
EPOCHS = param_provided['epoch']
hidden_size = param_provided['hidden_size']
LEARNING_RATE = param_provided['lr']
print("hidden_size: ", hidden_size, ", learning_rate: ", LEARNING_RATE)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.5)
 loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
train_losses = []; valid_losses = []
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 train_losses.append(train_loss)
 if valid is not None:
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 valid_losses.append(valid_loss)
 if epoch % 5 == 0:
print(f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
torch.save(model.state_dict() , model_path)
 elif EARLY_STOP == True:
early_step += 1
if early_step >= early_stopping_steps:
break
else:
if epoch % 10 == 0:
print(f"EPOCH: {epoch}, train_loss: {train_loss}")
 print("early stop with epoch: ", epoch)
 print(f"LAST EPOCH: {epoch}, train_loss: {train_loss}")
 if valid is None:
torch.save(model.state_dict() , model_path)
 return {"train_losses": train_losses, "valid_losses": valid_losses}<prepare_x_and_y>
train=pd.read_csv(".. /input/train.csv" )
def run_training_tune(trial, train, valid, feature_cols, target_cols, model_path, seed):
seed_everything(seed)
 x_train, y_train = train[feature_cols].values, train[target_cols].values
x_valid, y_valid = valid[feature_cols].values, valid[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 hidden_size = trial.suggest_int("hidden_size", 200, 500, step=50)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 lr = trial.suggest_float("lr", 1e-5, 1e-1, log=True)
 optimizer = torch.optim.Adam(model.parameters() , lr=lr, weight_decay=WEIGHT_DECAY)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.5)
 loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
train_losses = []; valid_losses = []
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 train_losses.append(train_loss)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 valid_losses.append(valid_loss)
 if epoch % 5 == 0:
print(f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
elif EARLY_STOP == True:
early_step += 1
if early_step >= early_stopping_steps:
break
return valid_losses[-1] 
<compute_test_metric>
test=pd.read_csv(".. /input/test.csv" )
def print_model_results(train_loss, valid_loss, train_len, valid_len):
total_num_inst = train_len + valid_len
train_loss_final = train_loss
valid_loss_final = valid_loss
entire_train_loss = train_len / total_num_inst * train_loss_final \
+ valid_len / total_num_inst * valid_loss_final
print("train loss: ", train_loss_final, " valid loss: ", valid_loss_final,
"entire loss: ", entire_train_loss )<train_model>
train.isnull().sum()
start_time = datetime.datetime.now()
time.sleep(5)
 end_time = datetime.datetime.now()
print("time elapsed: ", end_time - start_time )<load_from_csv>
train.isnull().sum()
<init_hyperparams>
train.pop("id" )
best_param_with_epoch = {'kfold_0': {'hidden_size': 450, 'lr': 0.0008925929114331024, 'epoch': 30},
'kfold_1': {'hidden_size': 400, 'lr': 0.0051852480304641155, 'epoch': 17},
'kfold_2': {'hidden_size': 200, 'lr': 0.0020302278497303823, 'epoch': 30},
'kfold_3': {'hidden_size': 300, 'lr': 0.0029643441803447576, 'epoch': 20},
'kfold_4': {'hidden_size': 250, 'lr': 0.006931570763035235, 'epoch': 16}}<define_variables>
test.pop("id" )
seed_everything(seed=1903)
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 100
BATCH_SIZE = 64
LEARNING_RATE = 1e-2
WEIGHT_DECAY = 1e-5
EARLY_STOPPING_STEPS = 11
EARLY_STOP = True
num_features = len(feature_cols)
 num_targets = len(target_cols)
 hidden_size = 50
if_tune = False
SEED = [1903]
prefix = "process_2"
models_dir = "/kaggle/working/trained_models/"
try:
os.makedirs(models_dir)
 except FileExistsError:
pass
for seed in SEED:
NFOLDS = 5
train_entire = pd.read_pickle(tmp_data_dir + "data_train_process2_5fold_1903.pkl")
 best_param = {}
for k_fold in np.arange(NFOLDS):
print("runing fold ", k_fold)
 train = train_entire
valid = None
data_test_x = pd.read_pickle(tmp_data_dir + "data_test.pkl")
 model_path = models_dir + prefix + "_model_fold_{0:d}.pth".format(k_fold)
 model_result = None
if if_tune:
start_time = datetime.datetime.now()
study = optuna.create_study(direction="minimize")
 study.optimize(lambda trial: run_training_tune(trial, train, valid, feature_cols, target_cols, model_path,
seed), n_trials=20)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: ", trial.value)
 print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))
best_param["kfold_" + str(k_fold)] = trial.params
end_time = datetime.datetime.now()
print("time elapsed: ", end_time - start_time)
 else:
start_time = datetime.datetime.now()
model_result = run_training(train, valid, feature_cols, target_cols,
model_path, seed,
param_provided = best_param_with_epoch["kfold_" + str(k_fold)])
 end_time = datetime.datetime.now()
print("time elapsed: ", end_time - start_time)
 if valid is None:
print("loss on the entire training data", model_result["train_losses"][-1])
 else:
print_model_results(model_result["train_losses"][-1], model_result["valid_losses"][-1],
train.shape[0], valid.shape[0] )<create_dataframe>
y_train=train.iloc[:,0]
x_test = data_test_x[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 for k_fold in np.arange(NFOLDS):
if best_param_with_epoch is not None:
hidden_size = best_param_with_epoch["kfold_" + str(k_fold)]['hidden_size']
model_path = models_dir + prefix + "_model_fold_{0:d}.pth".format(k_fold)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(model_path))
model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 pred_test = pd.DataFrame(data=predictions, columns=target_cols ).fillna(0)
 pred_test = pd.concat([data_test_x[["sig_id"]], pred_test], axis=1)
 pred_result_f = models_dir + prefix + "_fold_{0:d}_prediction.csv".format(k_fold)
 pred_test.to_csv(pred_result_f, index=False )<load_from_csv>
train.pop("target" )
folds_to_use = np.arange(NFOLDS)
 for i_count, k_fold in enumerate(folds_to_use):
pred_result_f = models_dir + prefix + "_fold_{0:d}_prediction.csv".format(k_fold)
 pred_test = pd.read_csv(pred_result_f, header=0)
 pred_test[target_cols] = pred_test[target_cols]/len(folds_to_use)
 if i_count == 0:
pred_test_average = pred_test.copy(deep=True)
 else:
pred_test_average[target_cols] = pred_test_average[target_cols] + pred_test[target_cols]<save_to_csv>
train1=train.copy()
y_train1=y_train.copy()
test1=test.copy()
sample_submission = pd.read_csv(".. /input/lish-moa/" + 'sample_submission.csv')
 sub = sample_submission.drop(columns=target_cols ).merge(pred_test_average, on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<set_options>
from sklearn.preprocessing import StandardScaler
%%HTML
<style type="text/css">
div.h1 {
background-color:
color: white;
padding: 8px;
padding-right: 300px;
font-size: 35px;
max-width: 1500px;
margin: auto;
margin-top: 50px;
}
div.h2 {
background-color:
color: white;
padding: 8px;
padding-right: 300px;
font-size: 35px;
max-width: 1500px;
margin: auto;
margin-top: 50px;
}
</style><set_options>
sc=StandardScaler()
warnings.filterwarnings('ignore' )<import_modules>
train=sc.fit_transform(train )
from sklearn.preprocessing import QuantileTransformer<load_from_csv>
test=sc.fit_transform(test )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
from sklearn.linear_model import LogisticRegression,Lasso , Ridge
from sklearn.model_selection import GridSearchCV, cross_val_score
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<normalization>
random_state=42
log_clf=LogisticRegression(random_state=random_state)
 param_grid={
'class_weight': ['balanced', None],
'C':[0.001, 0.01, 0.1, 1, 10, 100, 1000],
'penalty':['l1','l2']
}
grid=GridSearchCV(log_clf,param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1)
 grid.fit(train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_))
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
log_clf = LogisticRegression(C=0.1,random_state=42,class_weight='balanced',penalty='l1' )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<concatenate>
log_clf.fit(train,y_train )
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
pred_log=log_clf.predict(test )
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
submission3 = pd.read_csv('.. /input/sample_submission.csv' )
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape 
<merge>
submission3["target"]=pred_log
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
submission3.to_csv('submissionlog.csv', index=False )
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
from sklearn.feature_selection import RFE
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
selector = RFE(log_clf, 25, step=1)
 selector.fit(train,y_train )
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds<categorify>
submission1 = pd.read_csv('.. /input/sample_submission.csv')
 submission1['target'] = selector.predict_proba(test)
 submission1.to_csv('submissionRFE.csv', index=False )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<train_model>
from sklearn.ensemble import GradientBoostingRegressor
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<choose_model_class>
random_state=42
gbr=GradientBoostingRegressor(random_state=random_state)
 param_grid={
"learning_rate": [0.01, 0.05 , 0.1 , 1],
"max_depth":[5, 11, 17, 25, 37, 53],
"max_features":["log2","sqrt"],
"criterion": ["friedman_mse", "mae"],
"n_estimators":[10, 50, 200, 600, 1000]
}
grid=GridSearchCV(gbr,param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1)
 grid.fit(train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_))
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.4)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<categorify>
gbr=GradientBoostingRegressor(learning_rate=0.01, max_leaf_nodes=10, criterion='friedman_mse', max_depth=11, max_features='sqrt', n_estimators=1000 )
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
gbr.fit(train,y_train )
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
pred5=gbr.predict(test )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048 
<prepare_x_and_y>
submissionm = pd.read_csv('.. /input/sample_submission.csv')
 submissionm["target"]=pred5
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
j=0
for i in submissionm["target"]:
if i>1:
submissionm["target"][j]=1
j+=1
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<define_variables>
submissionm.to_csv('submission_gbr.csv', index=False )
SEED = [0, 1, 2, 3 ,4, 5 ,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<merge>
finalsub=(submissionm["target"]+submission1["target"])/2
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
<save_to_csv>
subf= pd.read_csv('.. /input/sample_submission.csv')
 subf["target"]=finalsub
<set_options><EOS> 
subf.to_csv('submission_fin.csv', index=False )