Kaggle competition: Restaurant Revenue Prediction
<SOS> metric: RMSE Kaggle data source: restaurant-revenue-prediction<choose_model_class>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import mean_squared_error, r2_score
from math import sqrt
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.neighbors import KNeighborsRegressor
import lightgbm as lgbm
class ResidualNet(nn.Module):
def __init__(self, num_features, num_targets, n_hidden):
super(ResidualNet, self ).__init__()
self.block1 = nn.Sequential(
 nn.Linear(num_features, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.3),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.3), 
)
 self.block2 = nn.Sequential(
 nn.Linear(num_features + n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.2),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.2),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.Dropout(0.2),
nn.BatchNorm1d(n_hidden), 
)
 self.block3 = nn.Sequential(
 nn.Linear(n_hidden + n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.1),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.1),
nn.Linear(n_hidden, num_targets)
)
 def get_freeze(self, target_size, freeze_all=True):
if freeze_all:
for param in self.parameters() :
param.requires_grad = False
self.block3 = nn.Sequential(
 nn.Linear(512 + 256, 512),
nn.BatchNorm1d(512),
nn.Dropout(0.1),
nn.ELU() ,
nn.Linear(512, 256),
nn.BatchNorm1d(256),
nn.Dropout(0.1),
nn.ELU() ,
nn.Linear(256, target_size)
)
 def forward(self, x):
x_1 = self.block1(x)
 x_1_cat = torch.cat([x, x_1], axis=-1)
 x_2 = self.block2(x_1_cat)
 x_2_cat = torch.cat([x_1, x_2], axis=-1)
 output = self.block3(x_2_cat)
 return output<normalization>
train = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/train.csv.zip")
 test = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/test.csv.zip")
 ref = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/sampleSubmission.csv")
 test_id = pd.DataFrame(test["Id"])
 display(ref.head() )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size + num_features, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, inputs):
x = self.batch_norm1(inputs)
 x_1 = F.leaky_relu(self.dense1(x), 1e-2)
 x = self.batch_norm2(x_1)
 x_1 = self.dropout2(x)
 x = torch.cat([x_1, inputs], axis=-1)
 x_2 = F.leaky_relu(self.dense2(x), 1e-2)
 x = self.batch_norm3(x_2)
 x_2 = self.dropout3(x)
 x = self.dense3(x_2)
 return x<choose_model_class>
a = train.isnull().sum()
a[a>0]
class SimpleNet(nn.Module):
def __init__(self, num_features, num_targets, n_hidden):
super(SimpleNet, self ).__init__()
self.layer1 = nn.Linear(num_features, n_hidden)
 self.layer2 = nn.Linear(n_hidden, num_targets)
 def forward(self, inputs):
x = self.layer1(inputs)
 x = F.relu(x)
 x = self.layer2(x)
 return x<define_variables>
print(train["City"].groupby(train["City"] ).count().count())
 print(train["City Group"].groupby(train["City Group"] ).count().count())
 print(train["Type"].groupby(train["Type"] ).count().count())
 print(test["City"].groupby(test["City"] ).count().count())
 print(test["City Group"].groupby(test["City Group"] ).count().count())
 print(test["Type"].groupby(test["Type"] ).count().count() )
PARAMS = {
"BATCH_SIZE": 1024,
"EPOCHS": 100,
"DEVICE": torch.device('cuda' if torch.cuda.is_available() else 'cpu'),
"PATIENCE": 10,
"NUM_FOLDS": 10,
"TARGETS_OUTPUT": 206,
"LEARNING_RATE": 1e-2,
"WEIGHT_DECAY" : 1e-5
}<choose_model_class>
train["kOpen Year"] = train["Open Date"].str[6:12]
test["kOpen Year"] = test["Open Date"].str[6:12]
train["Open Month"] = train["Open Date"].str[0:2].astype(int)
 test["Open Month"] = test["Open Date"].str[0:2].astype(int)
 train["gOpen Year's"] = train["Open Date"].str[8:9] + "0's"
test["gOpen Year's"] = test["Open Date"].str[8:9] + "0's"
train['Open Month'] = pd.cut(train['Open Month'],4,labels=['1','2','3','4'])
 test['Open Month'] = pd.cut(test['Open Month'],4,labels=['1','2','3','4'])
 train, test = train.drop(["Id","Open Date","City"], axis=1), test.drop(["Id","Open Date","City"], axis=1 )
MAX_EPOCH=200
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=3e-2, weight_decay=PARAMS["WEIGHT_DECAY"]),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.8,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=5,
)<import_modules>
train["gOpen Year's"].groupby(train["gOpen Year's"] ).count()
from pytorch_tabnet.metrics import Metric<import_modules>
corr_abs = np.absolute(train.loc[:,"P1":"revenue"].corr() ["revenue"].drop("revenue")).sort_values(ascending=False)
 ex_var = pd.DataFrame(corr_abs[corr_abs<0.1] ).index
train1 = train.drop(ex_var, axis=1)
 test1 = test.drop(ex_var, axis=1 )
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-10)
 return np.mean(-aux )<normalization>
train_dm = pd.get_dummies(train ).drop("City Group_Other", axis=1)
 test_dm = pd.get_dummies(test ).drop("City Group_Other", axis=1)
 print(train_dm.shape, test_dm.shape)
 print(train_dm.columns,"
", test_dm.columns )
class SmoothBCEwLogits(nn.Module):
def __init__(self, weight=None, reduction='mean', smoothing=0.0005):
super(SmoothBCEwLogits, self ).__init__()
self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0005):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def validation_fn(model, val_dataloader, criterion=F.binary_cross_entropy_with_logits):
net = model['model']
total_val_loss = 0
outputs = []
with torch.no_grad() :
for idx,(X, y)in enumerate(val_dataloader):
X, y = X.to(PARAMS["DEVICE"]), y.to(PARAMS["DEVICE"])
 preds = net(X)
 total_val_loss += criterion(preds[:, :206], y[:, :206] ).item()
outputs.append(preds.sigmoid().detach().cpu().numpy())
 outputs = np.concatenate(outputs)
 return outputs, total_val_loss / len(val_dataloader)
 def predict_fn(model, test_X, model_name=None):
model.eval()
test_dataset = MoADataset(test_X)
 test_loader = DataLoader(test_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 outputs = []
if model_name != 'TabNet':
with torch.no_grad() :
for idx, X in enumerate(test_loader):
X = X.to(PARAMS["DEVICE"])
 preds = model(X)
 outputs.append(preds.sigmoid().detach().cpu().numpy())
 return np.concatenate(outputs)
 else:
preds = model['model'].predict(test_X)
 preds = 1/(1 + np.exp(-model.predict(X_val)))
 return preds
def train_fn(model, optimizer, loss, train_loader, val_loader, scheduler, fold):
net, name = model['model'].to(PARAMS["DEVICE"]), model['name']
no_improvement_steps = 0
min_loss_val = 1
for epoch in range(PARAMS["EPOCHS"]):
if no_improvement_steps == PARAMS["PATIENCE"]:
print("Early stopping!")
 break
total_loss_train = 0
for idx,(X, y)in enumerate(train_loader):
optimizer.zero_grad()
X, y = X.to(PARAMS["DEVICE"]), y.to(PARAMS["DEVICE"])
 preds = net(X)
 loss_val = loss(preds, y)
 loss_val.backward()
optimizer.step()
scheduler.step()
total_loss_train += loss_val.item()
total_loss_train /= len(train_loader)
 outputs, total_loss_val = validation_fn(model, val_loader)
 if total_loss_val < min_loss_val:
min_loss_val = total_loss_val
torch.save(net.state_dict() , "{}_{}.pth".format(name, fold))
no_improvement_steps = 0
else:
no_improvement_steps += 1
print("Fold: %d Epoch: %d, train loss:%.6f | validation loss: %.6f"%(fold, epoch, total_loss_train, total_loss_val))
model['model'].load_state_dict(torch.load("{}_{}.pth".format(name, fold)))
 return model['model'], outputs
def return_model(model_name, input_size, target_size, hidden_size):
print(model_name)
 if model_name == 'ResNet':
return {"name":model_name,
"model":ResidualNet(input_size, target_size, 512)}
elif model_name == 'ModelMLP':
return {"name": model_name,
"model": ModelMLP(input_size, target_size, hidden_size)
 }
elif model_name == 'Model':
return {'name': model_name,
'model': Model(input_size, target_size, hidden_size)}
elif model_name == 'simple':
return {"name":'simple',
'model': SimpleNet(input_size, target_size, hidden_size)}
else:
raise RuntimeError("Such network doesn't exists")
 def train(model_names, train_X, train_y, groups, n_seeds, transfer_learning=None):
print("Train shape:", train_X.shape, " train_y shape:", train_y.shape)
 models_predictions = []
trained_models = []
for model_name in model_names:
val = np.random.randint(8, 30)
 n_hidden = np.random.randint(512, 1524)
 group_kfold = MultilabelStratifiedKFold(n_splits=PARAMS["NUM_FOLDS"], shuffle=True)
 preds = np.zeros(( train_y.shape[0], train_y.shape[1]), dtype=float)
 loss = SmoothBCEwLogits()
folds_models = []
for idx,(train_index, test_index)in enumerate(group_kfold.split(train_X, train_y)) :
X_train, y_train = train_X[train_index], train_y[train_index]
X_val, y_val = train_X[test_index], train_y[test_index]
if model_name != "TabNet":
if transfer_learning is not None:
model = return_model(model_name, transfer_learning['in_size'], transfer_learning['out_size'])
 model["model"].load_state_dict(torch.load("{}_{}.pth".format(model["name"], idx)))
 model["model"].get_freeze(train_y.shape[-1], True)
 else:
model = return_model(model_name, X_train.shape[-1], y_train.shape[-1], n_hidden)
 print("Train - Test:", len(train_index), len(test_index))
train_dataset = MoADataset(X_train, y_train)
 val_dataset = MoADataset(X_val, y_val)
 train_loader = DataLoader(train_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=True)
 val_loader = DataLoader(val_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 optimizer = Adam(model["model"].parameters() , lr=PARAMS["LEARNING_RATE"], weight_decay=PARAMS["WEIGHT_DECAY"])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=PARAMS["EPOCHS"],
steps_per_epoch=len(train_loader))
model, outputs = train_fn(model, optimizer, loss, train_loader, val_loader, scheduler, idx)
 preds[test_index] = outputs
else:
tabnet_params["n_a"] = val
tabnet_params['n_d'] = val
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train[:, :206],
eval_set=[(X_val, y_val[:, :206])],
eval_name = ["validation loss"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=PARAMS['PATIENCE'], batch_size=PARAMS["BATCH_SIZE"],
virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=SmoothBCEwLogits())
 preds[test_index] = 1/(1 + np.exp(-model.predict(X_val)))
 folds_models.append(model)
 trained_models.append({'name': model_name, 'fold_models':folds_models})
 models_predictions.append(preds)
 return trained_models, models_predictions<prepare_x_and_y>
train_fix = pd.DataFrame(train_dm, columns=a)
 test_fix = pd.DataFrame(test_dm, columns=a )
print("preds dimensions:", train_targets_scored.shape,train_targets_nonscored.shape)
 aux_train_y = train_targets_nonscored.iloc[:, 1:].to_numpy()
merged_train = train_targets_scored.merge(train_targets_nonscored, on='sig_id' ).iloc[:, 1:].to_numpy()
train_X = train_features.iloc[:, 1:-1].to_numpy()
train_y = train_targets_scored.iloc[:, 1:].to_numpy()
test_X = test_features.iloc[:, 1:].to_numpy()
groups = train_features['drug_id']<define_search_model>
train_fix = train_fix.fillna(0)
 test_fix = test_fix.fillna(0 )
input_size, target_size = train_X.shape[-1], train_y.shape[-1]
MLP = ModelMLP(input_size, target_size, 1024)
 ResNet = ResidualNet(input_size, target_size, 512)
 model_names = [ 'simple','TabNet', "Model", "TabNet", 'simple', 'Model', 'TabNet', 'simple', "Model"]<train_model>
X, y = train_fix.drop(["revenue"], axis=1), train_dm["revenue"]
test_fix = test_fix.drop("revenue", axis=1 )
transfer_learning = {'in_size':train_X.shape[-1],
'out_size': aux_train_y.shape[-1]}
models, predictions = train(model_names, train_X, train_y, groups, 1 )<predict_on_test>
kX = X.drop(X.loc[:,X.columns.str.startswith('gOpen')], axis=1)
 gX = X.drop(X.loc[:,X.columns.str.startswith('kOpen')], axis=1)
 ktest_fix = test_fix.drop(test_fix.loc[:,test_fix.columns.str.startswith('gOpen')], axis=1)
 gtest_fix = test_fix.drop(test_fix.loc[:,test_fix.columns.str.startswith('kOpen')], axis=1)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.3, shuffle=True, random_state=42)
def make_folds_predictions(models, test_X):
all_predictions = []
for model in models:
name = model['name']
folds_models = model['fold_models']
predictions = []
for fold_model in folds_models:
predictions.append(predict_fn(fold_model, test_X, name))
all_predictions.append(sum(predictions)/ len(predictions))
return all_predictions
def stack_predict(meta_models, weak_models, test_X, meta_with_source=False):
weak_preds = []
weak_preds = make_folds_predictions(weak_models, test_X)
 if meta_with_source:
stacked_input = np.concatenate(test_X + weak_preds, axis=-1)
 else:
stacked_input = np.concatenate(weak_preds, axis=-1)
 meta_predictions = make_folds_predictions(meta_models, stacked_input)
 final_predictions = sum(meta_predictions)/ len(meta_predictions)
 return final_predictions
def blend_stack_models(all_models, test_X):
predictions = []
for models in all_models:
predictions.append(stack_predict(models['meta_models'],
models['weak_models'],test_X))
return sum(predictions)/ len(predictions)
 def train_stack_models(weak_learners, meta_learners, train_X, train_y, meta_params, weak_params):
control_fold = KFold(n_splits=3, shuffle=True, random_state=0)
 predictions = np.zeros(train_y.shape)
 all_models = []
for fold_num,(train_index_ctl, test_index_ctl)in enumerate(control_fold.split(train_X, train_y)) :
weak_folds = MultilabelStratifiedKFold(n_splits=3, shuffle=True, random_state=1)
 train_X_ctl, train_y_ctl = train_X[train_index_ctl], train_y[train_index_ctl]
test_X_ctl, test_y_ctl = train_X[test_index_ctl], train_y[test_index_ctl]
weak_meta_models = []
for train_index, test_index in weak_folds.split(train_X_ctl, train_y_ctl):
weak_train_X, weak_train_y = train_X_ctl[train_index], train_y_ctl[train_index]
meta_train_X, meta_train_y = train_X_ctl[test_index], train_y_ctl[test_index]
PARAMS['NUM_FOLDS'] = 3
PARAMS['EPOCHS'] = 30
print("-------------TRAIN WEAK MODELS-------------")
 weak_models, _ = train(weak_learners, weak_train_X, weak_train_y, None, 1)
 weak_predictions = make_folds_predictions(weak_models, meta_train_X)
 weak_predictions = np.concatenate(weak_predictions, axis=-1)
 print("------------TRAIN META MODELS--------------")
 PARAMS['EPOCHS'] = 100
meta_models, _ = train(meta_learners, weak_predictions, meta_train_y, None, 1)
 weak_meta_models.append({ 'weak_models': weak_models,
'meta_models': meta_models})
 all_models.append(weak_meta_models)
 preds = blend_stack_models(weak_meta_models, test_X_ctl)
 print("Preds shape", preds.shape, predictions[test_index_ctl].shape)
 predictions[test_index_ctl] = preds
return [predictions], all_models 
<compute_test_metric>
gbm1 = GradientBoostingRegressor(n_estimators=200, learning_rate=0.01, criterion='mse', random_state=42)
 gb0 = GradientBoostingRegressor(n_estimators=150, learning_rate=0.01, criterion='mse', random_state=42)
 gb1 = GradientBoostingRegressor(n_estimators=100, learning_rate=0.01, criterion='mse', random_state=42)
 gb2 = GradientBoostingRegressor(n_estimators=75, learning_rate=0.005, criterion='mse', random_state=42)
 gb3 = GradientBoostingRegressor(n_estimators=50, learning_rate=0.005, criterion='mse', random_state=42)
 knn = KNeighborsRegressor()
def compute_score(y_pred, y_true):
assert y_pred.shape == y_true.shape
score = 0
scores = []
for i in range(PARAMS["TARGETS_OUTPUT"]):
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_
scores.append(score_)
 return score / PARAMS["TARGETS_OUTPUT"], scores<predict_on_test>
def rmse(mod):
pred_tr = mod.predict(X_train)
 pred_val = mod.predict(X_valid)
 rmse_tr = round(sqrt(mean_squared_error(pred_tr, y_train)) , 4)
 rmse_val = round(sqrt(mean_squared_error(pred_val, y_valid)) , 4)
 print(f"学習データに対するスコア:{rmse_tr}
検証データに対するスコア:{rmse_val}" )
def find_best(scores):
num_targets = len(scores[0])
 target_model = {}
for i in range(num_targets):
neurals_target_scores = list([scores[j][i] for j in range(len(scores)) ])
 min_score = min(neurals_target_scores)
 index = neurals_target_scores.index(min_score)
 target_model[i] = index
return target_model
def create_final_prediction(models_predictions, target_model):
final_prediction = np.zeros(models_predictions[0].shape)
 for i in range(final_prediction.shape[-1]):
final_prediction[:, i] = models_predictions[target_model[i]][:, i]
return final_prediction<groupby>
def cross_rmse(mod, X):
tr_scores = cross_val_score(mod, X, y, scoring='neg_root_mean_squared_error',cv=10)
 val_scores = cross_val_score(mod, X, y, scoring='neg_root_mean_squared_error', cv=10)
 val_r2 = cross_val_score(mod, X, y, scoring='r2', cv=10)
 検証データに対するスコア:{-val_scores}")
 print(f"学習に対する平均スコア:{-np.mean(tr_scores)}
検証データに対する平均スコア:{-np.mean(val_scores)}")
 print(f"学習に対するスコアの分散:{np.var(-tr_scores)}
検証に対するスコアの分散:{np.var(-val_scores)}")
 print(val_r2)
 print(np.mean(val_r2))
ensemble_prediction = sum([pred[:, :206] for pred in predictions])/ len(predictions)
 total_predictions = predictions + [ensemble_prediction]
print(len(total_predictions), len(predictions))<compute_test_metric>
gbm1.fit(gX, y)
 gb0.fit(gX, y)
 gb1.fit(gX, y)
 gb2.fit(gX, y)
 gb3.fit(gX, y)
 knn.fit(kX, y)
score = [compute_score(preds, train_y)[0] for preds in total_predictions]
scores = [compute_score(preds, train_y)[-1] for preds in total_predictions]
print(len(scores))<predict_on_test>
def submit(mod, test_fix):
pred = pd.DataFrame(mod.predict(test_fix))
submit = pd.concat([test_id, pred], axis=1 ).rename(columns={0: 'Prediction'})
 return submit
target_model = find_best(scores)
 final_prediction = create_final_prediction(total_predictions, target_model )<compute_test_metric>
def ensemble_submit(mod1, mod2, rate1):
pred = pd.DataFrame(mod1.predict(gtest_fix)*rate1 + mod2.predict(ktest_fix)*(1-rate1))
submit = pd.concat([test_id, pred], axis=1 ).rename(columns={0: 'Prediction'})
 return submit
def ensemble_submit3(mod1, mod2, mod3, rate):
pred = pd.DataFrame(mod1.predict(gtest_fix)*rate[0] + mod2.predict(ktest_fix)*rate[1] + mod3.predict(gtest_fix)*rate[2])
 submit = pd.concat([test_id, pred], axis=1 ).rename(columns={0: 'Prediction'})
 return submit
def ensemble_submitx3(mod1, mod2, mod3):
pred = pd.DataFrame(( mod1.predict(gtest_fix)* mod2.predict(ktest_fix)* mod3.predict(gtest_fix)) **(1/3))
submit = pd.concat([test_id, pred], axis=1 ).rename(columns={0: 'Prediction'})
 return submit
compute_score(final_prediction, train_y)[0]<predict_on_test>
nn = MLPRegressor(random_state=42)
 nn.fit(gX, y )
def final_preds(all_models, test_X, weights=None):
if weights is None:
weights = [1/len(model_names)
 for i in range(len(model_names)) ]
models_preds = []
for models in all_models:
model_name = models['name']
fold_preds = []
for fold_model in models['fold_models']:
if model_name == 'TabNet':
preds = fold_model.predict(test_X)
 preds = 1/(1 + np.exp(-preds))
else:
preds = predict_fn(fold_model, test_X)
 fold_preds.append(preds[:, :206])
 print(len(fold_preds))
models_preds.append(sum(fold_preds)/len(fold_preds))
return models_preds 
<predict_on_test>
gb4_kn6_submit3.to_csv("gb4_kn6_submit3.csv", index=False)
 gbkngb_55_submit.to_csv("gbkngb_55_submit.csv", index=False)
 gbkngb_523_submit.to_csv("gbkngb_523_submit.csv", index=False)
 gbkngb_433_submit.to_csv("gbkngb_433_submit.csv", index=False)
 test55.to_csv("test55.csv", index=False)