Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<compute_test_metric>
HTML('<center><iframe width="560" height="315" src="https://www.youtube.com/embed/PGzT3cTPah8" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe></center>' )
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )<split>
warnings.filterwarnings('ignore')
num_folds = 10
softmax = False
kf = GroupKFold(n_splits=num_folds)
 y = train_df["AdoptionSpeed"]
fold_splits = kf.split(train_df, y, rescuerID)
 oof = np.zeros(( y.shape))
test_preds = np.zeros(( test_df.shape[0]))
if softmax == True:
test_preds = np.zeros(( test_df.shape[0], 5))
oof = np.zeros(shape =(y.shape[0], 5))
for i,(dev_index, val_index)in enumerate(fold_splits):
tr_df = train_df.iloc[dev_index, :]
val_df = train_df.iloc[val_index, :]
X_train = get_input_features(tr_df)
 X_valid = get_input_features(val_df)
 X_test = get_input_features(test_df)
 model = make_model(softmax = softmax)
 ckpt = ModelCheckpoint("model" + str(i)+ ".h5", monitor='val_loss', save_best_only = True, verbose = False)
 rlr = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=5, mode='auto', verbose = False)
 if softmax == True:
y_train = np.zeros(shape =(tr_df.shape[0], 5))
y_valid = np.zeros(shape =(val_df.shape[0], 5))
for j, l in enumerate(tr_df['AdoptionSpeed'].values):
j = int(j)
 l = int(l)
 y_train[j,l] = 1
for j, l in enumerate(val_df['AdoptionSpeed'].values):
j = int(j)
 l = int(l)
 y_valid[j,l] = 1
else:
y_train = tr_df['AdoptionSpeed'].values / 4
y_valid = val_df['AdoptionSpeed'].values / 4
hist = model.fit(X_train, y_train, validation_data =(X_valid,y_valid), batch_size = 500, epochs = 35, verbose = 2, callbacks = [ckpt, rlr])
 model.load_weights("model" + str(i)+ ".h5")
 if softmax == True:
val_preds = model.predict(X_valid)
 oof[val_index] = val_preds
test_pred = model.predict(X_test)
 test_preds +=(test_pred/num_folds)
 else:
val_preds = model.predict(X_valid)[:, 0] * 4
y_valid = y_valid * 4
optR = OptimizedRounder()
optR.fit(val_preds, y_valid)
 coefficients = optR.coefficients()
val_pred_rounded = optR.predict(val_preds, coefficients)
 oof[val_index] = val_preds
test_pred = model.predict(X_test)[:, 0] * 4
test_preds +=(test_pred/num_folds)
 if softmax == True:
oof_rounded = np.argmax(oof, axis = 1)
 test_rounded = np.argmax(test_preds, axis = 1)
 else:
optR = OptimizedRounder()
optR.fit(oof, y)
 coefficients = optR.coefficients()
oof_rounded = optR.predict(oof, coefficients)
 test_rounded = optR.predict(test_preds, coefficients )<compute_test_metric>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
print(num_cols)
 print(quadratic_weighted_kappa(y, oof_rounded))
oof_nn = oof
test_preds_nn = test_preds 
<concatenate>
ROOT = '.. /input/lish-moa'
train_features_df = pd.read_csv(f'{ROOT}/train_features.csv')
 print(f'Train data has {train_features_df.shape[0]} rows and {train_features_df.shape[1]} columnns and looks like this:')
X_temp = deepcopy(pd.concat([train_df, test_df], axis = 0))<categorify>
dupRows_df = train_features_df[train_features_df.duplicated(subset = ['sig_id', 'cp_type'],
keep = False)]
dupRows_df.head()
print(f'There are {dupRows_df.shape[0]} duplicates.' )
n_bins = 5
for c in num_cols:
if X_temp[c].nunique() < 5:
continue
vals = X_temp[c].values.reshape(-1, 1)
 est = KBinsDiscretizer(n_bins=n_bins, encode='ordinal')
 est.fit(vals)
 vals = est.transform(vals ).reshape(1, -1)[0]
X_temp[c] = vals<split>
test_features_df = pd.read_csv(f'{ROOT}/test_features.csv')
 print(f'Test data has {test_features_df.shape[0]} rows and {test_features_df.shape[1]} columnns, has no duplicates and looks like this:')
 test_features_df.head()
train_df_ffm = X_temp.iloc[:train_df.shape[0], :]
test_df_ffm = X_temp.iloc[train_df.shape[0]:, :]<define_variables>
def intersec_using_sets_intersection(uniq_test, uniq_train):
return bool(set(uniq_test)& set(uniq_train))
field_features = defaultdict()
global max_val
max_val = 1
global max_val
line = []
label = []
dtypes = {}
for feature in num_cols:
dtypes[feature] = "float16"
for feature in cat_cols:
dtypes[feature] = "category"
dont_use = ['RescuerID', 'AdoptionSpeed', 'Description', 'PetID']
too_many_vals = [""]
categories = [k for k, v in dtypes.items() if k not in dont_use]
categories_index = dict(zip(categories, range(len(categories))))
def generate_ffm_file(index, train = True):
global max_val
ffeatures = []
if train == True:
path = 'train.libffm'
else:
path = 'valid.libffm'
with open(path, 'w')as the_file:
for t,(index, row)in enumerate(train_df_ffm.iloc[index, :].iterrows()):
if t % 5000 == 0:
print(t, len(field_features), max_val)
 label = [str(int(row['AdoptionSpeed'])) ]
ffeatures = []
for field in categories:
if field == 'AdoptionSpeed':
continue
feature = row[field]
if feature == '':
feature = "unk"
if field not in num_cols:
ff = field + '_____' + str(feature)
 else:
if feature == "unk" or float(feature)== -1:
ff = field + '_____' + str(0)
 else:
if field in too_many_vals:
ff = field + '_____' + str(int(round(math.log(1 + float(feature)))))
 else:
ff = field + '_____' + str(feature)
 if ff not in field_features:
if len(field_features)== 0:
field_features[ff] = 1
max_val += 1
else:
field_features[ff] = max_val + 1
max_val += 1
fnum = field_features[ff]
ffeatures.append('{}:{}:1'.format(categories_index[field], str(fnum)))
 line = label + ffeatures
the_file.write('{}
'.format(' '.join(line)) )<compute_train_metric>
def intersec_using_generator_expression(uniq_test, uniq_train):
return any(i in uniq_test for i in uniq_train )
optR = OptimizedRounder()
optR.fit(ffm_oof_train, y)
 coefficients = optR.coefficients()
train_ffm_rounded = optR.predict(ffm_oof_train, coefficients)
 print(quadratic_weighted_kappa(train_ffm_rounded, y))<compute_test_metric>
def time_sets() :
return intersec_using_sets_intersection(uniq_test, uniq_train)
 def time_generator() :
return intersec_using_generator_expression(uniq_test, uniq_train)
 duration_sets = timeit(time_sets, number = 3)
 duration_generator = timeit(time_generator, number = 3)
 print(f"Using the set-intersection takes {duration_sets / 3:.5f} sec, while the generator expression takes {duration_generator / 3:.3f} sec.Using set-intersection is {duration_generator/duration_sets:.0f} times faster!")
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted)) 
<create_dataframe>
intersec_using_sets_intersection(uniq_test, uniq_train )
train = deepcopy(train_df)
 test = deepcopy(test_df )<data_type_conversions>
sub_df = pd.read_csv(f"{ROOT}/sample_submission.csv")
 print(f"The sample submission contains: {sub_df.shape[0]} rows and {sub_df.shape[1]} columns." )
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()<drop_column>
def preprocess(df):
_df = df.copy()
_df.loc[:, 'cp_type'] = _df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 _df.loc[:, 'cp_dose'] = _df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del _df['sig_id']
return _df
train.drop(set(['Name', 'RescuerID', 'Description', 'Description1', 'PetID', 'AdoptionSpeed']), axis=1, inplace=True)
 test.drop(set(['Name', 'RescuerID', 'Description', 'Description1', 'PetID']), axis=1, inplace=True )<train_on_grid>
top_feats = [ 0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 14, 15,
16, 18, 19, 20, 21, 23, 24, 25, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 44, 45, 46,
48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76,
78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 92,
93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 105, 106, 107,
108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120,
121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 132, 133, 134,
135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
149, 150, 151, 152, 153, 154, 155, 157, 159, 160, 161, 163, 164,
165, 166, 167, 168, 169, 170, 172, 173, 175, 176, 177, 178, 180,
181, 182, 183, 184, 186, 187, 188, 189, 190, 191, 192, 193, 195,
197, 198, 199, 202, 203, 205, 206, 208, 209, 210, 211, 212, 213,
214, 215, 218, 219, 220, 221, 222, 224, 225, 227, 228, 229, 230,
231, 232, 233, 234, 236, 238, 239, 240, 241, 242, 243, 244, 245,
246, 248, 249, 250, 251, 253, 254, 255, 256, 257, 258, 259, 260,
261, 263, 265, 266, 268, 270, 271, 272, 273, 275, 276, 277, 279,
282, 283, 286, 287, 288, 289, 290, 294, 295, 296, 297, 299, 300,
301, 302, 303, 304, 305, 306, 308, 309, 310, 311, 312, 313, 315,
316, 317, 320, 321, 322, 324, 325, 326, 327, 328, 329, 330, 331,
332, 333, 334, 335, 338, 339, 340, 341, 343, 344, 345, 346, 347,
349, 350, 351, 352, 353, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 368, 369, 370, 371, 372, 374, 375, 376, 377,
378, 379, 380, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391,
392, 393, 394, 395, 397, 398, 399, 400, 401, 403, 405, 406, 407,
408, 410, 411, 412, 413, 414, 415, 417, 418, 419, 420, 421, 422,
423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435,
436, 437, 438, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450,
452, 453, 454, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465,
466, 468, 469, 471, 472, 473, 474, 475, 476, 477, 478, 479, 482,
483, 485, 486, 487, 488, 489, 491, 492, 494, 495, 496, 500, 501,
502, 503, 505, 506, 507, 509, 510, 511, 512, 513, 514, 516, 517,
518, 519, 521, 523, 525, 526, 527, 528, 529, 530, 531, 532, 533,
534, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547,
549, 550, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563,
564, 565, 566, 567, 569, 570, 571, 572, 573, 574, 575, 577, 580,
581, 582, 583, 586, 587, 590, 591, 592, 593, 595, 596, 597, 598,
599, 600, 601, 602, 603, 605, 607, 608, 609, 611, 612, 613, 614,
615, 616, 617, 619, 622, 623, 625, 627, 630, 631, 632, 633, 634,
635, 637, 638, 639, 642, 643, 644, 645, 646, 647, 649, 650, 651,
652, 654, 655, 658, 659, 660, 661, 662, 663, 664, 666, 667, 668,
669, 670, 672, 674, 675, 676, 677, 678, 680, 681, 682, 684, 685,
686, 687, 688, 689, 691, 692, 694, 695, 696, 697, 699, 700, 701,
702, 703, 704, 705, 707, 708, 709, 711, 712, 713, 714, 715, 716,
717, 723, 725, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738,
739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751,
752, 753, 754, 755, 756, 758, 759, 760, 761, 762, 763, 764, 765,
766, 767, 769, 770, 771, 772, 774, 775, 780, 781, 782, 783, 784,
785, 787, 788, 790, 793, 795, 797, 799, 800, 801, 805, 808, 809,
811, 812, 813, 816, 819, 820, 821, 822, 823, 825, 826, 827, 829,
831, 832, 833, 834, 835, 837, 838, 839, 840, 841, 842, 844, 845,
846, 847, 848, 850, 851, 852, 854, 855, 856, 858, 860, 861, 862,
864, 867, 868, 870, 871, 873, 874]
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model', num_iters = 1, num_folds = 19,):
cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], num_folds))
all_coefficients = np.zeros(( num_folds * num_iters, 4))
feature_importance_df = pd.DataFrame()
i = 1
for j in range(num_iters):
kf = GroupKFold(n_splits=num_folds)
 fold_splits = kf.split(train, target, rescuerID)
 for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test +=(pred_test_y / num_iters)
 pred_train[val_index] +=(pred_val_y/num_iters)
 all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
pred_full_test =(pred_full_test / num_folds)
 print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 7,
'learning_rate': 0.02,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.2,
'lambda_l2': 0.05,
'verbosity': -1,
'data_random_seed': 24,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance("gain"), coefficients, qwk 
<define_variables>
X_feature_cols = top_feats
Y_feature_cols = top_feats
no_drop = ['Color1', 'Color2', 'Color3', 'Dewormed', 'Fee', 'FurLength',
'Gender', 'Health', 'MaturitySize','PhotoAmt', 'Quantity','Sterilized', 'Type', 'Vaccinated',
'state_gdp', 'state_population',]<define_variables>
train_targets_df = pd.read_csv(f"{ROOT}/train_targets_scored.csv")
 del train_targets_df['sig_id']
Y_train = train_targets_df.astype(float ).values
X_train = preprocess(train_features_df ).astype(float ).values
X_test = preprocess(test_features_df ).astype(float ).values
X_feature_cols = preprocess(train_features_df ).columns
Y_feature_cols = train_targets_df.columns
keep_features =(['Age', 'Breed1', 'Breed2', 'Color1', 'Color2', 'Color3', 'Fee', 'Quantity',
'MaturitySize','FurLength','Vaccinated','Dewormed', 'Sterilized','Health','Quantity','Fee',
'VideoAmt','PhotoAmt', 'state_gdp','state_population', 'doc_sent_mag', 'doc_sent_score', 'vertex_x', 'vertex_y',
'bounding_confidence', 'bounding_importance','dominant_blue', 'dominant_green', 'dominant_red', 'dominant_pixel_frac',
'dominant_score', 'label_description', 'label_score',
'RescuerID_COUNT','State', 'Sterilized', 'agg_mean_Age', 'agg_mean_Color1',
'agg_mean_Color2', 'agg_mean_Color3', 'agg_mean_Dewormed',
'agg_mean_Fee', 'agg_mean_FurLength', 'agg_mean_Gender',
'agg_mean_PhotoAmt', 'agg_mean_Quantity', 'agg_mean_Sterilized',
'agg_mean_Vaccinated', 'agg_mean_VideoAmt', 'agg_std_Age',
'agg_std_Color1', 'agg_std_Color2', 'agg_std_Color3',
'agg_std_Dewormed', 'agg_std_Fee', 'agg_std_FurLength',
'agg_std_Gender', 'agg_std_PhotoAmt', 'agg_std_Quantity',
'agg_std_Sterilized', 'agg_std_Vaccinated', 'label_score',
'metadata_metadata_annots_score_MAX',
'metadata_metadata_annots_score_MEAN',
'metadata_metadata_annots_score_MIN',
'metadata_metadata_annots_score_SUM',
'metadata_metadata_annots_score_VAR',
'metadata_metadata_color_pixelfrac_MIN',
'metadata_metadata_color_pixelfrac_SUM',
'metadata_metadata_color_pixelfrac_VAR',
'metadata_metadata_color_score_SUM',
'metadata_metadata_color_score_VAR', 'vertex_x', 'vertex_y','RescuerID_UNIQUE']
+["svd1_" + str(i)for i in range(n_components)]
+["img_svd1_" + str(i)for i in range(n_components)]
+["img_svd2_" + str(i)for i in range(n_components)] 
)
<compute_train_metric>
def get_model(num_inputs, optimizer = 'ADAM', lr = 0.01):
"Creates and returns a model"
optimizer = tf.keras.optimizers.Adam(lr = lr)if optimizer == 'ADAM' else tf.keras.optimizers.SGD(lr = lr)
 model = tf.keras.Sequential([
Layers.Input(num_inputs),
Layers.BatchNormalization() ,
tfa.layers.WeightNormalization(tf.keras.layers.Dense(6144, activation = "elu")) ,
Layers.BatchNormalization() ,
Layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(6144, activation = "elu")) ,
Layers.BatchNormalization() ,
Layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation = "elu")) ,
Layers.BatchNormalization() ,
Layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = "sigmoid"))
])
 model.compile(loss = 'binary_crossentropy',
optimizer = optimizer,
metrics=["accuracy", "AUC"])
 return model
results = run_cv_model(train[keep_features], test[keep_features], y, runLGB, params, rmse, 'lgb' )<init_hyperparams>
metrics = []
for _target in train_targets_df.columns:
metrics.append(log_loss(train_targets_df.loc[:, _target], train_preds_df.loc[:, _target]))
print(f'OOF Metric: {np.mean(metrics)}' )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 1.0,
'colsample_bytree': 0.6,
'min_child_weight': 1.0,
'gamma' :.5,
'max_depth' : 5,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}<split>
sub_df.loc[test_features_df['cp_type']==1, Y_feature_cols] = 0
def run_xgb(params, X_train, X_test):
n_splits = 20
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = GroupKFold(n_splits=n_splits)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, y, rescuerID):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = y.iloc[train_idx]
y_val = y.iloc[valid_idx]
d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<define_variables>
sub_df.to_csv('submission.csv', index = False )