Kaggle competition: OSIC Pulmonary Fibrosis Progression
<SOS> metric: LaplaceLogLikelihood Kaggle data source: osic-pulmonary-fibrosis-progression<features_selection>
!pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index
!pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index
indicator.variables_<categorify>
import os
import cv2
import pydicom
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from tqdm.notebook import tqdm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error
from tensorflow_addons.optimizers import RectifiedAdam
from tensorflow.keras import Model
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.optimizers import Nadam
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from PIL import Image
X_train = indicator.transform(X_train)
 X_test = indicator.transform(X_test)
 submission = indicator.transform(submission)
 X_train.shape, X_test.shape, submission.shape<train_model>
def seed_everything(seed=2020):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed )
num_imputer = imp.MeanMedianImputer(imputation_method = 'median',
variables = vars_to_impute, 
)
 num_imputer.fit(X_train)
 num_imputer.imputer_dict_<normalization>
seed_everything(42 )
X_train = num_imputer.transform(X_train)
 X_test = num_imputer.transform(X_test)
 submission = num_imputer.transform(submission)
 X_train.shape, X_test.shape, submission.shape<count_missing_values>
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )
[c for c in X_train.columns if X_train[c].isnull().sum() >0]<count_missing_values>
Dropout_model = 0.385
FVC_weight = 0.2
Confidence_weight = 0.2
[c for c in X_test.columns if X_test[c].isnull().sum() >0]<count_missing_values>
commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed'] )
[c for c in submission.columns if submission[c].isnull().sum() >0]<categorify>
n=0
commits_df.loc[n,'commit_num'] = 1
commits_df.loc[n,'Dropout_model'] = 0.4
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8111
bath_bsmt = creation.MathematicalCombination(
 variables_to_combine=['BsmtHalfBath', 'BsmtFullBath'],
math_operations=['sum'],
new_variables_names=['BsmtBath_total'], 
)
 bath_ground = creation.MathematicalCombination(
 variables_to_combine=['FullBath', 'HalfBath'],
math_operations=['sum'],
new_variables_names=['Bath_total'], 
)
 X_train = bath_bsmt.fit_transform(X_train)
 X_test = bath_bsmt.transform(X_test)
 submission = bath_bsmt.transform(submission)
 X_train = bath_ground.fit_transform(X_train)
 X_test = bath_ground.transform(X_test)
 submission = bath_ground.transform(submission)
 X_train.shape, X_test.shape, submission.shape<prepare_x_and_y>
n=1
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8105
X_train.reset_index(drop=True, inplace=True)
 y_train.reset_index(drop=True, inplace=True )<split>
n=2
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.35
commits_df.loc[n,'LB_score'] = -6.8158
def nested_cross_val(model, grid):
cv_outer = KFold(n_splits=5, shuffle=True, random_state=1)
 outer_results = list()
for train_ix, test_ix in cv_outer.split(X_train):
xtrain, xtest = X_train.loc[train_ix, :], X_train.loc[test_ix, :]
ytrain, ytest = y_train[train_ix], y_train[test_ix]
cv_inner = KFold(n_splits=5, shuffle=True, random_state=1)
 search = GridSearchCV(model, grid, scoring='neg_root_mean_squared_error', cv=cv_inner, refit=True)
 search.fit(xtrain, ytrain)
 yhat = search.predict(xtest)
 rmse = mean_squared_error(ytest, yhat, squared=False)
 outer_results.append(rmse)
 print('>rmse_outer=%.3f, rmse_inner=%.3f, cfg=%s' %(rmse, search.best_score_, search.best_params_))
print('rmse_outer: %.3f +- %.3f' %(np.mean(outer_results), np.std(outer_results)))
 return search.fit(X_train, y_train )<choose_model_class>
n=3
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8107
gbm = GradientBoostingRegressor(
 loss='ls',
n_estimators=100,
criterion='friedman_mse',
min_samples_split=2,
max_depth=3,
random_state=0,
n_iter_no_change=2,
tol=0.0001, 
)
 param_grid = dict(
 loss=['ls', 'huber'],
n_estimators=[10, 20, 50, 100, 200, 500, 1000, 2000],
min_samples_split=[0.1, 0.3, 0.5],
max_depth=[1,2,3,4,None], 
)
 search = nested_cross_val(gbm, param_grid )<compute_train_metric>
n=4
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.3
commits_df.loc[n,'LB_score'] = -6.8125
X_train_preds = search.predict(X_train)
 X_test_preds = search.predict(X_test)
 submission_preds = search.predict(submission)
 print('Train rmse: ', mean_squared_error(y_train, X_train_preds, squared=False))
print('Test rmse: ', mean_squared_error(y_test, X_test_preds, squared=False))
print()
print('Train r2: ', r2_score(y_train, X_train_preds))
print('Test r2: ', r2_score(y_test, X_test_preds))<prepare_output>
n=5
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8089
np.exp(submission_preds )<choose_model_class>
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100
lgbm_param = {
"num_leaves": [6, 8, 20, 30],
"max_depth": [2, 4, 6, 8, 10],
"n_estimators": [50, 100, 200, 500],
'colsample_bytree': [0.3, 1.0],
}
lgbm = LGBMRegressor(
 learning_rate = 0.1,
min_child_weight = 0.4,
objective='regression',
random_state=0)
 search = nested_cross_val(lgbm, lgbm_param )<compute_train_metric>
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096
X_train_preds = search.predict(X_train)
 X_test_preds = search.predict(X_test)
 submission_preds = search.predict(submission)
 print('Train rmse: ', mean_squared_error(y_train, X_train_preds, squared=False))
print('Test rmse: ', mean_squared_error(y_test, X_test_preds, squared=False))
print()
print('Train r2: ', r2_score(y_train, X_train_preds))
print('Test r2: ', r2_score(y_test, X_test_preds))<prepare_output>
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100
np.exp(submission_preds )<save_to_csv>
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
my_submission = pd.DataFrame({'Id': id_, 'SalePrice': np.exp(submission_preds)})
 my_submission.to_csv('submission_lgbm_full.csv', index=False )<train_model>
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093
sel_perf = sel.SelectBySingleFeaturePerformance(
 estimator=DecisionTreeRegressor(random_state=2, max_depth=2),
scoring='r2',
cv=3,
threshold=0.01,
variables=None, 
)
 sel_perf.fit(X_train, y_train )<define_variables>
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283
print('total features: ', X_train.shape[1])
 print('features to drop: ', len(sel_perf.features_to_drop_))
print('remaining features: ', X_train.shape[1] - len(sel_perf.features_to_drop_))<train_model>
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087
rfm = RandomForestRegressor(
 n_estimators=100,
random_state=0,
max_depth=2, 
)
 rfe = sel.RecursiveFeatureElimination(
 estimator = rfm,
scoring ='neg_root_mean_squared_error',
cv=3,
threshold=0.001,
variables=None, 
)
 rfe.fit(X_train, y_train )<find_best_params>
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090
rfe.initial_model_performance_<categorify>
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092
X_train = sel_perf.transform(X_train)
 X_test = sel_perf.transform(X_test)
 submission = sel_perf.transform(submission)
 X_train.shape, X_test.shape, submission.shape<choose_model_class>
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093
gbm = GradientBoostingRegressor(
 loss='ls',
n_estimators=100,
criterion='friedman_mse',
min_samples_split=2,
max_depth=3,
random_state=0,
n_iter_no_change=2,
tol=0.0001,
)<define_search_space>
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402
param_grid = dict(
 loss=['ls', 'huber'],
n_estimators=[10, 20, 50, 100, 200, 500, 1000, 2000],
min_samples_split=[0.1, 0.3, 0.5],
max_depth=[1,2,3,4,None],
)<compute_train_metric>
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1
search = nested_cross_val(gbm, param_grid )<prepare_output>
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0
results = pd.DataFrame(search.cv_results_)
 results.head()<sort_values>
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )
results.sort_values(by='mean_test_score', ascending=False, inplace=True)
 results.reset_index(drop=True, inplace=True)
 results[['params','mean_test_score', 'std_test_score']]<predict_on_test>
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )
X_train_preds = search.predict(X_train)
 X_test_preds = search.predict(X_test)
 submission_preds = search.predict(submission )<compute_test_metric>
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )
print('Train rmse: ', mean_squared_error(y_train, X_train_preds, squared=False))
print('Test rmse: ', mean_squared_error(y_test, X_test_preds, squared=False))<compute_test_metric>
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a
print('Train r2: ', r2_score(y_train, X_train_preds))
print('Test r2: ', r2_score(y_test, X_test_preds))<prepare_output>
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )
np.exp(submission_preds )<save_to_csv>
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )
my_submission = pd.DataFrame({'Id': id_, 'SalePrice': np.exp(submission_preds)})
 my_submission.to_csv('submission_gbm_small.csv', index=False )<choose_model_class>
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )
lgbm_param = {
"num_leaves": [6, 8, 20, 30],
"max_depth": [2, 4, 6, 8, 10],
"n_estimators": [50, 100, 200, 500],
'colsample_bytree': [0.3, 1.0],
}
lgbm = LGBMRegressor(
 learning_rate = 0.1,
min_child_weight = 0.4,
objective='regression',
random_state=0 )<compute_test_metric>
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )
search = nested_cross_val(lgbm, lgbm_param )<predict_on_test>
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )
X_train_preds = search.predict(X_train)
 X_test_preds = search.predict(X_test)
 submission_preds = search.predict(submission )<compute_test_metric>
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )
print('Train rmse: ', mean_squared_error(y_train, X_train_preds,squared=False))
print('Test rmse: ', mean_squared_error(y_test, X_test_preds,squared=False))<compute_test_metric>
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()
print('Train r2: ', r2_score(y_train, X_train_preds))
print('Test r2: ', r2_score(y_test, X_test_preds))<prepare_output>
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )
np.exp(submission_preds )<save_to_csv>
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )
my_submission = pd.DataFrame({'Id': id_, 'SalePrice': np.exp(submission_preds)})
 my_submission.to_csv('submission_lgbm_small.csv', index=False )<compute_test_metric>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique() )
def mean_absolute_percentage_error(y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
 return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 100<load_from_csv>
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )
df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )
cat_columns = [cname for cname in df.columns if df[cname].dtype == "object"]<data_type_conversions>
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base
for col in df.columns:
if col in cat_columns:
df[col] = df[col].astype('category')
 for col in test.columns:
if col in cat_columns:
test[col] = test[col].astype('category' )<split>
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )
X = df.drop('SalePrice', axis=1)
 y = df['SalePrice']
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=0 )<find_best_model_class>
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']
%%time
best_score = -1
num_leaves = [6, 7]
n_estimators = [100, 400, 450, 500, 550]
learning_rate = [0.1, 0.075, 0.08, 0.07, 0.065]
for rate in learning_rate:
for leave in num_leaves:
for estimator in n_estimators:
lgb = lightgbm.LGBMRegressor(num_leaves=leave, n_estimators=estimator, learning_rate=rate, random_state=0)
 scores = -1 * cross_val_score(lgb, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error')
 if np.mean(scores)< best_score or best_score == -1:
best_score = np.mean(scores)
 best_leaves = leave
best_estimators = estimator
best_learning_rate = rate
print(np.sqrt(best_score))
print(f'
Средняя абсолютная ошибка: {np.sqrt(best_score)} %, листья: {best_leaves}, деревья: {best_estimators}, скорость обучения: {best_learning_rate}' )<train_model>
tr = data.loc[data.WHERE=='train']
chunk = data.loc[data.WHERE=='val']
sub = data.loc[data.WHERE=='test']
del data
lgb = lightgbm.LGBMRegressor(num_leaves=best_leaves, n_estimators=best_estimators, learning_rate=best_learning_rate)
 lgb.fit(X, y, categorical_feature=cat_columns, verbose=False )<predict_on_test>
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
 def score(y_true, y_pred):
tf.dtypes.cast(y_true, tf.float32)
 tf.dtypes.cast(y_pred, tf.float32)
 sigma = y_pred[:, 2] - y_pred[:, 0]
fvc_pred = y_pred[:, 1]
sigma_clip = tf.maximum(sigma, C1)
 delta = tf.abs(y_true[:, 0] - fvc_pred)
 delta = tf.minimum(delta, C2)
 sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))
metric =(delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
 return K.mean(metric)
 def qloss(y_true, y_pred):
qs = [0.2, 0.50, 0.8]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def mloss(_lambda):
def loss(y_true, y_pred):
return _lambda * qloss(y_true, y_pred)+(1 - _lambda)*score(y_true, y_pred)
 return loss
def make_model(nh):
z = L.Input(( nh,), name="Patient")
 x = L.Dense(100, activation="relu", name="d1" )(z)
 x = L.Dense(100, activation="relu", name="d2" )(x)
 p1 = L.Dense(3, activation="linear", name="p1" )(x)
 p2 = L.Dense(3, activation="relu", name="p2" )(x)
 preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
name="preds" )([p1, p2])
 model = M.Model(z, preds, name="CNN")
 model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])
 return model
pred = lgb.predict(test )<save_to_csv>
y = tr['FVC'].values.astype('float32')
 z = tr[FE].values.astype('float32')
 ze = sub[FE].values.astype('float32')
 nh = z.shape[1]
pe = np.zeros(( ze.shape[0], 3))
pred = np.zeros(( z.shape[0], 3))
output = pd.DataFrame({'Id': test.Id,
'SalePrice': pred})
 output.to_csv('submission.csv', index=False )<compute_test_metric>
NFOLD = 5
kf = KFold(n_splits=NFOLD )
lgb.score(X_valid, y_valid )<set_options>
%%time
cnt = 0
EPOCHS = 855
for tr_idx, val_idx in kf.split(z):
cnt += 1
print(f"FOLD {cnt}")
 net = make_model(nh)
 net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS,
validation_data=(z[val_idx], y[val_idx]), verbose=0)
 print("train", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))
print("val", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))
print("predict val...")
 pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)
 print("predict test...")
 pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0)/ NFOLD
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 pd.set_option('display.max_columns', None)
 train.head(5)
 test.head(5)
 print("The train data size before dropping Id feature is : {} ".format(train.shape))
print("The test data size before dropping Id feature is : {} ".format(test.shape))
print("*" * 70)
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 print("The train data size after dropping Id feature is : {} ".format(train.shape))
print("The test data size after dropping Id feature is : {} ".format(test.shape))
ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
 ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 sns.distplot(train['SalePrice'] , fit=norm, color = 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
train["SalePrice"] = np.log1p(train["SalePrice"])
 sns.distplot(train['SalePrice'] , fit=norm, color= 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
house_data = pd.concat(( train, test)).reset_index(drop=True)
 house_data.drop(['SalePrice'], axis=1, inplace=True)
 print("house_data size is : {}".format(house_data.shape))
data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head(20)
 f, ax = plt.subplots(figsize=(20, 12))
plt.xticks(rotation='60')
 sns.barplot(x=data_null.index, y=data_null, color = 'b')
 plt.xlabel('Features', fontsize=15)
 plt.ylabel('Percent of missing values', fontsize=15)
 plt.title('Percent missing data by feature', fontsize=15)
 corrmat = train.corr()
plt.subplots(figsize=(35,35))
sns.heatmap(corrmat, vmax=0.9, square=True, annot=True)
 house_data["PoolQC"] = house_data["PoolQC"].fillna("None")
 house_data["MiscFeature"] = house_data["MiscFeature"].fillna("None")
 house_data["Alley"] = house_data["Alley"].fillna("None")
 house_data["Fence"] = house_data["Fence"].fillna("None")
 house_data["FireplaceQu"] = house_data["FireplaceQu"].fillna("None")
 house_data["LotFrontage"] = house_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
house_data[col] = house_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
house_data[col] = house_data[col].fillna('None')
 house_data["MasVnrType"] = house_data["MasVnrType"].fillna("None")
 house_data["MasVnrArea"] = house_data["MasVnrArea"].fillna(0)
 house_data['MSZoning'] = house_data['MSZoning'].fillna(house_data['MSZoning'].mode() [0])
 house_data = house_data.drop(['Utilities'], axis=1)
 house_data["Functional"] = house_data["Functional"].fillna("Typ")
 house_data['Electrical'] = house_data['Electrical'].fillna(house_data['Electrical'].mode() [0])
 house_data['KitchenQual'] = house_data['KitchenQual'].fillna(house_data['KitchenQual'].mode() [0])
 house_data['Exterior1st'] = house_data['Exterior1st'].fillna(house_data['Exterior1st'].mode() [0])
 house_data['Exterior2nd'] = house_data['Exterior2nd'].fillna(house_data['Exterior2nd'].mode() [0])
 house_data['SaleType'] = house_data['SaleType'].fillna(house_data['SaleType'].mode() [0])
 house_data['MSSubClass'] = house_data['MSSubClass'].fillna("None")
 data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head()
house_data['MSSubClass'] = house_data['MSSubClass'].apply(str)
 house_data['OverallCond'] = house_data['OverallCond'].astype(str)
 house_data['YrSold'] = house_data['YrSold'].astype(str)
 house_data['MoSold'] = house_data['MoSold'].astype(str)
 cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(house_data[c].values))
house_data[c] = lbl.transform(list(house_data[c].values))
print('Shape all_data: {}'.format(house_data.shape))
house_data['TotalSF'] = house_data['TotalBsmtSF'] + house_data['1stFlrSF'] + house_data['2ndFlrSF']
numeric_feats = house_data.dtypes[house_data.dtypes != "object"].index
skewed_feats = house_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10)
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
house_data[feat] = boxcox1p(house_data[feat], lam)
 house_data = pd.get_dummies(house_data)
 print(house_data.shape)
 train = house_data[:ntrain]
test = house_data[ntrain:]
n_folds = 8
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1)
 averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<install_modules>
sigma_opt = mean_absolute_error(y, pred[:, 1])
 unc = pred[:,2] - pred[:, 0]
sigma_mean = np.mean(unc)
 print(sigma_opt, sigma_mean )
<load_from_csv>
sub['FVC1'] = 1.*pe[:, 1]
sub['Confidence1'] = pe[:, 2] - pe[:, 0]
subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()
subm.loc[~subm.FVC1.isnull() ].head(10 )
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
Train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 Train = Train.set_index('Id')
 Test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 Test = Test.set_index('Id')
 preprocessing_path = '.. /input/house-price-eda'
train_scale = pd.read_csv(f'{preprocessing_path}/train_scaled.csv')
 train_scale = train_scale.set_index('Id')
 test_scale = pd.read_csv(f'{preprocessing_path}/test_scaled.csv')
 try:
test_scale = test_scale.rename(columns={'Unnamed: 0':'Id'})
 except:
pass
test_scale = test_scale.set_index('Id')
 Train = Train.loc[train_scale.index]
listing = glob.glob(f'{preprocessing_path}/*_one_hot_pickle')
 for model_file in listing:
col = model_file.split('_one_')[0]
col = col.split('/')[-1]
enc = pickle.load(open(model_file, 'rb'))
new_cols = pd.DataFrame(enc.transform(Train[[col]] ).toarray() ,
columns=f'{col}_' + enc.categories_[0])
 new_cols.index = Train.index
Train = pd.concat([Train, new_cols], axis = 1)
 Train = Train.drop(col, axis = 1)
 for log_col in train_scale.columns[train_scale.columns.str.endswith('log')]:
col = log_col.split('_log')[0]
Train[log_col] = np.log(Train[col])
 Train = Train.drop(col, axis = 1)
 Train = Train[train_scale.columns]
for model_file in listing:
col = model_file.split('_one_')[0]
col = col.split('/')[-1]
enc = pickle.load(open(model_file, 'rb'))
new_cols = pd.DataFrame(enc.transform(Test[[col]] ).toarray() ,
columns=f'{col}_' + enc.categories_[0])
 new_cols.index = Test.index
Test = pd.concat([Test, new_cols], axis = 1)
 Test = Test.drop(col, axis = 1)
 for log_col in test_scale.columns[test_scale.columns.str.endswith('log')]:
col = log_col.split('_log')[0]
Test[log_col] = np.log(Test[col])
 Test = Test.drop(col, axis = 1)
 Test = Test[test_scale.columns]<feature_engineering>
subm.loc[~subm.FVC1.isnull() ,'FVC'] = subm.loc[~subm.FVC1.isnull() ,'FVC1']
if sigma_mean<70:
subm['Confidence'] = sigma_opt
else:
subm.loc[~subm.FVC1.isnull() ,'Confidence'] = subm.loc[~subm.FVC1.isnull() ,'Confidence1']
<feature_engineering>
otest = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 for i in range(len(otest)) :
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]
subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1
<define_variables>
subm[["Patient_Week","FVC","Confidence"]].to_csv("submission_regression.csv", index=False )
def getFoldValidation(length, K = 4):
diff = int(length/K)
 fraction = length % K
batchValidation = [[k*diff,(k + 1)*diff] for k in range(K)]
batchValidation[-1][1] = batchValidation[-1][1] + fraction - 1
return batchValidation
batchValidation = np.array(getFoldValidation(len(Train)) )<categorify>
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()
def getTrainTestBatch() :
fold = 0
while fold < batchValidation.shape[0]:
train_batch = pd.concat([Train.iloc[batchValidation[i][0]:batchValidation[i][1]]
for i in range(batchValidation.shape[0])if i != fold])
 test_batch = Train.iloc[batchValidation[fold][0]:batchValidation[fold][1]]
yield train_batch, test_batch, fold
fold += 1
train_batch, test_batch, fold = next(getTrainTestBatch())
<import_modules>
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_test_metric>
df = df1[['Patient_Week']].copy()
df['FVC'] = FVC_weight*df1['FVC'] +(1-FVC_weight)*df2['FVC']
df['Confidence'] = Confidence_weight*df1['Confidence'] +(1-Confidence_weight)*df2['Confidence']
df.head()
<import_modules><EOS> 
df.to_csv('submission.csv', index=False )