Kaggle competition: Don't Overfit! II
concatenate, Flatten)
 p_rebuild_datasets = True
p_subsample_lines = 0
k_datasets_prefix = ".. /input/"
train_data_file = "application_train.csv"
predict_data_file = "application_test.csv"
bureau_data_file = "bureau.csv"
bureau_balance_data_file = "bureau_balance.csv"
pos_cash_balance_data_file = "POS_CASH_balance.csv"
credit_card_balance_data_file = "credit_card_balance.csv"
previous_application_data_file = "previous_application.csv"
installments_payments_data_file = "installments_payments.csv"
p_data_model_version = "KerasNN_PAConv_SimpleV5"
p_objective = "TARGET"
p_index = "SK_ID_CURR"<feature_engineering>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def preproc_ds(df1, df2,combined=True,keys=None):
if combined:
df = df1.append(df2, ignore_index=True)[data.columns.tolist() ]
categoricals = df.columns[df.dtypes=="O"]
for col in df.columns[df.dtypes!="O"]:
if col != "TARGET" and not col in keys:
mean = df[col].mean()
std = df[col].std()
df1[col] =(df1[col].fillna(mean)- mean)/ std
df2[col] =(df2[col].fillna(mean)- mean)/ std
print("combined")
 for col in df.columns[df.dtypes=="O"]:
print(col)
 le = LabelEncoder()
le.fit(df[col].fillna('PreProcEmpty'))
print(le.classes_)
 df1[col] = le.transform(df1[col].fillna('PreProcEmpty'))
df2[col] = le.transform(df2[col].fillna('PreProcEmpty'))
return(df1, df2, categoricals)
 else:
print("single")
 categoricals = df1.columns[df1.dtypes=="O"]
for col in df1.columns[df1.dtypes!="O"]:
if col != "TARGET" and not col in keys:
mean = df1[col].mean()
std = df1[col].std()
df1[col] =(df1[col].fillna(mean)- mean)/ std
for col in df1.columns[df1.dtypes=="O"]:
print(col)
 le = LabelEncoder()
le.fit(df1[col].fillna('PreProcEmpty'))
print(le.classes_)
 df1[col] = le.transform(df1[col].fillna('PreProcEmpty'))
return(df1,categoricals)
<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def read_df(filename,subdir,index):
df = pd.read_csv(subdir + filename ).sort_values(by=index)
 if index != None:
df.set_index(index, inplace=False)
 return(df)
<define_variables>
train.target.value_counts()
def generate_conv_tensor_simple(X,X_index, products,PA_cp):
PA = PA_cp[PA_cp[p_index].isin(X[p_index])]
st_products = 0
samples = len(X)
 PA_length = len(PA)
 max_len_detected = 0
max_prod_detected = 0
nPA_in_vars = len(PA.iloc[0])
 PA_nX_tensor = np.zeros(( samples,products,nPA_in_vars))
X_sample=0
x_progress_tic = samples/100
x_progress = 0
chunks_PA =(1,products,nPA_in_vars-2)
 PA_iloc = 0
for X_iloc in range(samples):
X_row = X.iloc[X_iloc,:]
PA_row = PA.iloc[PA_iloc,:]
product_n = 0
while PA_row[X_index] == X_row[X_index]:
if(product_n < products):
PA_nX_tensor[X_iloc,product_n,:]= PA_row
st_products = st_products + 1
product_n = product_n + 1
PA_iloc = PA_iloc + 1
if PA_iloc >= PA_length: break
PA_row = PA.iloc[PA_iloc,:]
x_progress = x_progress + 1
if x_progress > x_progress_tic:
x_progress_tic = x_progress_tic + samples/100
prog = "\r" + str(x_progress/samples*100)
 print(prog, end="", flush=True)
 print("Products ",st_products)
 return(PA_nX_tensor[:,:,2:],chunks_PA )<categorify>
train.target.value_counts() / len(train )
def pickledump(filename,dumpobj):
outfile = open(filename,'wb')
 pickle.dump(dumpobj,outfile, protocol=4)
 outfile.close()
def pickleload(filename):
print("Cargando pickle : ", filename)
 infile = open(filename,'rb')
 dumpobj = pickle.load(infile)
 infile.close()
return(dumpobj)
 def h5dump(dataset,array,hp5yf,chunks):
return(hp5yf.create_dataset(dataset, data=array, compression="lzf",chunks=chunks))
def build_dump_tensor(X,p_index,p_products,h5py_dataset,PA,h5pyfilename):
print("Saving :", h5py_dataset)
 ts_pa,chunks_pa = generate_conv_tensor_simple(X,p_index, p_products,PA)
 h5pyf = h5py.File(h5pyfilename, "a", driver = "core")
 ts_pa = h5dump(h5py_dataset, ts_pa,h5pyf,chunks_pa)
 h5pyf.flush()
h5pyf.close()
del h5pyf, ts_pa
gc.collect()
return() 
<create_dataframe>
train.isna().sum().sum()
p_products = 24
h5py_file_filename = "h5py_"+ p_data_model_version + ".hdf5"
h5py_file_bd_filename = "h5py_"+ p_data_model_version + "_bd.hdf5"
h5py_file_pa_filename = "h5py_"+ p_data_model_version + "_pa.hdf5"
if p_rebuild_datasets == True:
data = read_df(train_data_file,k_datasets_prefix,p_index)
 data_unclassified = read_df(predict_data_file,k_datasets_prefix,p_index)
 data, data_unclassified, data_categs = preproc_ds(data, data_unclassified,True,[p_index,"TARGET"])
 data = data.sort_values(by=[p_index], ascending = True)
 data_unclassified = data_unclassified.sort_values(by=[p_index], ascending = True)
 data_unclassified["SK_ID_CURR"].to_csv("./" + p_data_model_version + "unclassified_ids.csv", header=True, index=None, sep=',', mode='w')
 data = data.reindex(columns=data.drop(data_categs,axis=1 ).columns.tolist() +data_categs.tolist())
 data_unclassified = data_unclassified.reindex(columns=data_unclassified.drop(data_categs,axis=1 ).columns.tolist() +data_categs.tolist())
 X_bd, bd_categs = preproc_ds(read_df(bureau_data_file,k_datasets_prefix,[p_index,"SK_ID_BUREAU"]),None,False,[p_index,"SK_ID_BUREAU"])
 X_bd = X_bd.sort_values(by=[p_index, 'SK_ID_BUREAU','DAYS_CREDIT' ], ascending=[1,1,1])
 X_bd = X_bd.reindex(columns=X_bd.drop(bd_categs,axis=1 ).columns.tolist() +bd_categs.tolist())
 X_pa, pa_categs = preproc_ds(read_df(previous_application_data_file,k_datasets_prefix,[p_index,"SK_ID_PREV"]),None,False,[p_index,"SK_ID_PREV"])
 X_pa = X_pa.sort_values(by=[p_index, 'SK_ID_PREV' ], ascending=[1,1])
 X_pa = X_pa.reindex(columns=X_pa.drop(pa_categs,axis=1 ).columns.tolist() +pa_categs.tolist())
 if p_subsample_lines != 0:
data = data.iloc[:p_subsample_lines]
X_predict = data_unclassified
if len(data)> 100000: test_size = 10000
else: test_size = 0.10
X_train, X_test = train_test_split(data, test_size=test_size, random_state=42, shuffle = True)
 X_train = X_train.sort_values(by=[p_index], ascending = True)
 y_train = X_train[p_objective]
X_train = X_train.drop(p_objective, axis=1)
 X_test = X_test.sort_values(by=[p_index], ascending = True)
 y_test = X_test[p_objective]
X_test = X_test.drop(p_objective, axis=1)
 h5py_file = h5py.File(h5py_file_filename, "a", driver = "core")
 X_train_h5 = h5dump("pd_"+ p_data_model_version +"_X_train",X_train,h5py_file,(5,X_train.shape[1]),)
 X_test_h5 = h5dump("pd_"+ p_data_model_version +"_X_test",X_test,h5py_file,(5,X_test.shape[1]))
X_predict_h5 = h5dump("pd_"+ p_data_model_version +"_X_predict",X_predict,h5py_file,(5,X_predict.shape[1]))
y_train_h5 = h5dump("pd_"+ p_data_model_version +"_y_train",y_train,h5py_file,None)
 y_test_h5 = h5dump("pd_"+ p_data_model_version +"_y_test",y_test,h5py_file,None)
 X_categs = pickledump("pk_"+ p_data_model_version +"_X_categs",data_categs)
 h5py_file.flush()
h5py_file.close()
build_dump_tensor(X_test, p_index, p_products,"pd_"+ p_data_model_version +'_bd_test',X_bd, h5py_file_bd_filename)
 build_dump_tensor(X_predict, p_index, p_products,"pd_"+ p_data_model_version +'_bd_predict',X_bd, h5py_file_bd_filename)
 build_dump_tensor(X_train, p_index, p_products,"pd_"+ p_data_model_version +'_bd',X_bd, h5py_file_bd_filename)
 build_dump_tensor(X_test,p_index, p_products,"pd_"+ p_data_model_version +'_pa_test',X_pa,h5py_file_pa_filename)
 build_dump_tensor(X_predict, p_index, p_products,"pd_"+ p_data_model_version +'_pa_predict',X_pa,h5py_file_pa_filename)
 build_dump_tensor(X_train, p_index, p_products,"pd_"+ p_data_model_version +"_pa",X_pa,h5py_file_pa_filename)
 pickledump("pk_"+ p_data_model_version +"_bd_categs",bd_categs)
 pickledump("pk_"+ p_data_model_version +"_pa_categs",pa_categs)
<prepare_x_and_y>
test.isna().sum().sum()
h5py_file = h5py.File("h5py_"+ p_data_model_version + ".hdf5", "r")
 h5py_file_bd = h5py.File(h5py_file_bd_filename, "r")
 h5py_file_pa = h5py.File(h5py_file_pa_filename, "r")
 X_train = h5py_file["pd_"+ p_data_model_version +"_X_train"][:,1:]
X_train_labels = h5py_file["pd_"+ p_data_model_version +"_X_train"][:,:1]
X_test = h5py_file["pd_"+ p_data_model_version +"_X_test"][:,1:]
X_test_labels = h5py_file["pd_"+ p_data_model_version +"_X_test"][:,:1]
X_predict = h5py_file["pd_"+ p_data_model_version +"_X_predict"][:,1:]
y_train = h5py_file["pd_"+ p_data_model_version +"_y_train"]
y_test = h5py_file["pd_"+ p_data_model_version +"_y_test"]
X_categs = pickleload("pk_"+ p_data_model_version +"_X_categs")
 X_bd_tensor = h5py_file_bd["pd_"+ p_data_model_version +"_bd"]
X_test_bd_tensor = h5py_file_bd["pd_"+ p_data_model_version +"_bd_test"]
X_predict_bd_tensor = h5py_file_bd["pd_"+ p_data_model_version +"_bd_predict"]
X_bd_categs = pickleload("pk_"+ p_data_model_version +"_bd_categs")
 X_pa_tensor = h5py_file_pa["pd_"+ p_data_model_version +"_pa"]
X_test_pa_tensor = h5py_file_pa["pd_"+ p_data_model_version +"_pa_test"]
X_predict_pa_tensor = h5py_file_pa["pd_"+ p_data_model_version +"_pa_predict"]
X_pa_categs = pickleload("pk_"+ p_data_model_version +"_pa_categs" )<define_variables>
s = corr.unstack().drop_duplicates()
so = s.sort_values(kind="quicksort")
 so = so.drop_duplicates()
pa_in_vars = X_pa_tensor.shape[2]
bd_in_vars = X_bd_tensor.shape[2]
main_in_vars = X_train.shape[1]
total_samples = len(X_train )<prepare_x_and_y>
s["target"].sort_values() [:10]
class roc_callback(Callback):
def __init__(self,training_data,validation_data,early_stopping=False, patience=10, min_delta=0, checkpoint_file = None):
self.x_val = validation_data[0]
self.y_val = validation_data[1]
self.auc_val = []
self.early_stopping = early_stopping
self.patience = patience
self.min_delta = min_delta
self.wait = 0
self.stopped_epoch = 0
self.monitor_op = np.greater
self.checkpoint_file = checkpoint_file
def on_train_begin(self, logs={}):
self.wait = 0
self.stopped_epoch = 0
self.best = 0
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_val = self.model.predict(self.x_val)
 fpr, tpr, thresholds = metrics.roc_curve(self.y_val, y_pred_val, pos_label=1)
 auc_val=metrics.auc(fpr, tpr)
 self.auc_val.append(auc_val)
 print('\rSKLearn roc-auc_val: %s' % str(round(auc_val,4)) ,end=100*' '+'
')
 if self.early_stopping == True:
current = auc_val
if self.monitor_op(current-self.min_delta,self.best):
self.best = current
self.wait = 0
if not self.checkpoint_file is None:
self.model.save_weights(self.checkpoint_file)
 print("Saved checkpoint: ", self.checkpoint_file)
 else:
self.wait += 1
if self.wait >= self.patience:
self.stopped_epoch = epoch
self.model.stop_training = True
print("Early Stopping Best: ",str(round(self.best,5)) , "Epochs w/o improvement: ", self.wait, "Must score ", str(round(self.best+self.min_delta,5)) , "before ",self.patience-self.wait," Epochs to extend",'
')
 return
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return 
<compute_test_metric>
s["target"].sort_values(ascending=False)[:10]
def auc_m(y_true, y_pred):
ptas = tf.stack([binary_PTA(y_true,y_pred,k)for k in np.linspace(0, 1, 1000)],axis=0)
 pfas = tf.stack([binary_PFA(y_true,y_pred,k)for k in np.linspace(0, 1, 1000)],axis=0)
 pfas = tf.concat([tf.ones(( 1,)) ,pfas],axis=0)
 binSizes = -(pfas[1:]-pfas[:-1])
 s = ptas*binSizes
return K.sum(s, axis=0)
 def binary_PFA(y_true, y_pred, threshold=K.variable(value=0.5)) :
y_pred = K.cast(y_pred >= threshold, 'float32')
 N = K.sum(1 - y_true)
 FP = K.sum(y_pred - y_pred * y_true)
 return FP/N
def binary_PTA(y_true, y_pred, threshold=K.variable(value=0.5)) :
y_pred = K.cast(y_pred >= threshold, 'float32')
 P = K.sum(y_true)
 TP = K.sum(y_pred * y_true)
 return TP/P 
<categorify>
print("Top most highly positive correlated features:")
 print(so[(so<1)&(so>0.2)].sort_values(ascending=False))
print()
print("Top most highly megative correlated features:")
 print(so[(so < - 0.2)] )
def crop(dimension, start, end=None):
if not end is None:
def func(x):
if dimension == 0:
return x[start: end]
if dimension == 1:
return x[:, start: end]
if dimension == 2:
return x[:, :, start: end]
if dimension == 3:
return x[:, :, :, start: end]
if dimension == 4:
return x[:, :, :, :, start: end]
else:
def func(x):
if dimension == 0:
return x[start]
if dimension == 1:
return x[:, start]
if dimension == 2:
return x[:, :, start]
if dimension == 3:
return x[:, :, :, start]
if dimension == 4:
return x[:, :, :, :, start]
return Lambda(func)
 def gen_conv_branch(name,
products,
pa_filters,
pa_filters1,
pa_filters2,
lc,
lc2,
features_pa,
regu,
pa_categs,
inputs,
NN_data,
NN_val_data,
data_pa,
data_test_pa):
b_pa_input = Input(shape=(products,features_pa),name=name+"_PA_main", dtype='float32')
 inputs.append(b_pa_input)
 NN_data.append(data_pa)
 NN_val_data.append(data_test_pa)
 noncat_features = int(b_pa_input.shape[2]-len(pa_categs))
x_noncat_pa = crop(2,0,noncat_features )(b_pa_input)
 x_cat_pa = [x_noncat_pa]
i=0
if len(pa_categs)> 0:
for categ in pa_categs:
cat_layer_pa = crop(2,noncat_features+i )(b_pa_input)
 cardinality = len(np.unique(data_pa[:,:,-len(pa_categs)+i]))
target_cardinality = int(cardinality ** 0.25)
 cat_layer_pa = Embedding(cardinality, target_cardinality )(cat_layer_pa)
 cat_layer_pa = Reshape(( products,target_cardinality,))(cat_layer_pa)
 x_cat_pa.append(cat_layer_pa)
 i+=1
b_pa_x= concatenate(x_cat_pa)
 else:
b_pa_x= b_pa_input
b_pa_x = Conv1D(filters=pa_filters,
kernel_size=(1),
input_shape=(products,features_pa),
activation="relu",
kernel_regularizer=regularizers.l2(regu),
strides=1 )(b_pa_x)
 b_pa_x = Conv1D(filters=pa_filters1,
kernel_size=(1),
input_shape=(products,pa_filters),
activation="relu",
kernel_regularizer=regularizers.l2(regu),
strides=1 )(b_pa_x)
 b_pa_x = Conv1D(filters=pa_filters2,
kernel_size=(1),
input_shape=(products,pa_filters1),
activation="relu",
kernel_regularizer=regularizers.l2(regu),
strides=1 )(b_pa_x)
 b_pa_x = Flatten()(b_pa_x)
 b_pa_x = Dropout(0.3 )(b_pa_x)
 b_pa_x = Dense(lc, activation="relu",kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(regu))(b_pa_x)
 b_pa_x = Dropout(0.3 )(b_pa_x)
 b_pa_x = Dense(lc2, activation="relu",kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(regu))(b_pa_x)
 return(b_pa_x, inputs, NN_data, NN_val_data)
 def plot_roc(model,test_data,test_labels):
prediction = model.predict(test_data)
 fpr, tpr, thresholds = metrics.roc_curve(test_labels, prediction, pos_label=1)
 auc=metrics.auc(fpr, tpr)
 plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
lw=lw, label='ROC curve '+ str(round(auc,6)))
 plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
 plt.xlim([0.0, 1.0])
 plt.ylim([0.0, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('Receiver operating characteristic example')
 plt.legend(loc="lower right")
 fig = plt.gcf()
plt.show()
fig.savefig('./work/'+p_data_model_version+'_ROC.png')
 return(auc)
<init_hyperparams>
fs_params = dict()
fs_params["descriptives"] = False
fs_params["standardization"] = False
fs_params["percentiles"] = False
fs_params["squared"] = False
fs_params["frequency"] = False
gc.collect() ;
turn = 0
for df in [test, train]:
if turn == 0:
print("Train set")
 turn = 1
else:
print("Test set")
 if(fs_params["descriptives"] == True):
print('\t*descriptive statistics Feature Engineering:')
 df['sum'] = df[numerical_features].sum(axis=1)
 df['min'] = df[numerical_features].min(axis=1)
 df['max'] = df[numerical_features].max(axis=1)
 df['mean'] = df[numerical_features].mean(axis=1)
 df['std'] = df[numerical_features].std(axis=1)
 df['skew'] = df[numerical_features].skew(axis=1)
 df['kurt'] = df[numerical_features].kurtosis(axis=1)
 df['med'] = df[numerical_features].median(axis=1)
 print('\t*descriptive statistics Feature Engineering done!')
 if(fs_params["standardization"] == True):
print('\t*Standardizing the data:')
 scaler = StandardScaler()
df[numerical_features] = scaler.fit_transform(df[numerical_features])
 print('\t*Data Standardized!')
 if(fs_params["percentiles"] == True):
print('\t*percentiles Feature Engineering:')
 perc_list = [1,2,5,10,25,50,60,75,80,85,95,99]
for i in perc_list:
df['perc_'+str(i)] = df[numerical_features].apply(lambda x: np.percentile(x, i), axis=1)
 print('\t*Done percentiles Feature Engineering!')
 if(fs_params["squared"] == True):
print('\t*Loading Squared data:')
 for i in numerical_features:
df['var_sq_'+str(i)] = np.square(df[str(i)])
 print('\t*Done squaring!')
 if(fs_params["frequency"] == True):
print('\t*Loading frequency:')
 for var in numerical_features:
hist, bin_edges = np.histogram(df[var], bins=1000, density=True)
 df['hist_'+var] = [ hist[np.searchsorted(bin_edges,ele)-1] for ele in df[var]]
print('\t*Done Loading frequency!')
 gc.collect() ;
class_weight = {0: 1.,
1: 16.}
regu=10**-7.56
regm=10**-4.2
learn_rate = 10**-2.38
epochs=50
batch_size = 2**12
main_input = Input(shape=(main_in_vars,),name='main_input', dtype='float32')
 model_inputs = [main_input]
NNtrain_data = [X_train]
NNtest_data = [X_test]
noncat_features = int(main_input.shape[1]-len(X_categs))
x_noncat = crop(1,0,noncat_features )(main_input)
 x_cat = [x_noncat]
i=0
for categ in X_categs:
cat_layer = crop(1,noncat_features+i )(main_input)
 cardinality = len(np.unique(X_train[:,-len(X_categs)+i]))
target_cardinality = int(cardinality ** 0.25)
 cat_layer = Embedding(cardinality, target_cardinality )(cat_layer)
 cat_layer = Reshape(( target_cardinality,))(cat_layer)
 x_cat.append(cat_layer)
 i+=1
x= concatenate(x_cat)
 x = Dense(90, activation="relu", kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(regm))(x)
 x = Dropout(0.5 )(x)
 bd_x, model_inputs, NNtrain_data, NNtest_data = gen_conv_branch("bd_input",p_products,
90, 60, 10,
60, 20, bd_in_vars,regu,
X_bd_categs,
model_inputs, NNtrain_data, NNtest_data,
X_bd_tensor, X_test_bd_tensor)
 x = concatenate([bd_x, x])
 pcb_x, model_inputs, NNtrain_data, NNtest_data = gen_conv_branch("pa_input",p_products,
90, 60, 10,
60,20,pa_in_vars,regu,
X_pa_categs,
model_inputs, NNtrain_data, NNtest_data,
X_pa_tensor, X_test_pa_tensor)
 x = concatenate([pcb_x, x])
 x = Dense(150, activation="relu",kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(regm))(x)
 x = Dropout(0.5 )(x)
 x = Dense(30, activation="relu",kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(regm))(x)
 main_output = Dense(1, activation='sigmoid', name='main_output' )(x)
 model = Model(inputs=model_inputs, outputs=main_output)
 roc_cbk = roc_callback(training_data=None,validation_data=(NNtest_data, y_test),
early_stopping= True, patience=8, min_delta=0.0001,
checkpoint_file = './'+p_data_model_version+'_weights.hdf5')
 callbacks_list = [roc_cbk]
model.compile(optimizer=optimizers.Adam(lr=learn_rate), loss='binary_crossentropy', metrics=[auc_m])
<train_model>
y = train['target']
X = train.drop(['target', "id"], axis=1)
 xtest = test.drop("id", axis="columns")
 std_scaler_flag = False
robust_scaler_flag = False
min_max_scalar_flag = False
if std_scaler_flag == True:
print("StandardScaler!")
 sc0 = StandardScaler()
sc0.fit(X)
 X = sc0.transform(X)
 X = pd.DataFrame(X, columns=train.drop(['target', "id"], axis=1 ).columns)
 xtest = sc0.transform(xtest)
 xtest = pd.DataFrame(xtest, columns = test.drop("id", axis="columns" ).columns)
 print(X.head(6))
else:
print("No scaling at all!" )
fitlog = model.fit(NNtrain_data, y_train, validation_data=(NNtest_data, y_test),
epochs=epochs, batch_size=batch_size,
class_weight=class_weight, callbacks=callbacks_list, shuffle="batch" )<predict_on_test>
clf_stats_df = pd.DataFrame(columns=["clf_name", "F1-score", "auc-score"] )
NNpredict_data = [X_predict, X_predict_bd_tensor, X_predict_pa_tensor ]
model.load_weights('./'+p_data_model_version+'_weights.hdf5')
 prediction = model.predict(NNpredict_data )<load_from_csv>
def xgboost_all_purpose(X, y, type_of_training, name, num_of_folds=3, params=None, in_folds_sampling = False, max_early_stopping = 100):
global clf_stats_df
if params is None:
params = dict()
params["learning_rate"] = 0.1
params["n_estimators"] = 500
params["max_depth"] = 3
params["min_child_weight"] = 1
params["gamma"] = 0
params["subsample"] = 1
params["colsample_bytree"] = 1
params["colsample_bylevel"] = 1
params["reg_alpha"] = 0
params["reg_lambda"] = 1
params["scale_pos_weight"] = np.round(y.value_counts() [0] / y.value_counts() [1],3)
 params["max_delta_step"] = 1
print("params", params)
 print("max_early_stopping:", max_early_stopping)
 if type_of_training == "baseline":
print("baseline")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test = np.zeros(len(test))
num_fold = 0
folds = StratifiedKFold(n_splits=num_of_folds, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_stra_xgb = xgb.XGBClassifier(learning_rate=params["learning_rate"],
n_estimators=params["n_estimators"],
max_depth=params["max_depth"],
min_child_weight=params["min_child_weight"],
gamma=params["gamma"],
subsample=params["subsample"],
colsample_bytree=params["colsample_bytree"],
colsample_bylevel=params["colsample_bylevel"],
objective= 'binary:logistic',
nthread=-1,
scale_pos_weight=params["scale_pos_weight"],
reg_alpha = params["reg_alpha"],
reg_lambda = params["reg_lambda"],
max_delta_step = params["max_delta_step"],
seed=42)
 clf_stra_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=max_early_stopping, eval_metric='auc', verbose=100)
 predictions = clf_stra_xgb.predict(xvalid)
 predictions_probas = clf_stra_xgb.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test += clf_stra_xgb.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(12, 38)})
 xgb.plot_importance(clf_stra_xgb, title='Feature importance', xlabel='F score', ylabel='Features')
 clf_stats_df = clf_stats_df.append({"clf_name": name,
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()
gc.collect() ;
return clf_stra_xgb, predictions_test
elif type_of_training == "train_test_equal":
print("train_test_equal_flag")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test = np.zeros(len(test))
num_fold = 0
print("resample xtrain set to be equal to the size of the test set")
 temp_df = pd.DataFrame(xtrain, columns=xtrain.columns)
 temp_df["target"] = ytrain
temp_df = temp_df.sample(frac = len(test)/ len(xtrain), replace = True)
 ytrain = temp_df['target']
xtrain = temp_df.drop(['target'], axis=1)
 del temp_df;
print(xtrain.shape, test.shape)
 folds = StratifiedKFold(n_splits=num_of_folds, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_stra_xgb = xgb.XGBClassifier(learning_rate=params["learning_rate"],
n_estimators=params["n_estimators"],
max_depth=params["max_depth"],
min_child_weight=params["min_child_weight"],
gamma=params["gamma"],
subsample=params["subsample"],
colsample_bytree=params["colsample_bytree"],
colsample_bylevel=params["colsample_bylevel"],
objective= 'binary:logistic',
nthread=-1,
scale_pos_weight=params["scale_pos_weight"],
reg_alpha = params["reg_alpha"],
reg_lambda = params["reg_lambda"],
max_delta_step = params["max_delta_step"],
seed=42)
 clf_stra_xgb.fit(xtrain_stra, ytrain_stra, eval_set=[(xtrain_stra, ytrain_stra),(xvalid_stra, yvalid_stra)],
early_stopping_rounds=max_early_stopping, eval_metric='auc', verbose=100)
 predictions = clf_stra_xgb.predict(xvalid)
 predictions_probas = clf_stra_xgb.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test += clf_stra_xgb.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(12, 38)})
 xgb.plot_importance(clf_stra_xgb, title='Feature importance', xlabel='F score', ylabel='Features')
 clf_stats_df = clf_stats_df.append({"clf_name": name,
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()
gc.collect() ;
return clf_stra_xgb, predictions_test
else:
print("Please specify for the argument 'type_of_training'one of the following parameters:(baseline, oversampling, smote, undersampling, augmentation_by_fraction)" )
submission = pd.read_csv("./" + p_data_model_version + "unclassified_ids.csv", sep=',')
 submission["TARGET"] = prediction<save_to_csv>
type_of_training = "baseline"
num_of_folds = 4
in_folds_sampling = False
clf_xgb, predictions_test_xgb = xgboost_all_purpose(X,y, num_of_folds = num_of_folds,
type_of_training =type_of_training,
in_folds_sampling = in_folds_sampling,
max_early_stopping = 200,
name="clf_xgb")
 del clf_xgb
submission.to_csv('submission.csv', index=False )<save_to_csv>
gc.collect()
feature_selection_flag = True
if feature_selection_flag == True:
gc.collect() ;
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, test_size=0.3, random_state=42)
 fs_model = xgb.XGBClassifier(nthread = -1,
objective = 'binary:logistic',
eval_metric = 'auc',
silent=1,
tree_method='auto',
max_depth = 3,
scale_pos_weight = np.round(y.value_counts() [0] / y.value_counts() [1],3))
fs_model.fit(xtrain, ytrain)
 perm = PermutationImportance(fs_model, random_state=42 ).fit(xvalid, yvalid )
submission.to_csv('submission.csv', index=False )<load_from_csv>
eli5.show_weights(perm, feature_names = xvalid.columns.tolist() , top=100 )
ghosts_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 ghosts_df.head() 
<set_options>
if feature_selection_flag == True:
max_selected_features = 50
sel = SelectFromModel(perm, max_features = max_selected_features, prefit=True)
 feature_idx = sel.get_support()
selected_feature_names = X.columns[feature_idx]
X_fs_50 = X[selected_feature_names]
print(X_fs_50.shape)
 max_selected_features = 25
sel = SelectFromModel(perm, max_features = max_selected_features, prefit=True)
 feature_idx = sel.get_support()
selected_feature_names = X.columns[feature_idx]
X_fs_25 = X[selected_feature_names]
print(X_fs_25.shape)
 max_selected_features = 15
sel = SelectFromModel(perm, max_features = max_selected_features, prefit=True)
 feature_idx = sel.get_support()
selected_feature_names = X.columns[feature_idx]
X_fs_15 = X[selected_feature_names]
print(X_fs_15.shape )
sns.set_style('whitegrid')
 sns.pairplot(ghosts_df, hue="type" )<categorify>
fs_clf_xgb, predictions_test_fs_50_xgb = xgboost_all_purpose(X_fs_50,
y,
type_of_training =type_of_training,
num_of_folds = num_of_folds,
max_early_stopping= 100,
name="fs_clf_50_xgb" )
ghosts_df = ghosts_df.drop("id",axis=1)
 def remap_as_int(dataframe, indice, value, newValue):
print("value={} reassigned to id={}".format(value, newValue))
dataframe.loc[dataframe[indice] == value, indice] = newValue
def enumerate_list_unique(dataframe, indice):
return list(enumerate(np.unique(dataframe[indice])))
 monsters_list = enumerate_list_unique(ghosts_df, 'type')
 monsters_list = [(0, 'Ghost'),(1, 'Ghoul'),(2, 'Goblin')]
print("All known types of monsters = {}".format(np.unique(ghosts_df['type'])))
 for index, monster in monsters_list:
remap_as_int(ghosts_df, 'type', monster, index)
 colors_list = [(0, 'black'),(0.2, 'blood'),(0.4, 'blue'),(0.6, 'clear'),(0.8, 'green'),(1, 'white')]
print("All known colors of monsters = {}".format(np.unique(ghosts_df['color'])))
 for index, color in colors_list:
remap_as_int(ghosts_df, 'color', color, index )<prepare_x_and_y>
fs_clf_xgb, predictions_test_fs_25_xgb = xgboost_all_purpose(X_fs_25,
y,
type_of_training =type_of_training,
num_of_folds = num_of_folds,
max_early_stopping= 100,
name="fs_clf_25_xgb" )
remove_indexes = ["color", "hair_length", "has_soul", "rotting_flesh"]
variant = 'all'
test = False
show_test_result = False
alphas = [0.00001, 0.01, 0.1, 0.5, 1, 10]
if test:
X_train, X_test, y_train, y_test = train_test_split(ghosts_df.drop(remove_indexes + ["type"],axis=1), ghosts_df["type"], test_size=0.2, random_state=0)
 variant = 'all'
else:
X_train = ghosts_df.drop(remove_indexes + ["type"],axis=1)
 y_train = ghosts_df["type"]
create_features(test_df)
 def print_results(grid_search):
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
if show_test_result:
y_pred = grid_search.predict(X_test)
 print('Test-accuracy={}'.format(accuracy_score(y_test, y_pred)))
 for index, color in colors_list:
remap_as_int(test_df, 'color', color, index)
 test_df['color'] = test_df['color'].astype(float)
 test_df = test_df.drop(remove_indexes, axis=1)
 X_submission = test_df.drop("id",axis=1 ).copy()<train_on_grid>
fs_clf_xgb, predictions_test_fs_15_xgb = xgboost_all_purpose(X_fs_15,
y,
type_of_training =type_of_training,
num_of_folds = num_of_folds,
max_early_stopping= 100,
name="fs_clf_15_xgb" )
if variant is 'lr' or variant is 'all':
logreg = LogisticRegression()
parameter_grid = {'C': alphas,
'solver' : ['newton-cg', 'lbfgs', 'sag'],
'multi_class' : ['multinomial', 'ovr']}
grid_search_logit = GridSearchCV(logreg, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_logit.fit(X_train, y_train)
 print_results(grid_search_logit )<train_on_grid>
as_it_is = True
smote_flag = False
adasyn_flag = False
oversampling_flag = False
augment_flag = False
def prepare_for_tuning(X, y, type_of_training):
if type_of_training == "baseline":
print("no resampling at all!")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 elif type_of_training == "train_test_equal":
print("train_test_equal")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 print("resample xtrain set to be equal to the size of the test set")
 temp_df = pd.DataFrame(xtrain, columns=xtrain.columns)
 temp_df["target"] = ytrain
temp_df = temp_df.sample(frac = len(test)/ len(xtrain), replace = True)
 ytrain = temp_df['target']
xtrain = temp_df.drop(['target'], axis=1)
 del temp_df;
print(xtrain.shape, test.shape)
 elif type_of_training == "oversampling":
print("oversampling")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 ros = RandomOverSampler(random_state=42)
 X_resampled, y_resampled = ros.fit_resample(xtrain, ytrain)
 print(sorted(Counter(y_resampled ).items()))
xtrain=pd.DataFrame(X_resampled, columns = X.columns)
 ytrain = y_resampled
del X_resampled
del y_resampled
elif type_of_training == "augmentation_by_fraction":
print("augmentation")
 xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.3)
 print("ytrain target values count before augmentation:
", sorted(Counter(ytrain ).items()))
X_y = pd.DataFrame(xtrain, columns=X.columns)
 X_y["target"] = ytrain
X_y = X_y.sample(frac=3, replace=True)
 X_y.target.value_counts()
ytrain = X_y['target']
print("ytrain target values count after oversampling:
",sorted(Counter(ytrain ).items()))
xtrain = X_y.drop(['target'], axis=1)
 del X_y
rus = RandomUnderSampler(sampling_strategy=0.33, random_state=42)
 X_resampled, y_resampled = rus.fit_resample(xtrain, ytrain)
 print("ytrain target values count after Augmentation:
",sorted(Counter(y_resampled ).items()))
xtrain=pd.DataFrame(X_resampled, columns = X.columns)
 ytrain = y_resampled
del X_resampled
del y_resampled
gc.collect() ;
return xtrain, ytrain
if variant is 'svc' or variant is 'all':
svc = SVC()
parameter_grid = {'C': alphas,
'kernel' : ['linear', 'rbf', 'sigmoid'],
'gamma' : alphas}
grid_search_svc = GridSearchCV(svc, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_svc.fit(X_train, y_train)
 print_results(grid_search_svc )<train_on_grid>
def bayesian_tuning(xtrain, ytrain):
bayes_cv_tuner = BayesSearchCV(
 estimator = xgb.XGBClassifier(
 nthread = -1,
objective = 'binary:logistic',
eval_metric = 'auc',
silent=1,
tree_method='auto' 
),
search_spaces = {
'learning_rate':(0.01, 1.0, 'log-uniform'),
'min_child_weight':(0, 10),
'n_estimators':(50, 100),
'max_depth':(0, 12),
'gamma':(1e-2, 10, 'log-uniform'),
'subsample':(0.01, 1.0, 'uniform'),
'colsample_bytree':(0.01, 1.0, 'uniform'),
'colsample_bylevel':(0.01, 1.0, 'uniform'),
'scale_pos_weight':(0.01, 1.0, 'uniform'),
'reg_lambda':(1e-1, 10, 'log-uniform'),
'reg_alpha':(1e-2, 1.0, 'log-uniform'),
'max_delta_step':(0, 10, 'uniform'),
'scale_pos_weight':(1e-2, 1, 'uniform')
 },
scoring = 'roc_auc',
cv = StratifiedKFold(
 n_splits=3,
shuffle=True,
random_state=42 
),
n_jobs = 1,
n_iter = 7,
verbose = 0,
refit = True,
random_state = 42 
)
 def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best ROC-AUC: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
result = bayes_cv_tuner.fit(xtrain, ytrain, callback = status_print)
 return result
xtrain, ytrain = prepare_for_tuning(X, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )
if variant is 'rf' or variant is 'all':
random_forest = RandomForestClassifier(random_state=1)
 parameter_grid = {'min_samples_leaf': [1, 5, 10],
'n_estimators': [10, 30, 50, 100],
'min_samples_split': [2,3]}
grid_search_rf = GridSearchCV(random_forest, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_rf.fit(X_train, y_train)
 print_results(grid_search_rf )<train_on_grid>
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_xgb, predictions_test_tuned_xgb = xgboost_all_purpose(X,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_xgb" )
if variant is 'knn' or variant is 'all':
knn = KNeighborsClassifier()
parameter_grid = {'n_neighbors': [3, 5, 10],
'leaf_size': [5, 10, 30, 50],
'metric': ['minkowski', 'euclidean']}
grid_search_knn = GridSearchCV(knn, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_knn.fit(X_train, y_train)
 print_results(grid_search_knn )<train_on_grid>
xtrain, ytrain = prepare_for_tuning(X_fs_50, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )
if variant is 'nn' or variant is 'all':
neural_net = MLPClassifier(solver='lbfgs', random_state=1)
 parameter_grid = {'hidden_layer_sizes': [(15,1),(20,1),(15,3),(20,3),(15,6),(20,6)],
'alpha': alphas}
grid_search_nn = GridSearchCV(neural_net, param_grid=parameter_grid, cv=StratifiedKFold(4))
grid_search_nn.fit(X_train, y_train)
 print_results(grid_search_nn )<predict_on_test>
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_fs_50_xgb, predictions_test_tuned_fs_50_xgb = xgboost_all_purpose(X_fs_50,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_fs_50_xgb" )
Y_pred = grid_search_svc.predict(X_submission)
 Y_pred = Y_pred.astype(object)
 Y_pred[Y_pred == monsters_list[0][0]] = monsters_list[0][1]
Y_pred[Y_pred == monsters_list[1][0]] = monsters_list[1][1]
Y_pred[Y_pred == monsters_list[2][0]] = monsters_list[2][1]<save_to_csv>
xtrain, ytrain = prepare_for_tuning(X_fs_25, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )
submission = pd.DataFrame({
"id": test_df["id"],
"type": Y_pred
})
 submission.to_csv('new.csv', index=False )<load_from_csv>
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_fs_25_xgb, predictions_test_tuned_fs_25_xgb = xgboost_all_purpose(X_fs_25,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_fs_25_xgb" )
data_frame_train = pd.read_csv('.. /input/train.csv')
 data_frame_test = pd.read_csv(".. /input/test.csv" )<create_dataframe>
xtrain, ytrain = prepare_for_tuning(X_fs_15, y, type_of_training=type_of_training)
 result = bayesian_tuning(xtrain, ytrain )
print(data_frame_train.dtypes )<groupby>
gc.collect()
result.best_params_['n_estimators'] = 3000
num_of_folds = 4
tuned_clf_fs_15_xgb, predictions_test_tuned_fs_15_xgb = xgboost_all_purpose(X_fs_15,
y,
type_of_training = type_of_training,
num_of_folds=num_of_folds,
params = result.best_params_,
max_early_stopping = 200,
in_folds_sampling = False,
name="tuned_clf_fs_15_xgb" )
class_count = data_frame_train.groupby('type' ).size()
print(class_count )<drop_column>
gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_xgb
submission.to_csv('clf_xgb.csv', index=False)
 if feature_selection_flag == True:
gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_fs_50_xgb
submission.to_csv('fs_50_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_fs_25_xgb
submission.to_csv('fs_25_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_fs_15_xgb
submission.to_csv('fs_15_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_xgb
submission.to_csv('tuned_clf_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_fs_50_xgb
submission.to_csv('tuned_clf_fs_50_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_fs_25_xgb
submission.to_csv('tuned_clf_fs_25_xgb.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_tuned_fs_15_xgb
submission.to_csv('tuned_clf_fs_15_xgb.csv', index=False )
df = data_frame_train["type"]
indexes_test = data_frame_test["id"]
data_frame_train = data_frame_train.drop(["type","color","id"],axis=1)
 data_frame_test = data_frame_test.drop(["color","id"],axis=1 )<categorify>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.1)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test = np.zeros(len(test))
num_fold = 0
num_of_folds = 10
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_logit = LogisticRegression(random_state=42,
C=0.1,
solver='liblinear',
class_weight ="balanced",
penalty = 'l1',
multi_class='ovr' ).fit(xtrain_stra, ytrain_stra)
 predictions = clf_logit.predict(xvalid)
 predictions_probas = clf_logit.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test += clf_logit.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 clf_stats_df = clf_stats_df.append({"clf_name": "clf_logit",
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()
data_frame_train = pd.get_dummies(data_frame_train)
 data_frame_test = pd.get_dummies(data_frame_test )<split>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.1)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test_rfe = np.zeros(len(test))
num_fold = 0
num_of_folds = 10
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
clf_logit = LogisticRegression(random_state=42,
C=0.1,
solver='liblinear',
class_weight ="balanced",
penalty = 'l1',
multi_class='ovr')
 selector = RFE(clf_logit, 21)
 selector.fit(xtrain_stra, ytrain_stra)
 clf_logit_rfe = selector
predictions = clf_logit_rfe.predict(xvalid)
 predictions_probas = clf_logit_rfe.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test_rfe += clf_logit_rfe.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 clf_stats_df = clf_stats_df.append({"clf_name": "clf_logit",
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()
x_train, x_test, y_train, y_test = train_test_split(data_frame_train, df, test_size=0.3, random_state=0 )<train_model>
xtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify = y, random_state=42, test_size=0.1)
 start_time = time.time()
predictions_probas_list = np.zeros([len(yvalid), 2])
 predictions_test_voting = np.zeros(len(test))
num_fold = 0
num_of_folds = 10
folds = StratifiedKFold(n_splits=10, shuffle=False, random_state = 42)
 for train_index, valid_index in folds.split(xtrain, ytrain):
xtrain_stra, xvalid_stra = xtrain.iloc[train_index,:], xtrain.iloc[valid_index,:]
ytrain_stra, yvalid_stra = ytrain.iloc[train_index], ytrain.iloc[valid_index]
print()
print("Stratified Fold:", num_fold)
 num_fold = num_fold + 1
print()
log_clf = LogisticRegression(C = 0.1, class_weight= 'balanced', penalty= 'l1', solver= 'liblinear',random_state=42)
 rand_clf = RandomForestClassifier(max_depth=2,random_state=42)
 svm_clf = SVC(gamma='auto',probability=True, random_state=42)
 extra_clf = ExtraTreesClassifier(max_depth=2,random_state=42)
 voting_clf = VotingClassifier(
 estimators = [('lr',log_clf),('rf',rand_clf),('ex',extra_clf),('sv',svm_clf)],
voting='soft')
 voting_clf.fit(xtrain_stra, ytrain_stra)
 predictions = voting_clf.predict(xvalid)
 predictions_probas = voting_clf.predict_proba(xvalid)
 predictions_probas_list += predictions_probas/num_of_folds
predictions_test_voting += voting_clf.predict_proba(test[xtrain.columns])[:,1]/num_of_folds
predictions = np.argmax(predictions_probas, axis=1)
 print()
print(classification_report(yvalid, predictions))
print()
print("CV f1_score", f1_score(yvalid, predictions, average = "macro"))
print()
print("CV roc_auc_score", roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro"))
print()
print("elapsed time in seconds: ", time.time() - start_time)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_confusion_matrix(yvalid, predictions, normalize=True)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_roc(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_ks_statistic(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_precision_recall(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_cumulative_gain(yvalid, predictions_probas)
 sns.set(rc={'figure.figsize':(8,8)})
 skplt.metrics.plot_lift_curve(yvalid, predictions_probas)
 clf_stats_df = clf_stats_df.append({"clf_name": "clf_voting",
"F1-score":f1_score(yvalid, predictions, average = "macro"),
"auc-score": roc_auc_score(yvalid, predictions_probas_list[:,1], average = "macro")}, ignore_index=True)
 print()
<save_to_csv><EOS> 
gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test
submission.to_csv('clf_logit.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_rfe
submission.to_csv('clf_logit_rfe.csv', index=False)
 gc.collect() ;
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions_test_voting
submission.to_csv('clf_logit_voting.csv', index=False )