Kaggle competition: Don't Overfit! II
trees = train[(train['Horizontal_Distance_To_Fire_Points'] > outlier_function(train, 'Horizontal_Distance_To_Fire_Points')[0])& 
(train['Horizontal_Distance_To_Fire_Points'] < outlier_function(train, 'Horizontal_Distance_To_Fire_Points')[1])]
trees.shape<count_unique_values>
submission = pd.read_csv('.. /input/sample_submission.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 train_df = pd.read_csv(".. /input/train.csv" )
size=10
Uni = []
for i in range(size+1,len(train.columns)-1):
Uni.append(pd.unique(train[train.columns[i]].values)) 
<prepare_output>
%matplotlib inline
x = train.iloc[:,15:55]
y = train.iloc[:,11:15]
y = pd.DataFrame(y)
 x = pd.DataFrame(x)
 s2 = pd.Series(x.columns[np.where(x!=0)[1]])
 s3 = pd.Series(y.columns[np.where(y!=0)[1]])
 train['soil_type'] = s2
train['Wilderness_Area'] = s3
train.head()<drop_column>
print('
print('
print('% target = 1: {}%'.format(round(float(len(train_df[train_df['target'] == 1])) / len(train_df)* 100)))
 tax=round(float(len(train_df[train_df['target'] == 1])) / len(train_df)* 100 )
df_viz = train.iloc[:, 0:15]
df_viz = df_viz.drop(['Wilderness_Area1', 'Wilderness_Area2', 'Wilderness_Area3',
'Wilderness_Area4'], axis = 1)
 df_viz.head()<prepare_x_and_y>
for df in [test_df, train_df]:
d=pd.Series(d)
 d=(d + 1 ).tolist()
for x in d:
df[df.columns[x]]=(df[df.columns[x]]-df[df.columns[x]].min())/(df[df.columns[x]].max() -df[df.columns[x]].min())
 df[df.columns[x]]=np.log(( 1+df[df.columns[x]]))
train=trees<feature_engineering>
sample= ['target','16', '33', '80', '91', '217', '295']
def add_feature(data):
data['Ele_minus_VDtHyd'] = data.Elevation-data.Vertical_Distance_To_Hydrology
data['Ele_plus_VDtHyd'] = data.Elevation+data.Vertical_Distance_To_Hydrology
data['Distanse_to_Hydrolody'] =(data['Horizontal_Distance_To_Hydrology']**2+data['Vertical_Distance_To_Hydrology']**2)**0.5
data['Hydro_plus_Fire'] = data['Horizontal_Distance_To_Hydrology']+data['Horizontal_Distance_To_Fire_Points']
data['Hydro_minus_Fire'] = data['Horizontal_Distance_To_Hydrology']-data['Horizontal_Distance_To_Fire_Points']
data['Hydro_plus_Road'] = data['Horizontal_Distance_To_Hydrology']+data['Horizontal_Distance_To_Roadways']
data['Hydro_minus_Road'] = data['Horizontal_Distance_To_Hydrology']-data['Horizontal_Distance_To_Roadways']
data['Fire_plus_Road'] = data['Horizontal_Distance_To_Fire_Points']+data['Horizontal_Distance_To_Roadways']
data['Fire_minus_Road'] = data['Horizontal_Distance_To_Fire_Points']-data['Horizontal_Distance_To_Roadways']
return data<feature_engineering>
train = add_feature(train)
 test = add_feature(test )<prepare_x_and_y>
result=pd.DataFrame()
X_train = train.drop(['Id','Cover_Type'], axis = 1)
 y_train = train.Cover_Type
X_test = test.drop(['Id'], axis = 1 )<train_on_grid>
y = train_df['target']
X = train_df.drop(['target','id'], axis=1)
 X_ol=X[d]
print('Train set:', X.shape, y.shape, X_ol.shape )
%%time
lr_pipe = Pipeline(
 steps = [ 
('scaler', MinMaxScaler()), 
('classifier', LogisticRegression(solver='lbfgs', n_jobs=-1))
] 
)
 lr_param_grid = {
'classifier__C': [1, 10, 100,1000],
}
np.random.seed(1)
 grid_search = GridSearchCV(lr_pipe, lr_param_grid, cv=5, refit='True')
 grid_search.fit(X_train, y_train)
 print(grid_search.best_score_)
 print(grid_search.best_params_)
<train_on_grid>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=6)
 print('Train set:', X_train.shape, y_train.shape)
 print('Test set:', X_test.shape, y_test.shape)
 X_train_ol=X_train[d]
X_test_ol=X_test[d]
print('Train set select:', X_train_ol.shape, y_train.shape)
 print('Test set select:', X_test_ol.shape, y_test.shape )
%%time
rf_pipe = Pipeline(
 steps = [ 
('classifier', RandomForestClassifier(n_estimators=500))
] 
)
 param_grid = {
'classifier__min_samples_leaf': [1,4,7],
'classifier__max_depth': [34,38,32],
}
np.random.seed(1)
 rf_grid_search = GridSearchCV(rf_pipe, param_grid, cv=5, refit='True', n_jobs=-1)
 rf_grid_search.fit(X_train, y_train)
 print(rf_grid_search.best_score_)
 print(rf_grid_search.best_params_ )<compute_train_metric>
print('% target = 1 in train set: {}%'.format(round(float(len(y_train[y_train==1])) / len(y_train)* 100)))
 print('% target = 1 in test set: {}%'.format(round(float(len(y_test[y_test==1])) / len(y_test)* 100)) )
rf_model = rf_grid_search.best_estimator_
cv_score = cross_val_score(rf_model, X_train, y_train, cv = 5)
 print(cv_score)
 print("Accuracy: %0.2f(+/- %0.2f)" %(cv_score.mean() , cv_score.std() * 2))<find_best_params>
regr_all = linear_model.LinearRegression(fit_intercept=True, normalize=True)
 regr_all.fit(X_train,y_train )
rf = rf_grid_search.best_estimator_.steps[0][1]<train_on_grid>
y_hat= regr_all.predict(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat)[100-tax])
 result['regr_all']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % regr_all.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))
%%time
xgd_pipe = Pipeline(
 steps = [ 
('classifier', XGBClassifier(n_estimators=50, subsample=0.5))
] 
)
 param_grid = {
'classifier__learning_rate' : [0.45],
'classifier__min_samples_split' : [8, 16, 32],
'classifier__min_samples_leaf' : [2],
'classifier__max_depth': [15]
}
np.random.seed(1)
 xgd_grid_search = GridSearchCV(xgd_pipe, param_grid, cv=5,
refit='True', verbose = 10, n_jobs=-1)
 xgd_grid_search.fit(X_train, y_train)
 print(xgd_grid_search.best_score_)
 print(xgd_grid_search.best_params_ )<compute_train_metric>
regr_ol = linear_model.LinearRegression(fit_intercept=True, normalize=True)
 regr_ol.fit(X_train_ol,y_train )
xgd_model = xgd_grid_search.best_estimator_
cv_score = cross_val_score(xgd_model, X_train, y_train, cv = 5)
 print(cv_score)
 print("Accuracy: %0.2f(+/- %0.2f)" %(cv_score.mean() , cv_score.std() * 2))<train_on_grid>
regr_ol.predict(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat)[100-tax])
 result['regr_ol']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % regr_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))
xrf_pipe = Pipeline(
 steps = [ 
('classifier', ExtraTreesClassifier(n_estimators=500,random_state=0, criterion = 'entropy'))
] 
)
 xrf_param_grid = {
'classifier__min_samples_leaf': [1,4,7],
'classifier__max_depth': [34,38,32],
}
np.random.seed(1)
 xrf_grid_search = GridSearchCV(xrf_pipe, xrf_param_grid, cv=5, refit='True', n_jobs=-1)
 xrf_grid_search.fit(X_train, y_train)
 print(xrf_grid_search.best_score_)
 print(xrf_grid_search.best_params_ )<compute_train_metric>
best=score[score['AUC ROC']>score['AUC ROC'].max() *0.95]
best
xrf_model = xrf_grid_search.best_estimator_
cv_score = cross_val_score(xrf_model, X_train, y_train, cv = 5)
 print(cv_score)
 print("Accuracy: %0.2f(+/- %0.2f)" %(cv_score.mean() , cv_score.std() * 2))<train_model>
regr_sel = RFE(regr_all, 4, step=1,verbose=0)
 regr_sel.fit(X_train,y_train )
final_model = xrf_grid_search.best_estimator_.steps[0][1]
final_model.fit(X_train, y_train)
<predict_on_test>
y_hat= regr_sel.predict(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat)[100-tax])
 result['regre_sel']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % regr_sel.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))
y_pred = final_model.predict(X_test )<count_values>
param_grid = { 'criterion': ['gini','entropy'],
'splitter': ['best','random'],
'max_depth': [2,4,10,20,None],
'max_features' : ['auto', 'sqrt', 'log2']}
Tree = DecisionTreeClassifier()
Counter(y_pred )<prepare_output>
grid = GridSearchCV(estimator=Tree, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
submission = pd.DataFrame({'Id': test.Id,
'Cover_Type': y_pred})
 submission.head()<save_to_csv>
Tree = DecisionTreeClassifier(**best_param)
 Tree.fit(X_train,y_train )
submission.to_csv('submission.csv', index=False )<save_to_csv>
y_hat= Tree.predict(X_test)
 result['Tree']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % Tree.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))
submission.to_csv('submission.csv', index=False )<load_from_csv>
grid = GridSearchCV(estimator=Tree, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
train=pd.read_csv('/kaggle/input/forest-cover-type-prediction/train.csv', index_col='Id')
 test=pd.read_csv('/kaggle/input/forest-cover-type-prediction/test.csv', index_col='Id')
 submission=pd.read_csv('/kaggle/input/forest-cover-type-prediction/sampleSubmission.csv', index_col='Id' )<prepare_x_and_y>
Tree_ol = DecisionTreeClassifier(**best_param)
 Tree_ol.fit(X_train_ol,y_train )
Ytrain=train['Cover_Type']
train=train[list(test)]<count_missing_values>
y_hat= Tree_ol.predict(X_test_ol)
 result['Tree_ol']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % Tree_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))
train.isnull().sum()<count_missing_values>
Tree_selec = RFE(Tree, 94, step=1,verbose=0)
 Tree_selec.fit(X_train,y_train )
train.isnull().sum()<count_missing_values>
y_hat= Tree_selec.predict(X_test)
 result['Tree_selec']=y_hat
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat - y_test)** 2))
print('Variance score: %.2f' % Tree_selec.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat))
test.isnull().sum()<count_missing_values>
param_grid = { 'C': [0.001,0.1,10.0,100.0],
'fit_intercept': [True,False],
'class_weight': ['balanced', None],
'solver' : ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']}
LR = LogisticRegression()
test.isnull().sum()<define_variables>
grid = GridSearchCV(estimator=LR, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
for col in ['Wilderness_Area1', 'Wilderness_Area2', 'Wilderness_Area3', 'Wilderness_Area4', 'Soil_Type1', 'Soil_Type2', 'Soil_Type3', 'Soil_Type4', 'Soil_Type5', 'Soil_Type6',
'Soil_Type7', 'Soil_Type8', 'Soil_Type9', 'Soil_Type10', 'Soil_Type11', 'Soil_Type12', 'Soil_Type13', 'Soil_Type14', 'Soil_Type15', 'Soil_Type16', 'Soil_Type17',
'Soil_Type18', 'Soil_Type19', 'Soil_Type20', 'Soil_Type21', 'Soil_Type22', 'Soil_Type23', 'Soil_Type24', 'Soil_Type25', 'Soil_Type26', 'Soil_Type27', 'Soil_Type28',
'Soil_Type29', 'Soil_Type30', 'Soil_Type31', 'Soil_Type32', 'Soil_Type33', 'Soil_Type34', 'Soil_Type35', 'Soil_Type36', 'Soil_Type37', 'Soil_Type38', 'Soil_Type39',
'Soil_Type40']:
if train[col].nunique() != test[col].nunique() :
print(f'Different Values train-test : {col}' )<drop_column>
LR = LogisticRegression(**best_param)
 LR.fit(X_train,y_train )
train=train.drop(columns=['Soil_Type7','Soil_Type15'])
 test=test.drop(columns=['Soil_Type7','Soil_Type15'] )<count_values>
y_hat= LR.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['LR']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % LR.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
sns.countplot(Ytrain)
 Ytrain.value_counts()<count_unique_values>
grid = GridSearchCV(estimator=LR, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
print(f"Num_Kind of plants living in water? : {Ytrain.loc[train.loc[train['Distance_To_Hydrology']==0].index].nunique() }" )<feature_engineering>
LR_ol = LogisticRegression(**best_param)
 LR_ol.fit(X_train_ol,y_train )
sns.distplot(np.log(train['Distance_To_Hydrology']+1), fit=norm)
 train['Distance_To_Hydrology']=np.log(train['Distance_To_Hydrology']+1)
 test['Distance_To_Hydrology']=np.log(test['Distance_To_Hydrology']+1)
 train.loc[train['Distance_To_Hydrology']==0,'Distance_To_Hydrology']=np.nan
test.loc[test['Distance_To_Hydrology']==0,'Distance_To_Hydrology']=np.nan<choose_model_class>
y_hat= LR_ol.predict_proba(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['LR_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % LR_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
qut=QuantileTransformer(output_distribution='normal')
 pipeline=make_pipeline(qut, PCA(n_components=1))<categorify>
best=score[score['AUC ROC']>score['AUC ROC'].max() *0.95]
best
train['Elevation_Roadways']=pipeline.fit_transform(train[['Elevation','Horizontal_Distance_To_Roadways']])
 test['Elevation_Roadways']=pipeline.transform(test[['Elevation','Horizontal_Distance_To_Roadways']] )<prepare_x_and_y>
LR_selec = RFE(LR, 14, step=1,verbose=0)
 LR_selec.fit(X_train,y_train )
alpha=train[['Hillshade_9am','Hillshade_3pm']]
alpha2=test[['Hillshade_9am','Hillshade_3pm']]<categorify>
y_hat= LR_selec.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['LR_select']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % LR_selec.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
train['Hillshade9am_Hillshade_3pm']=pipeline.fit_transform(alpha)
 test['Hillshade9am_Hillshade_3pm']=pipeline.transform(alpha2 )<prepare_x_and_y>
param_grid = { 'activation': ['identity', 'logistic', 'tanh', 'relu'],
'solver': ['lbfgs', 'sgd', 'adam'],
'learning_rate': ['constant', 'invscaling', 'adaptive']}
MLPerseptron = MLPClassifier()
alpha=train[['Aspect','Hillshade9am_Hillshade_3pm']]
alpha2=test[['Aspect','Hillshade9am_Hillshade_3pm']]<categorify>
grid = GridSearchCV(estimator=MLPerseptron, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
train['Aspect_Hillshade']=pipeline.fit_transform(alpha)
 test['Aspect_Hillshade']=pipeline.transform(alpha2 )<prepare_x_and_y>
MLPerseptron = MLPClassifier(**best_param)
 MLPerseptron.fit(X_train,y_train )
alpha=train[['Slope','Hillshade_Noon']]
alpha2=test[['Slope','Hillshade_Noon']]<categorify>
y_hat= MLPerseptron.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['MLPerseptron']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % MLPerseptron.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
train['Slope_Hillshade']=pipeline.fit_transform(alpha)
 test['Slope_Hillshade']=pipeline.transform(alpha2 )<prepare_x_and_y>
grid = GridSearchCV(estimator=MLPerseptron, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
Xtrain=train
Xtest=test<train_model>
MLPerseptron_ol = MLPClassifier(**best_param)
 MLPerseptron_ol.fit(X_train_ol,y_train )
rf=RandomForestClassifier(n_estimators=5000, random_state=18, n_jobs=-1)
 rf.fit(Xtrain.fillna(-999), Ytrain)
 feature_importance_df=pd.DataFrame(data=None, columns=['feature','importances'])
 feature_importance_df['importances']=rf.feature_importances_
feature_importance_df['feature']=Xtrain.columns
feature_importance_df=feature_importance_df.sort_values(by='importances', ascending=False)
 feature_importance_df.tail()<choose_model_class>
y_hat= MLPerseptron_ol.predict_proba(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['MLPerseptron_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % MLPerseptron_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
params={'n_estimators': 1222, 'learning_rate': 0.07307234151834806, 'num_leaves': 96, 'colsample_bytree': 0.8972376156262298,
'subsample': 0.9312856106293543, 'min_child_samples': 1}
lightgbm=LGBMClassifier(random_state=18, subsample_freq=1, silent=False, **params )<save_to_csv>
grid = GridSearchCV(estimator=g_NB, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1,cv=2)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
lightgbm.fit(Xtrain, Ytrain)
 predictions=lightgbm.predict(Xtest)
 submission['Cover_Type']=predictions
submission.to_csv('LGBSingleModel.csv')
 submission.head()<import_modules>
g_NB = GaussianNB(**best_param)
 g_NB.fit(X_train, y_train )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')

train = train.fillna(0)
 test = test.fillna(0)
 column_target = 'Cover_Type'
column_id = 'Id'
data = train.dropna(axis='columns')
 print("Total data: ",len(data))
X = data
y = data[column_target]
c_sample = 0.99
if c_sample < 1.0:
X_null, X, y_null, y = train_test_split(X, y, test_size=c_sample, random_state=0)
 ID = X[column_id]
y = X[column_target]
X = X.drop([column_id, column_target], axis=1 ).select_dtypes(include=[np.number])
 train_select_atributes = X.columns
print("Amostragem: ",len(X))


X = X.values
y = y.values

 model = RandomForestClassifier(random_state=0, n_estimators=500)
 kf = StratifiedKFold(n_splits=10)
 outcomes = []
for train_index, test_index in kf.split(X, y):
Xtrain, Xtest = X[train_index], X[test_index]
ytrain, ytest = y[train_index], y[test_index]
model.fit(Xtrain, ytrain)
 expected = ytest
predictions = model.predict(Xtest)
 accuracy = accuracy_score(ytest, predictions)
 outcomes.append(accuracy)
 mean_outcome = np.array(outcomes ).mean()
print("Mean Accuracy:", mean_outcome)

X_test = test[train_select_atributes]
x_test_id = test[column_id]
predictions = model.predict(X_test)
 predictions = pd.DataFrame(predictions, columns = ["Cover_Type"])
 result = pd.concat([x_test_id, predictions], axis=1, sort=False)
 result.to_csv("result.csv", mode = 'w', index=False)

isfineClass = False
for i in range(len(y)) :
if y[i]==0:
isfineClass=True
break;
if isfineClass==False:
for i in range(len(y)) :
v = y[i]
y[i] = v-1
c_sample = 0.1
if c_sample < 1.0:
X_null, X, y_null, y = train_test_split(X, y, test_size=c_sample, random_state=0)
 print("Amostragem para a visualização: ", len(X))
tsne = TSNE(n_components=2, random_state=0)
 X_2d = tsne.fit_transform(X)
 plt.figure(figsize=(6, 5))
colors = ["
for i in range(len(y)) :
v = y[i]
plt.plot(X_2d[i, 0], X_2d[i, 1], 'o', color=colors[v], alpha=0.3)
 plt.show() 
<load_from_csv>
y_hat= g_NB.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['g_NB']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % g_NB.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
application_train=pd.read_csv(r".. /input/application_train.csv")
 application_test=pd.read_csv(r".. /input/application_test.csv")
 bureau_balance=pd.read_csv(r".. /input/bureau_balance.csv")
 bureau=pd.read_csv(r".. /input/bureau.csv")
 credit_card_balance=pd.read_csv(r".. /input/credit_card_balance.csv")
 POS_cash=pd.read_csv(r".. /input/POS_CASH_balance.csv")
 bureau=pd.read_csv(r".. /input/bureau.csv")
 previous_application=pd.read_csv(r".. /input/previous_application.csv")
 install_payment=pd.read_csv(r".. /input/installments_payments.csv" )<count_missing_values>
grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
total_null=application_train.isnull().sum().sort_values(ascending=False)
 percentage=(application_train.isnull().sum() /application_train.isnull().count() *100 ).sort_values(ascending=False)
 missing_train_data=pd.concat([total_null,percentage],axis=1,keys=["Total_null","Percentage"] )<count_missing_values>
grid.fit(X_train_ol,y_train)
 best_param=grid.best_params_
g_NB_ol = GaussianNB(**best_param)
 g_NB_ol.fit(X_train_ol, y_train )
total_null=POS_cash.isnull().sum().sort_values(ascending=False)
 percentage=(POS_cash.isnull().sum() /POS_cash.isnull().count() *100 ).sort_values(ascending=False)
 missing_POS_data=pd.concat([total_null,percentage],axis=1,keys=["Total_null","Percentage"] )<sort_values>
y_hat= g_NB.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['g_NB_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % g_NB.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
total_null=bureau.isnull().sum().sort_values(ascending=False)
 percentage=(bureau.isnull().sum() /bureau.isnull().count() *100 ).sort_values(ascending=False)
 missing_bureau_data=pd.concat([total_null,percentage],axis=1,keys=["Total_null","Percentage"])
 missing_bureau_data.head(15 )<count_missing_values>
param_grid = { 'n_neighbors': list(range(4,100)) ,
'weights': ['uniform','distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'p' : [1,2]}
knn = KNeighborsClassifier()
total_null=bureau_balance.isnull().sum().sort_values(ascending=False)
 percentage=(bureau_balance.isnull().sum() /bureau_balance.isnull().count() *100 ).sort_values(ascending=False)
 missing_bureau_balance=pd.concat([total_null,percentage],axis=1,keys=["Total_null","Percentage"])
 missing_bureau_balance.head(15 )<count_missing_values>
grid = GridSearchCV(estimator=knn, param_grid=param_grid, scoring= 'roc_auc', verbose=1, n_jobs=-1)
 grid.fit(X_train,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
total_null=previous_application.isnull().sum().sort_values(ascending=False)
 percentage=(previous_application.isnull().sum() /previous_application.isnull().count() *100 ).sort_values(ascending=False)
 missing_previous_application=pd.concat([total_null,percentage],axis=1,keys=["Total_null","Percentage"])
 missing_previous_application.head(15 )<count_missing_values>
knn = KNeighborsClassifier(**best_param)
 knn.fit(X_train, y_train )
total_null=install_payment.isnull().sum().sort_values(ascending=False)
 percentage=(install_payment.isnull().sum() /install_payment.isnull().count() *100 ).sort_values(ascending=False)
 missing_installment=pd.concat([total_null,percentage],axis=1,keys=["Total_null","Percentage"])
 missing_installment.head()<count_missing_values>
y_hat= knn.predict_proba(X_test)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['knn']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % knn.score(X_test, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
total_null=application_test.isnull().sum().sort_values(ascending=False)
 percentage=(application_test.isnull().sum() /application_test.isnull().count() *100 ).sort_values(ascending=False)
 missing_app_test=pd.concat([total_null,percentage],axis=1,keys=["Total_null","Percentage"])
 missing_app_test.head(15 )<count_unique_values>
grid.fit(X_train_ol,y_train)
 print("Best Score= " + str(grid.best_score_))
print("Best Parameters= "+str(grid.best_params_))
best_param=grid.best_params_
application_train.select_dtypes("object" ).nunique()<categorify>
grid.fit(X_train_ol,y_train)
 best_param=grid.best_params_
knn_ol = KNeighborsClassifier(**best_param)
 knn_ol.fit(X_train_ol, y_train )
lbl=LabelEncoder()
lbl_count=0
for i in application_train:
if application_train[i].dtype=='object':
if len(list(application_train[i].unique())) <= 2:
lbl.fit(application_train[i])
 application_train[i]=lbl.transform(application_train[i])
 application_test[i]=lbl.transform(application_test[i])
 lbl_count +=1
print('%d column were encoded.'%lbl_count)
<count_unique_values>
y_hat= knn_ol.predict_proba(X_test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[100-tax])
 result['knn_ol']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)))
 print("Residual sum of squares: %.2f" % np.mean(( y_hat[:,1] - y_test)** 2))
print('Variance score: %.2f' % knn_ol.score(X_test_ol, y_test))
print('AUC ROC: %.2f' % roc_auc_score(y_test, y_hat[:,1]))
application_train.select_dtypes("object" ).nunique()<categorify>
temp=pd.DataFrame()
temp['mean']=result.mean(axis=1)
 result2=pd.concat([result, temp], axis=1 )
application_train=pd.get_dummies(application_train)
 application_test=pd.get_dummies(application_test)
 print("application_train feature shape:",application_train.shape)
 print("application_test feature shape:",application_test.shape )<concatenate>
FinalResult=pd.DataFrame()
for results in result2.columns:
y_hat=result2[results]
AUC_ROC=roc_auc_score(y_test, y_hat)
 FinalResult[results]=[AUC_ROC]
FinalResult.transpose()
train_target=application_train['TARGET']
application_train,application_test=application_train.align(application_test,axis=1,join='inner' )<feature_engineering>
import sklearn.metrics as metrics
import matplotlib.pyplot as plt
application_train['TARGET']=train_target<count_missing_values>
X_Test = test_df.drop(columns=["id"])
 X_Test_ol=X_Test[d]
LR_ol.fit(X_ol,y)
application_train.isnull().sum()<count_missing_values>
y_hat= knn_ol.predict_proba(X_Test_ol)
 y_hat=y_hat*0.5/(np.sort(y_hat[:,1])[round(( y_hat[:,1].size)*(100-tax)/100)])
 submission['target']=y_hat[:,1]
print('% target = 1: {}%(in the train sample 64%)'.format(round(float(len(y_hat[y_hat[:,1]>=0.5])) / len(y_hat)* 100)) )
application_train.isnull().sum()<categorify>
submission.to_csv('submission.csv', index=False)
 submission.head(20 )