Kaggle competition: Don't Overfit! II
<feature_engineering><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<feature_engineering>
import pandas as pd
import numpy as np
train['dist_hydr']=np.sqrt(1+train['dist_hydr'])
 test['dist_hydr']=np.sqrt(1+test['dist_hydr'])
<feature_engineering>
df_train = pd.read_csv('.. /input/train.csv', index_col='id')
 df_test = pd.read_csv('.. /input/test.csv', index_col='id')
 df_train.shape, df_test.shape
train.Horizontal_Distance_To_Hydrology=np.sqrt(1+train.Horizontal_Distance_To_Hydrology)
 test.Horizontal_Distance_To_Hydrology=np.sqrt(1+test.Horizontal_Distance_To_Hydrology )<feature_engineering>
target = 'target'
X_train = df_train.drop([target], axis=1 ).values
y_train = df_train[target]
train['Horizontal_Distance_To_Fire_Points']=np.log(1+train.Horizontal_Distance_To_Fire_Points)
 test['Horizontal_Distance_To_Fire_Points']=np.log(1+test.Horizontal_Distance_To_Fire_Points)
<normalization>
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from stability_selection import StabilitySelection
sd=StandardScaler()<feature_engineering>
Cs = np.logspace(-5, 5, 21)
 pipe = Pipeline(steps=[ 
('scaler', StandardScaler()), 
('clf', LogisticRegression(solver='lbfgs', verbose=1, n_jobs=1, random_state=42))
])
 ss = StabilitySelection(base_estimator=pipe,
lambda_name='clf__C', lambda_grid=Cs,
n_jobs=-1, verbose=1, random_state=42 )
df_train=train.iloc[:,1:11]
df_train['dist_hydr']=train['dist_hydr']
df_train.info()<feature_engineering>
%%time
ss.fit(X_train, y_train )
df_test=test.iloc[:,1:11]
df_test['dist_hydr']=test['dist_hydr']
df_test.info()<normalization>
from stability_selection import plot_stability_path
sd.fit(df_train)
 df_train=sd.transform(df_train )<feature_engineering>
ss_indices = np.where(ss.get_support() == True)[0]
ss_indices
train['dist_hydr']=df_train[:,10]<feature_engineering>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import RepeatedStratifiedKFold
from mlxtend.feature_selection import SequentialFeatureSelector
df_test=sd.transform(df_test)
 test.iloc[:,1:11]=df_test[:,0:10]
test['dist_hydr']=df_test[:,10]<drop_column>
pipe = Pipeline(steps=[ 
('scaler', StandardScaler()), 
('clf', LogisticRegression(solver='lbfgs', verbose=1, n_jobs=1, random_state=42))
])
 cv = RepeatedStratifiedKFold(2, 50, random_state=42)
 sfs = SequentialFeatureSelector(pipe, k_features=(5,10), floating=True,
scoring='roc_auc', cv=cv, verbose=1, n_jobs=-1 )
train.drop(columns=['Id'],axis=1,inplace=True )<drop_column>
%%time
sfs.fit(X_train, y_train )
Id=test['Id']
test.drop(columns=['Id'],axis=1,inplace=True )<filter>
import matplotlib.pyplot as plt
print(train_corr.loc['Soil_Type38','Soil_Type39'])
 print(train_corr.loc['Soil_Type38','Wilderness_Area1'])
 print(train_corr.loc['Soil_Type39','Wilderness_Area1'] )<feature_engineering>
from sklearn.model_selection import GridSearchCV
train['soil_type38,39']=train['Soil_Type38']+train['Soil_Type39']
train['soil_38_Wilde_area_1']=train['Soil_Type38']+train['Wilderness_Area1']
train['soil_39_Wilde_area_1']=train['Soil_Type39']+train['Wilderness_Area1']
test['soil_type38,39']=test['Soil_Type38']+test['Soil_Type39']
test['soil_38_Wilde_area_1']=test['Soil_Type38']+test['Wilderness_Area1']
test['soil_39_Wilde_area_1']=test['Soil_Type39']+test['Wilderness_Area1']<prepare_x_and_y>
from sklearn.model_selection import GridSearchCV
X=train.drop(columns='Cover_Type',axis=1)
 y=train['Cover_Type']<split>
Cs = np.logspace(-5, 5, 21)
 params = {
'clf__C': Cs
}
pipe = Pipeline(steps=[ 
('scaler', StandardScaler()), 
('clf', LogisticRegression(solver='lbfgs', verbose=1, n_jobs=1, random_state=42))
])
 cv = RepeatedStratifiedKFold(2, 100, random_state=42)
 grid = GridSearchCV(estimator=pipe, param_grid=params, cv=cv,
return_train_score=True,
scoring='roc_auc', verbose=1, n_jobs=-1 )
x_train,x_valid,y_train,y_valid=train_test_split(X,y,test_size=0.25 )<import_modules>
%%time
grid.fit(X_train, y_train )
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from sklearn.naive_bayes import GaussianNB<define_variables>
pipe = grid.best_estimator_
pipe
clf_accuracy=[]<compute_train_metric>
import matplotlib.pyplot as plt
lg=LogisticRegression(max_iter=1000)
 lg.fit(x_train,y_train)
 pred=lg.predict(x_valid)
 clf_accuracy.append(accuracy_score(y_valid,pred))
print(accuracy_score(y_valid,pred))<train_model>
X_test = df_test.values
for t in [ss, sfs]:
X_test = t.transform(X_test )
model=SVC(kernel='rbf')
 model.fit(x_train,y_train)
 pred=(model.predict(x_valid))
clf_accuracy.append(accuracy_score(y_valid,pred))
print(accuracy_score(y_valid,pred))<compute_train_metric>
pipe.fit(X_train, y_train )
rand=RandomForestClassifier()
rand.fit(x_train,y_train)
 pred=rand.predict(x_valid)
 clf_accuracy.append(accuracy_score(y_valid,pred))
print(accuracy_score(y_valid,pred))<compute_train_metric>
y_pred = pipe.predict_proba(X_test)[:, 1]
xgb=XGBClassifier(max_depth=7)
 xgb.fit(x_train,y_train)
 pred=xgb.predict(x_valid)
 clf_accuracy.append(accuracy_score(y_valid,pred))
print(accuracy_score(y_valid,pred))<compute_train_metric>
df_subm = pd.read_csv('.. /input/sample_submission.csv', index_col='id' )
nb=GaussianNB()
nb.fit(x_train,y_train)
 pred=nb.predict(x_valid)
 clf_accuracy.append(accuracy_score(y_valid,pred))
print(accuracy_score(y_valid,pred)) 
<define_variables>
df_subm[target] = y_pred
<predict_on_test><EOS> 
df_subm.to_csv('submission.csv' )