Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<choose_model_class>
%matplotlib inline
label=LabelEncoder()
clf = GradientBoostingClassifier(n_estimators=200,learning_rate=0.15, random_state=0)
 kfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)
 scoring = {'acc': 'accuracy', 'f1': 'f1_micro'}<compute_train_metric>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df.head()
result = cross_validate(clf,train, y_mind, scoring=scoring,
cv=kfolds, n_jobs=-1, verbose=1 )<compute_test_metric>
test_df.Fare.fillna(test_df.Fare.mean() , inplace=True)
 data_df = train_df.append(test_df)
 passenger_id=test_df['PassengerId']
train_df.drop(['PassengerId'], axis=1, inplace=True)
 test_df.drop(['PassengerId'], axis=1, inplace=True)
 test_df.shape
print('Y_mind model performance:')
 pprint(result)
 for key in result:
print(key + ' : ', result[key].mean() )<compute_train_metric>
train_df=train_df[train_df['Fare']<400]
result = cross_validate(clf,train, y_energy, scoring=scoring,
cv=kfolds, n_jobs=-1, verbose=1 )<compute_test_metric>
train_df['Sex'] = train_df.Sex.apply(lambda x: 0 if x == "female" else 1)
 test_df['Sex'] = test_df.Sex.apply(lambda x: 0 if x == "female" else 1 )
print('Y_energy model performance:')
 pprint(result)
 for key in result:
print(key + ' : ', result[key].mean() )<compute_train_metric>
pd.options.display.max_columns = 99
test_df['Fare'].fillna(test_df['Fare'].mean() ,inplace=True)
 train_df.head()
result = cross_validate(clf,train, y_nature, scoring=scoring,
cv=kfolds, n_jobs=-1, verbose=1 )<compute_test_metric>
for name_string in data_df['Name']:
data_df['Title']=data_df['Name'].str.extract('([A-Za-z]+)\.',expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_df.replace({'Title': mapping}, inplace=True )
print('Y_nature model performance:')
 pprint(result)
 for key in result:
print(key + ' : ', result[key].mean() )<compute_train_metric>
for name_string in data_df['Name']:
data_df['Title']=data_df['Name'].str.extract('([A-Za-z]+)\.',expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_df.replace({'Title': mapping}, inplace=True)
 data_df['Title'].value_counts()
train_df['Title']=data_df['Title'][:891]
test_df['Title']=data_df['Title'][891:]
titles=['Mr','Miss','Mrs','Master','Rev','Dr']
for title in titles:
age_to_impute = data_df.groupby('Title')['Age'].mean() [titles.index(title)]
data_df.loc[(data_df['Age'].isnull())&(data_df['Title'] == title), 'Age'] = age_to_impute
data_df.isnull().sum()
train_df['Age']=data_df['Age'][:891]
test_df['Age']=data_df['Age'][891:]
test_df.isnull().sum()
result = cross_validate(clf,train, y_tactics, scoring=scoring,
cv=kfolds, n_jobs=-1, verbose=1 )<compute_test_metric>
train_df.groupby('Survived' ).mean()
print('Y_tactics model performance:')
 pprint(result)
 for key in result:
print(key + ' : ', result[key].mean() )<predict_on_test>
train_df.groupby('Sex' ).mean()
clf.fit(train,y_mind)
 X_t = test
predictions_Mind = pd.DataFrame(clf.predict(X_t))<predict_on_test>
train_df['family_size'] = train_df.SibSp + train_df.Parch+1
test_df['family_size'] = test_df.SibSp + test_df.Parch+1
clf.fit(train,y_energy)
 predictions_Energy = pd.DataFrame(clf.predict(X_t))<predict_on_test>
def family_group(size):
a = ''
if(size <= 1):
a = 'loner'
elif(size <= 4):
a = 'small'
else:
a = 'large'
return a
train_df['family_group'] = train_df['family_size'].map(family_group)
 test_df['family_group'] = test_df['family_size'].map(family_group )
clf.fit(train,y_tactics)
 predictions_Tactic = pd.DataFrame(clf.predict(X_t))<predict_on_test>
train_df['is_alone'] = [1 if i<2 else 0 for i in train_df.family_size]
test_df['is_alone'] = [1 if i<2 else 0 for i in test_df.family_size]
clf.fit(train,y_nature)
 predictions_Nature = pd.DataFrame(clf.predict(X_t))<concatenate>
train_df['child'] = [1 if i<16 else 0 for i in train_df.Age]
test_df['child'] = [1 if i<16 else 0 for i in test_df.Age]
train_df.child.value_counts()
submission = pd.concat([predictions_Mind,predictions_Energy,predictions_Nature,predictions_Tactic], axis=1 )<prepare_output>
train_df['calculated_fare'] = train_df.Fare/train_df.family_size
test_df['calculated_fare'] = test_df.Fare/test_df.family_size
submission['index'] = submission['index'] +1
submission.columns = ['id', 'mind', 'energy', 'nature', 'tactics']
submission.head()<save_to_csv>
train_df.calculated_fare.mode()
submission.to_csv('submission_Gradient.csv', index=False )<import_modules>
train_df['fare_group'] = train_df['calculated_fare'].map(fare_group)
 test_df['fare_group'] = test_df['calculated_fare'].map(fare_group )
if not 'sklearn' in sys.modules.keys() :
pip.main(['install', 'sklearn'])
<load_from_csv>
train_df = pd.get_dummies(train_df, columns=['Title',"Pclass",'Embarked', 'family_group', 'fare_group'], drop_first=True)
 test_df = pd.get_dummies(test_df, columns=['Title',"Pclass",'Embarked', 'family_group', 'fare_group'], drop_first=True)
 train_df.drop(['Cabin', 'family_size','Ticket','Name', 'Fare'], axis=1, inplace=True)
 test_df.drop(['Ticket','Name','family_size',"Fare",'Cabin'], axis=1, inplace=True)
train = pd.read_csv('.. /input/train.csv')
 train_y = train.Quality
predictor_cols = ['fixed.acidity','volatile.acidity','citric.acid','residual.sugar','chlorides','free.sulfur.dioxide','total.sulfur.dioxide','density','pH','sulphates','alcohol']
train_X = train[predictor_cols]
imp = SimpleImputer(missing_values=np.nan, strategy='mean')
 imp = imp.fit(train_X )<train_model>
pd.options.display.max_columns = 99
train_X_imp = imp.transform(train_X)
 my_model = RandomForestClassifier(n_estimators=100)
 my_model.fit(train_X_imp, train_y )<create_dataframe>
train_df['age_group'] = train_df['Age'].map(age_group_fun)
 test_df['age_group'] = test_df['Age'].map(age_group_fun )
feature_importances = pd.DataFrame(my_model.feature_importances_,
index = train_X.columns,
columns=['importance'] ).sort_values('importance', ascending=False)
 feature_importances<predict_on_test>
train_df = pd.get_dummies(train_df,columns=['age_group'], drop_first=True)
 test_df = pd.get_dummies(test_df,columns=['age_group'], drop_first=True)
 train_df.drop(['Age','calculated_fare'],axis=1,inplace=True)
 test_df.drop(['Age','calculated_fare'],axis=1,inplace=True )
test = pd.read_csv('.. /input/test.csv')
 test_X = test[predictor_cols]
test_X_imp = imp.transform(test_X)
 predicted_q = my_model.predict(test_X_imp)
 print(predicted_q)
<save_to_csv>
train_df.head()
train_df.drop(['Title_Rev','age_group_old','age_group_teenager','age_group_senior_citizen','Embarked_Q'],axis=1,inplace=True)
 test_df.drop(['Title_Rev','age_group_old','age_group_teenager','age_group_senior_citizen','Embarked_Q'],axis=1,inplace=True )
my_submission = pd.DataFrame({'Id': test.Id, 'Quality': predicted_q})
 my_submission.to_csv('submission.csv', index=False )<import_modules>
X = train_df.drop('Survived', 1)
 y = train_df['Survived']
warnings.filterwarnings('ignore')
 print(os.listdir(".. /input")) 
<define_variables>
classifiers = [
KNeighborsClassifier(3),
svm.SVC(probability=True),
DecisionTreeClassifier() ,
CatBoostClassifier() ,
XGBClassifier() ,
RandomForestClassifier() ,
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
GaussianNB() ,
LinearDiscriminantAnalysis() ,
QuadraticDiscriminantAnalysis() ,
LogisticRegression() ]
log_cols = ["Classifier", "Accuracy"]
log= pd.DataFrame(columns=log_cols)
baseline_tree_score = 0.23092278864723115
baseline_neuralnetwork_score = 0.5480561937041435<load_from_csv>
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split,StratifiedShuffleSplit
train = pd.read_csv('.. /input/kaggletutorial/covertype_train.csv')
 test = pd.read_csv('.. /input/kaggletutorial/covertype_test.csv' )<define_variables>
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import BaggingClassifier
from sklearn.metrics import accuracy_score,classification_report, precision_recall_curve, confusion_matrix
train_index = train.shape[0]<init_hyperparams>
std_scaler = StandardScaler()
X = std_scaler.fit_transform(X)
 testframe = std_scaler.fit_transform(test_df)
 testframe.shape
lgbm_param = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.06,
"num_leaves": 16,
"max_depth": 6,
"colsample_bytree": 0.7,
"subsample": 0.8,
"reg_alpha": 0.1,
"reg_lambda": 0.1,
"nthread":8
}<choose_model_class>
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.20,random_state=1000 )
def keras_model(input_dims):
model = Sequential()
model.add(Dense(input_dims, input_dim=input_dims))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims//2))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam', metrics=['accuracy'])
 return model
def keras_history_plot(history):
plt.plot(history.history['loss'], 'y', label='train loss')
 plt.plot(history.history['val_loss'], 'r', label='val loss')
 plt.xlabel('epoch')
 plt.ylabel('loss')
 plt.legend(loc='upper right')
 plt.show()<split>
logreg = LogisticRegression(solver='liblinear', penalty='l1')
 logreg.fit(X_train,y_train)
 predict=logreg.predict(X_test)
 print(accuracy_score(y_test,predict))
print(confusion_matrix(y_test,predict))
print(precision_score(y_test,predict))
print(recall_score(y_test,predict))
def baseline_tree_cv(train):
train_df = train.copy()
y_value = train_df["Cover_Type"]
del train_df["Cover_Type"], train_df["ID"]
NFOLD = 5
folds = StratifiedKFold(n_splits= NFOLD, shuffle=True, random_state=2018)
 total_score = 0
best_iteration = 0
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, y_value)) :
train_x, train_y = train_df.iloc[train_idx], y_value.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], y_value.iloc[valid_idx]
evals_result_dict = {}
dtrain = lgbm.Dataset(train_x, label=train_y)
 dvalid = lgbm.Dataset(valid_x, label=valid_y)
 clf = lgbm.train(lgbm_param, train_set=dtrain, num_boost_round=3000, valid_sets=[dtrain, dvalid],
early_stopping_rounds=200, evals_result=evals_result_dict, verbose_eval=500)
 predict = clf.predict(valid_x)
 cv_score = log_loss(valid_y, predict)
 total_score += cv_score
best_iteration = max(best_iteration, clf.best_iteration)
 print('Fold {} LogLoss : {}'.format(n_fold + 1, cv_score))
lgbm.plot_metric(evals_result_dict)
 plt.show()
print("Best Iteration", best_iteration)
 print("Total LogLoss", total_score / NFOLD)
 print("Baseline model Score Diff", total_score / NFOLD - baseline_tree_score)
 del train_df
return best_iteration
def baseline_keras_cv(train):
train_df = train.copy()
y_value = train_df['Cover_Type']
del train_df['Cover_Type'], train_df['ID']
model = keras_model(train_df.shape[1])
 callbacks = [
EarlyStopping(
 patience=10,
verbose=10)
 ]
NFOLD = 5
folds = StratifiedKFold(n_splits= NFOLD, shuffle=True, random_state=2018)
 total_score = 0
best_epoch = 0
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df, y_value)) :
train_x, train_y = train_df.iloc[train_idx], y_value.iloc[train_idx]
valid_x, valid_y = train_df.iloc[valid_idx], y_value.iloc[valid_idx]
history = model.fit(train_x.values, train_y.values, nb_epoch=30, batch_size = 64, validation_data=(valid_x.values, valid_y.values),
verbose=1, callbacks=callbacks)
 keras_history_plot(history)
 predict = model.predict(valid_x.values)
 null_count = np.sum(pd.isnull(predict))
if null_count > 0:
print("Null Prediction Error: ", null_count)
 predict[pd.isnull(predict)] = predict[~pd.isnull(predict)].mean()
cv_score = log_loss(valid_y, predict)
 total_score += cv_score
best_epoch = max(best_epoch, np.max(history.epoch))
print('Fold {} LogLoss : {}'.format(n_fold + 1, cv_score))
print("Best Epoch: ", best_epoch)
 print("Total LogLoss", total_score/NFOLD)
 print("Baseline model Score Diff", total_score/NFOLD - baseline_neuralnetwork_score )<categorify>
C_vals = [0.0001, 0.001, 0.01, 0.1,0.13,0.2,.15,.25,.275,.33, 0.5,.66, 0.75, 1.0, 2.5, 4.0,4.5,5.0,5.1,5.5,6.0, 10.0, 100.0, 1000.0]
penalties = ['l1','l2']
param = {'penalty': penalties, 'C': C_vals, }
grid = GridSearchCV(logreg, param,verbose=False, cv = StratifiedKFold(n_splits=5,random_state=10,shuffle=True), n_jobs=1,scoring='accuracy' )
def outlier_binary(frame, col, outlier_range):
outlier_feature = col + '_Outlier'
frame[outlier_feature] = 0
frame.loc[frame[col] > outlier_range, outlier_feature] = 1
return frame
def outlier_divide_ratio(frame, col, outlier_range):
outlier_index = frame[col] >= outlier_range
outlier_median = frame.loc[outlier_index, col].median()
normal_median = frame.loc[frame[col] < outlier_range, col].median()
outlier_ratio = outlier_median / normal_median
frame.loc[outlier_index, col] = frame.loc[outlier_index, col]/outlier_ratio
return frame
def frequency_encoding(frame, col):
freq_encoding = frame.groupby([col] ).size() /frame.shape[0]
freq_encoding = freq_encoding.reset_index().rename(columns={0:'{}_Frequncy'.format(col)})
 return frame.merge(freq_encoding, on=col, how='left')
 def binning_category_combine_feature(frame, col1, col2, col1_quantile, col2_quantile):
print(col1, ' ', col2, 'Bining Combine')
 col1_quantile = np.arange(0,1.1,col1_quantile)
 col2_quantile = np.arange(0,1.1,col2_quantile)
 col1_label = '{}_quantile_label'.format(col1)
 frame[col1_label] = pd.qcut(frame[col1], q=col1_quantile, labels = ['{}_quantile_{:.1f}'.format(col1, col)for col in col1_quantile][1:])
 col2_label = '{}_quantile_label'.format(col2)
 frame[col2_label] = pd.qcut(frame[col2], q=col2_quantile, labels = ['{}_quantile_{:.1f}'.format(col2, col)for col in col2_quantile][1:])
 combine_label = 'Binnig_{}_{}_Combine'.format(col1, col2)
 frame[combine_label] = frame[[col1_label, col2_label]].apply(lambda row: row[col1_label] +'_'+ row[col2_label] ,axis=1)
 for col in [col1_label, col2_label, combine_label]:
frame[col] = frame[col].factorize() [0]
gc.collect()
return frame, [col1_label, col2_label, combine_label]<feature_engineering>
grid.fit(X_train,y_train)
 print(grid.best_params_)
 print(grid.best_score_)
 print(grid.best_estimator_ )
def tree_data_preprocessing(train, test):
train_index = train.shape[0]
all_data = pd.concat([train, test])
 del all_data['oil_Type']
all_column_set = set(all_data.columns)
 category_feature = []
for col in all_data.loc[:, all_data.dtypes=='object'].columns:
all_data[col] = all_data[col].factorize() [0]
category_feature.append(col)
 numerical_feature = list(all_column_set - set(category_feature)- set(['Cover_Type','ID']))
all_data['Elevation'] = np.log1p(all_data['Elevation'])
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = frequency_encoding(all_data, 'Soil_Type')
 all_data = frequency_encoding(all_data, 'Wilderness_Area')
 aspect_train = all_data.loc[all_data['Aspect'].notnull() ]
aspect_test = all_data.loc[all_data['Aspect'].isnull() ]
del aspect_train["Cover_Type"], aspect_train['ID']
del aspect_test["Cover_Type"], aspect_test['ID']
numerical_feature_woaspect = numerical_feature[:]
numerical_feature_woaspect.remove('Aspect')
 sc = StandardScaler()
aspect_train[numerical_feature_woaspect] = sc.fit_transform(aspect_train[numerical_feature_woaspect])
 aspect_test[numerical_feature_woaspect] = sc.transform(aspect_test[numerical_feature_woaspect])
 y_value = aspect_train['Aspect']
del aspect_train['Aspect'], aspect_test['Aspect']
knn = KNeighborsRegressor(n_neighbors=7)
 knn.fit(aspect_train,y_value)
 predict = knn.predict(aspect_test)
 sns.distplot(predict)
 sns.distplot(all_data['Aspect'].dropna())
 plt.title('KNN Aspect Null Imputation')
 plt.show()
all_data.loc[all_data['Aspect'].isnull() ,'Aspect'] = predict
all_data['Horizontal_Distance_To_Hydrology'] = all_data['Horizontal_Distance_To_Hydrology']/1000
all_data['HF1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Fire_Points'])
 all_data['HF4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Fire_Points']
all_data['HR1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Roadways']
all_data['HR2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Roadways']
all_data['HR3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['HR4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Roadways']
all_data['HH1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Vertical_Distance_To_Hydrology']
all_data['HH2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Vertical_Distance_To_Hydrology']
all_data['HH3'] = np.log1p(abs(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Vertical_Distance_To_Hydrology']))
all_data['HH4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Vertical_Distance_To_Hydrology']
all_data['FR1'] = all_data['Horizontal_Distance_To_Fire_Points'] + all_data['Horizontal_Distance_To_Roadways']
all_data['FR2'] = all_data['Horizontal_Distance_To_Fire_Points'] - all_data['Horizontal_Distance_To_Roadways']
all_data['FR3'] = np.log1p(all_data['Horizontal_Distance_To_Fire_Points'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['FR4'] = all_data['Horizontal_Distance_To_Fire_Points'] / all_data['Horizontal_Distance_To_Roadways']
all_data['Direct_Distance_Hydrology'] =(all_data['Horizontal_Distance_To_Hydrology']**2+all_data['Vertical_Distance_To_Hydrology']**2)**0.5
all_data.loc[np.isinf(all_data['HF4']),'HF4'] = 0
all_data.loc[np.isinf(all_data['HR4']),'HR4'] = 0
all_data.loc[np.isinf(all_data['HH4']),'HH4'] = 0
all_data.loc[np.isinf(all_data['FR4']),'FR4'] = 0
all_data[['HF4','HH4']] = all_data[['HF4','HH4']].fillna(0)
 all_data, new_col = binning_category_combine_feature(all_data, 'Elevation', 'Aspect', 0.1, 0.1)
 for col in new_col:
all_data = frequency_encoding(all_data, col)
 train_df = all_data.iloc[:train_index]
test_df = all_data.iloc[train_index:]
soil_mean_encoding = train_df.groupby(['Soil_Type'])['Cover_Type'].agg({'Soil_Type_Mean':'mean',
'Soil_Type_Std':'std',
'Soil_Type_Size':'size',
'Soil_Type_Sum':'sum'} ).reset_index()
train_df = train_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 test_df = test_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 wildness_mean_encoding = train_df.groupby(['Wilderness_Area'])['Cover_Type'].agg({'Wilderness_Area_Mean':'mean',
'Wilderness_Area_Std':'std',
'Wilderness_Area_Size':'size',
'Wilderness_Area_Sum':'sum'} ).reset_index()
train_df = train_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 test_df = test_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 del all_data, predict, aspect_train, aspect_test
gc.collect()
return train_df, test_df<feature_engineering>
logreg_grid = LogisticRegression(penalty=grid.best_params_['penalty'], C=grid.best_params_['C'])
 logreg_grid.fit(X_train,y_train)
 y_pred = logreg_grid.predict(X_test)
 logreg_accy = round(accuracy_score(y_test, y_pred), 3)
 print(logreg_accy)
 print(confusion_matrix(y_test,y_pred))
print(precision_score(y_test,y_pred))
print(recall_score(y_test,y_pred))
def nn_data_preprocessing(train, test):
train_index = train.shape[0]
all_data = pd.concat([train, test])
 del all_data['oil_Type']
all_column_set = set(all_data.columns)
 category_feature = []
for col in all_data.loc[:, all_data.dtypes=='object'].columns:
all_data[col] = all_data[col].factorize() [0]
category_feature.append(col)
 numerical_feature = list(all_column_set - set(category_feature)- set(['Cover_Type','ID']))
all_data['Elevation'] = np.log1p(all_data['Elevation'])
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_binary(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Fire_Points', 10000)
 all_data = outlier_divide_ratio(all_data, 'Horizontal_Distance_To_Roadways', 10000)
 all_data = frequency_encoding(all_data, 'Soil_Type')
 all_data = frequency_encoding(all_data, 'Wilderness_Area')
 aspect_train = all_data.loc[all_data['Aspect'].notnull() ]
aspect_test = all_data.loc[all_data['Aspect'].isnull() ]
del aspect_train["Cover_Type"], aspect_train['ID']
del aspect_test["Cover_Type"], aspect_test['ID']
numerical_feature_woaspect = numerical_feature[:]
numerical_feature_woaspect.remove('Aspect')
 sc = StandardScaler()
aspect_train[numerical_feature_woaspect] = sc.fit_transform(aspect_train[numerical_feature_woaspect])
 aspect_test[numerical_feature_woaspect] = sc.transform(aspect_test[numerical_feature_woaspect])
 y_value = aspect_train['Aspect']
del aspect_train['Aspect'], aspect_test['Aspect']
knn = KNeighborsRegressor(n_neighbors=7)
 knn.fit(aspect_train,y_value)
 predict = knn.predict(aspect_test)
 sns.distplot(predict)
 sns.distplot(all_data['Aspect'].dropna())
 plt.title('KNN Aspect Null Imputation')
 plt.show()
all_data.loc[all_data['Aspect'].isnull() ,'Aspect'] = predict
all_data['Horizontal_Distance_To_Hydrology'] = all_data['Horizontal_Distance_To_Hydrology']/1000
all_data['HF1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Fire_Points']
all_data['HF3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Fire_Points'])
 all_data['HF4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Fire_Points']
all_data['HR1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Horizontal_Distance_To_Roadways']
all_data['HR2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Horizontal_Distance_To_Roadways']
all_data['HR3'] = np.log1p(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['HR4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Horizontal_Distance_To_Roadways']
all_data['HH1'] = all_data['Horizontal_Distance_To_Hydrology'] + all_data['Vertical_Distance_To_Hydrology']
all_data['HH2'] = all_data['Horizontal_Distance_To_Hydrology'] - all_data['Vertical_Distance_To_Hydrology']
all_data['HH3'] = np.log1p(abs(all_data['Horizontal_Distance_To_Hydrology'] * all_data['Vertical_Distance_To_Hydrology']))
all_data['HH4'] = all_data['Horizontal_Distance_To_Hydrology'] / all_data['Vertical_Distance_To_Hydrology']
all_data['FR1'] = all_data['Horizontal_Distance_To_Fire_Points'] + all_data['Horizontal_Distance_To_Roadways']
all_data['FR2'] = all_data['Horizontal_Distance_To_Fire_Points'] - all_data['Horizontal_Distance_To_Roadways']
all_data['FR3'] = np.log1p(all_data['Horizontal_Distance_To_Fire_Points'] * all_data['Horizontal_Distance_To_Roadways'])
 all_data['FR4'] = all_data['Horizontal_Distance_To_Fire_Points'] / all_data['Horizontal_Distance_To_Roadways']
all_data['Direct_Distance_Hydrology'] =(all_data['Horizontal_Distance_To_Hydrology']**2+all_data['Vertical_Distance_To_Hydrology']**2)**0.5
all_data.loc[np.isinf(all_data['HF4']),'HF4'] = 0
all_data.loc[np.isinf(all_data['HR4']),'HR4'] = 0
all_data.loc[np.isinf(all_data['HH4']),'HH4'] = 0
all_data.loc[np.isinf(all_data['FR4']),'FR4'] = 0
all_data[['HF4','HH4']] = all_data[['HF4','HH4']].fillna(0)
 all_data, new_col = binning_category_combine_feature(all_data, 'Elevation', 'Aspect', 0.1, 0.1)
 for col in new_col:
all_data = frequency_encoding(all_data, col)
 all_data.drop(columns=new_col,axis=1,inplace=True)
 before_one_hot = set(all_data.columns)
 for col in category_feature:
all_data = pd.concat([all_data,pd.get_dummies(all_data[col],prefix=col)],axis=1)
 one_hot_feature = set(all_data.columns)- before_one_hot
train_df = all_data.iloc[:train_index]
test_df = all_data.iloc[train_index:]
soil_mean_encoding = train_df.groupby(['Soil_Type'])['Cover_Type'].agg({'Soil_Type_Mean':'mean',
'Soil_Type_Std':'std',
'Soil_Type_Size':'size',
'Soil_Type_Sum':'sum'} ).reset_index()
train_df = train_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 test_df = test_df.merge(soil_mean_encoding, on='Soil_Type', how='left')
 wildness_mean_encoding = train_df.groupby(['Wilderness_Area'])['Cover_Type'].agg({'Wilderness_Area_Mean':'mean',
'Wilderness_Area_Std':'std',
'Wilderness_Area_Size':'size',
'Wilderness_Area_Sum':'sum'} ).reset_index()
train_df = train_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 test_df = test_df.merge(wildness_mean_encoding, on='Wilderness_Area', how='left')
 train_df.drop(columns=category_feature, axis=1, inplace=True)
 test_df.drop(columns=category_feature, axis=1, inplace=True)
 scale_feature = list(set(train_df.columns)-one_hot_feature-set(['Cover_Type','ID']))
sc = StandardScaler()
train_df[scale_feature] = sc.fit_transform(train_df[scale_feature])
 test_df[scale_feature] = sc.transform(test_df[scale_feature])
 return train_df, test_df<merge>
ABC=AdaBoostClassifier()
ABC.fit(X_train,y_train)
 predict=ABC.predict(X_test)
 print(accuracy_score(y_test,predict))
print(confusion_matrix(y_test,predict))
print(precision_score(y_test,predict))
org_train_df, org_test_df = tree_data_preprocessing(train, test )<create_dataframe>
n_estimator=[50,60,100,150,200,300]
learning_rate=[0.001,0.01,0.1,0.2,]
hyperparam={'n_estimators':n_estimator,'learning_rate':learning_rate}
gridBoost=GridSearchCV(ABC,param_grid=hyperparam,verbose=False, cv = StratifiedKFold(n_splits=5,random_state=15,shuffle=True), n_jobs=1,scoring='accuracy' )
train_df = org_train_df.copy()
test_df = org_test_df.copy()<init_hyperparams>
gridBoost.fit(X_train,y_train)
 print(gridBoost.best_score_)
 print(gridBoost.best_estimator_ )
lgbm_param = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'binary_logloss',
"learning_rate": 0.03,
"num_leaves": 24,
"max_depth": 6,
"colsample_bytree": 0.65,
"subsample": 0.7,
"reg_alpha": 0.1,
"reg_lambda": 0.2,
"nthread":8
}<prepare_x_and_y>
gridBoost.best_estimator_.fit(X_train,y_train)
 predict=gridBoost.best_estimator_.predict(X_test)
 print(accuracy_score(y_test,predict))
y_value = train_df["Cover_Type"]
del train_df["Cover_Type"], train_df["ID"]
del test_df["Cover_Type"], test_df["ID"]<split>
xgb=XGBClassifier(max_depth=2, n_estimators=700, learning_rate=0.009,nthread=-1,subsample=1,colsample_bytree=0.8)
 xgb.fit(X_train,y_train)
 predict=xgb.predict(X_test)
 print(accuracy_score(y_test,predict))
print(confusion_matrix(y_test,predict))
print(precision_score(y_test,predict))
print(recall_score(y_test,predict))
<predict_on_test>
lda=LinearDiscriminantAnalysis()
lda.fit(X_train,y_train)
 predict=lda.predict(X_test)
 print(accuracy_score(y_test,predict))
print(precision_score(y_test,predict))
print(recall_score(y_test,predict))
dtrain = lgbm.Dataset(train_df, label=y_value)
 clf = lgbm.train(lgbm_param, train_set=dtrain, num_boost_round=5000)
 predict = clf.predict(test_df )<save_to_csv>
dectree = DecisionTreeClassifier(criterion="entropy",
max_depth=5,
class_weight = 'balanced',
min_weight_fraction_leaf = 0.009,
random_state=2000)
 dectree.fit(X_train, y_train)
 y_pred = dectree.predict(X_test)
 dectree_accy = round(accuracy_score(y_pred, y_test), 3)
 print(dectree_accy)
 print(confusion_matrix(y_test,y_pred))
print(precision_score(y_test,y_pred))
print(recall_score(y_test,y_pred))
submission = pd.read_csv('.. /input/kaggletutorial/sample_submission.csv')
 submission['Cover_Type'] = predict
submission.to_csv('lgbm_last.csv', index=False )<compute_test_metric>
randomforest = RandomForestClassifier(n_estimators=100,max_depth=5,min_samples_split=20,max_features=0.2, min_samples_leaf=8,random_state=20)
 randomforest.fit(X_train, y_train)
 y_pred = randomforest.predict(X_test)
 random_accy = round(accuracy_score(y_pred, y_test), 3)
 print(random_accy)
 print(precision_score(y_test,y_pred))
print(recall_score(y_test,y_pred))
print(confusion_matrix(y_test,y_pred))
org_train_df, org_test_df = nn_data_preprocessing(train, test )<create_dataframe>
BaggingClassifier = BaggingClassifier()
BaggingClassifier.fit(X_train, y_train)
 y_pred = BaggingClassifier.predict(X_test)
 bagging_accy = round(accuracy_score(y_pred, y_test), 3)
 print(bagging_accy )
train_df = org_train_df.copy()
test_df = org_test_df.copy()<choose_model_class>
voting_classifier = VotingClassifier(estimators=[ 
('logreg',logreg), 
('random_forest', randomforest), 
('decision_tree',dectree), 
('XGB Classifier', xgb), 
('BaggingClassifier', BaggingClassifier)])
 voting_classifier.fit(X_train,y_train)
 y_pred = voting_classifier.predict(X_test)
 voting_accy = round(accuracy_score(y_pred, y_test), 3)
 print(voting_accy )
def keras_model(input_dims):
model = Sequential()
model.add(Dense(input_dims, input_dim=input_dims))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims//2))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(input_dims//5))
model.add(BatchNormalization())
 model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam', metrics=['accuracy'])
 return model 
<prepare_x_and_y>
model = CatBoostClassifier(verbose=False, one_hot_max_size=3)
 model.fit(X_train,y_train)
 y_pred=model.predict(X_test)
 acc = round(accuracy_score(y_pred, y_test), 3)
 print(acc )
y_value = train_df['Cover_Type']
del train_df['Cover_Type'], train_df['ID']
del test_df['Cover_Type'], test_df['ID']
model = keras_model(train_df.shape[1])
 callbacks = [
EarlyStopping(
 patience=10,
verbose=10)
 ]<train_model>
y_predict=model.predict(testframe )
<train_model>
temp = pd.DataFrame(pd.DataFrame({
"PassengerId": passenger_id,
"Survived": y_predict
}))
temp.to_csv(".. /working/submission3.csv", index = False )