Kaggle competition: Forest Cover Type Prediction
drop_features = ['MSSubClass','OverallCond','BsmtFinType2','BsmtFinSF2','LowQualFinSF','BsmtHalfBath','PoolArea','MiscVal','MoSold', 'YrSold','GarageArea',
'1stFlrSF','TotRmsAbvGrd','GarageYrBlt','LotArea','Alley','LandContour','LandSlope','Heating','Electrical','BsmtExposure','BldgType',
'Utilities','Functional']
df.drop(drop_features,axis=1,inplace=True)
<count_missing_values>
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense,Dropout
def missing_values(data):
values = data.isnull().sum().sort_values(ascending = False)
 percentage =(data.isnull().sum() /data.isnull().count() *100 ).sort_values(ascending = False)
 missing_df = pd.concat([values,percentage], axis=1, keys=['Values', 'Percentage%'])
 return missing_df
missing_df = missing_values(df)
 missing_df.head(25 )<count_values>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
df.drop([ 'PoolQC','MiscFeature','Fence','FireplaceQu'], axis=1, inplace=True)
 num_colums = ['LotFrontage','MasVnrArea','BsmtFullBath','BsmtFinSF1','TotalBsmtSF','BsmtUnfSF','GarageCars']
for col in num_colums:
df[col].fillna(df[col].mean() , inplace = True)
 catg_colums = ['GarageCond','GarageFinish','GarageQual','GarageType','BsmtCond','BsmtQual','BsmtFinType1','MasVnrType','MSZoning','Exterior1st','Exterior2nd',
'SaleType','KitchenQual']
for col in catg_colums:
df[col].fillna(df[col].value_counts().index[0], inplace=True )<categorify>
train = pd.read_csv(".. /input/train.csv")
 test =pd.read_csv(".. /input/test.csv")
df = pd.get_dummies(df, drop_first=True)
 df.head()<prepare_x_and_y>
train.isnull().sum()
minmax = MinMaxScaler()
normal_df = pd.DataFrame(minmax.fit_transform(df))<split>
train.isnull().sum()
train = normal_df.iloc[:1455,:].values
test = normal_df.iloc[1455:,:].values<import_modules>
test.isnull().sum()
from sklearn.linear_model import LinearRegression,BayesianRidge,Ridge,Lasso,HuberRegressor,ElasticNet
from sklearn.ensemble import RandomForestRegressor,AdaBoostRegressor,BaggingRegressor,GradientBoostingRegressor
from catboost import CatBoostRegressor
from sklearn.svm import SVR
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
from mlxtend.regressor import StackingCVRegressor
import tensorflow as tf
from sklearn.model_selection import cross_val_score, KFold
from sklearn.metrics import mean_squared_error,make_scorer,r2_score
from sklearn.model_selection import RandomizedSearchCV
from keras.callbacks import ModelCheckpoint
from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten<train_on_grid>
test.isnull().sum()
parameters = {'alpha':[0.001,0.01,0.1,1,2,3,40,50,100,200, 230, 250,265, 270, 275, 290, 300, 500]}
ridge_1 = Ridge()
randomSearch = RandomizedSearchCV(ridge_1, param_distributions=parameters, n_jobs=-1, cv=10)
 randomSearch.fit(train,y)
 bestpara = randomSearch.best_params_
def model(reg,train,label):

 cross_val = KFold(n_splits=10 ,shuffle= True , random_state= 0)
 rmsle = make_scorer(r2_score)
 validation = cross_val_score(reg ,train, label, cv=cross_val, scoring=rmsle,n_jobs=-1)
 return validation.mean()
model_1 = BayesianRidge()
score_1 = model(model_1,train,y)
 model_2 = CatBoostRegressor()
score_2 = model(model_2,train,y)
 ridge = Ridge(alpha=1)
 score = model(ridge,train,y)

stack_model = StackingCVRegressor(regressors=(model_1,model_2), meta_regressor=ridge)
 final_score = model(stack_model,train,y)
 stack_gen = stack_model.fit(train,y )<compute_test_metric>
for items in train.columns:
if train[items].std() == 0:
print(items)
 train.drop(items,axis = 1,inplace = True )
print('Bayesian Ridge Accuracy = {0:.2f}%'.format(score_1*100))
print('CatBoost Accuracy = {0:.2f}%'.format(score_2*100))
print('Ridge Accuracy = {0:.2f}%'.format(score*100))
print('Stack Accuracy = {0:.2f}%'.format(final_score*100))<save_to_csv>
test.drop("Soil_Type7",axis = 1,inplace = True)
 test.drop("Soil_Type15",axis = 1,inplace =True )
test_pred = stack_gen.predict(test)
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission['SalePrice'] = np.floor(np.expm1(test_pred))
submission = submission[['Id', 'SalePrice']]
submission.to_csv('blend_submission.csv',index=False)
 print('submission file is created' )<import_modules>
train["net_hyd_distance"]=np.sqrt(train["Vertical_Distance_To_Hydrology"]**2 + train["Horizontal_Distance_To_Hydrology"]**2)
 test["net_hyd_distance"]=np.sqrt(test["Vertical_Distance_To_Hydrology"]**2 + test["Horizontal_Distance_To_Hydrology"]**2 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import skew
from sklearn.preprocessing import LabelEncoder
import lightgbm as lgb<load_from_csv>
train["mean_distance_aminity"] =(train["Horizontal_Distance_To_Hydrology"] + train["Horizontal_Distance_To_Roadways"] + train["Horizontal_Distance_To_Fire_Points"])/3
test["mean_distance_aminity"] =(test["Horizontal_Distance_To_Hydrology"] + test["Horizontal_Distance_To_Roadways"] + test["Horizontal_Distance_To_Fire_Points"])/3
submission = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission.head()<load_from_csv>
train["closeness1"] = train["Horizontal_Distance_To_Hydrology"] - train["Horizontal_Distance_To_Roadways"]
train["closeness2"] = train["Horizontal_Distance_To_Hydrology"] - train["Horizontal_Distance_To_Fire_Points"]
train["closeness3"] = train["Horizontal_Distance_To_Roadways"] - train["Horizontal_Distance_To_Fire_Points"]
test["closeness1"] = test["Horizontal_Distance_To_Hydrology"] - test["Horizontal_Distance_To_Roadways"]
test["closeness2"] = test["Horizontal_Distance_To_Hydrology"] - test["Horizontal_Distance_To_Fire_Points"]
test["closeness3"] = test["Horizontal_Distance_To_Roadways"] - test["Horizontal_Distance_To_Fire_Points"]
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 train.head()<load_from_csv>
predictor = train.drop(["Id","Cover_Type"],axis = 1)
 target = train["Cover_Type"]
test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 test.head()<train_model>
sc = StandardScaler()
print('=========== train infomation ===========')
 train.info()
print('
=========== test infomation ===========')
 test.info()<concatenate>
rnd = RandomForestClassifier()
rnd.fit(predictor,target )
data = pd.concat([train, test])
 data.shape<drop_column>
importances = rnd.feature_importances_
indices = np.argsort(importances)[::-1]
numerics = data.loc[:,data.dtypes != 'object'].drop('Id', axis=1)
 numerics.head()<feature_engineering>
train["Elevation_roadways"] = train["Elevation"] -.02*train["Horizontal_Distance_To_Roadways"]
train["Elevation_vd"] = train["Elevation"] - train["Vertical_Distance_To_Hydrology"]
train["Elevation_hd"] = train["Elevation"] -.2*train["Horizontal_Distance_To_Hydrology"]
test["Elevation_roadways"] = test["Elevation"] -.02*test["Horizontal_Distance_To_Roadways"]
test["Elevation_vd"] = test["Elevation"] - test["Vertical_Distance_To_Hydrology"]
test["Elevation_hd"] = test["Elevation"] -.2*test["Horizontal_Distance_To_Hydrology"]
log_numerics = np.log1p(numerics)
 log_numerics.head()<categorify>
predictor = train.drop(["Id","Cover_Type"],axis = 1)
 target = train.Cover_Type
cat_data = pd.get_dummies(data.loc[:, cat_cols], drop_first=True)
 cat_data.head()<concatenate>
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
optimized_data = pd.concat([data['Id'], cat_data, log_numerics], axis=1)
 optimized_data.head()<prepare_x_and_y>
rnd1 = RandomForestClassifier(random_state = 123,n_estimators = 175 )
train = optimized_data[:train.shape[0]]
test = optimized_data[train.shape[0]:].drop(['Id', 'SalePrice'], axis=1)
 X_train = train.drop(['Id', 'SalePrice'], axis=1)
 y_train = train['SalePrice']<predict_on_test>
print(cross_val_score(rnd1,predictor,target,cv = 3 ).mean() )
lgb_train = lgb.Dataset(X_train, y_train)
 params = {
'objective' : 'regression',
'metric' : {'rmse'}
}
gbm = lgb.train(params, lgb_train)
 pred = gbm.predict(test )<save_to_csv>
ex = ExtraTreesClassifier(random_state =123,n_estimators = 1000)
 ex1 = AdaBoostClassifier(base_estimator = ex,n_estimators = 5)
pred = np.expm1(pred)
 results = pd.Series(pred, name='SalePrice')
 submission = pd.concat([submission['Id'], results], axis=1)
 submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
cross_val_score(ex1,predictor,target,cv = 3 ).mean()
warnings.filterwarnings('ignore')
<load_from_csv>
ex.fit(predictor,target )
train_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<feature_engineering>
prediction = ex.predict(test.drop("Id",axis = 1))
test_df['SalePrice'] = np.nan
test_df.head()<prepare_x_and_y>
test["Cover_Type"] = prediction
train_df = train_df.drop(['Id'], axis = 1)
 test_df = test_df.drop(['Id'], axis = 1)
 target = train_df['SalePrice']
all_data = pd.concat([train_df, test_df], ignore_index = True)
 all_data.head()<count_missing_values>
my_submission = test[["Id","Cover_Type"]]
def missing_values_table(df):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() / len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 mis_val_table_ren_columns = mis_val_table_ren_columns[
mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
 '% of Total Values', ascending=False ).round(1)
 print("Your selected dataframe has " + str(df.shape[1])+ " columns.
 "
"There are " + str(mis_val_table_ren_columns.shape[0])+
" columns that have missing values.")
 return mis_val_table_ren_columns<count_missing_values>
test["Cover_Type"].value_counts(normalize = True)*100
missing_values = missing_values_table(all_data)
 missing_values<count_values>
prediction = cross_val_predict(ex1,predictor,target,cv = 3 )
all_data.dtypes.value_counts()<count_unique_values>
from sklearn.metrics import confusion_matrix
all_data.select_dtypes('object' ).apply(pd.Series.nunique, axis = 0 )<categorify>
from sklearn.metrics import classification_report
all_data['PoolQC'] = all_data['PoolQC'].fillna('NA')
 print(all_data['PoolQC'].isnull().sum())
 all_data.head()<filter>
print(classification_report(target,prediction))
all_data.loc[(all_data['PoolQC'] == 'NA')&(all_data['PoolArea'] != 0)]<feature_engineering>
train1 = train
all_data.at[2420, 'PoolQC'] = 'Fa'
all_data.at[2599, 'PoolQC'] = 'Fa'
all_data.at[2503, 'PoolQC'] = 'Fa'<categorify>
predictor = train.drop(["Id","Cover_Type"],axis = 1)
 target = train.Cover_Type
maps = {
'Ex': 5,
'Gd': 4,
'TA': 3,
'Fa': 2,
'NA': 1
}
all_data['PoolQC'] = all_data['PoolQC'].replace(maps)
 all_data.head()<categorify>
target = target.replace([1,2],0 )
all_data[all_data['LotFrontage'].isnull() == True].SalePrice.median()
all_data['LotFrontage'] = all_data['LotFrontage'].fillna(all_data['LotFrontage'].median())
 all_data['GarageCond'] = all_data['GarageCond'].fillna('NA' ).replace(maps)
 all_data['GarageQual'] = all_data['GarageQual'].fillna('NA' ).replace(maps)
 mapp = {
'Fin': 4,
'RFn': 3,
'Unf': 2,
'NA': 1
}
all_data['GarageFinish'] = all_data['GarageFinish'].fillna('NA' ).replace(mapp)
 all_data['GarageType'] = all_data['GarageType'].fillna('NA')
 all_data['GarageYrBlt'] = all_data['GarageYrBlt'].fillna(-1)
 all_data['GarageArea'] = all_data['GarageArea'].fillna(0.0)
 all_data['GarageCars'] = all_data['GarageCars'].fillna(0.0)
 all_data['BsmtHalfBath'] = all_data['BsmtHalfBath'].fillna(0.0)
 all_data['BsmtFullBath'] = all_data['BsmtFullBath'].fillna(0.0)
 mapp = {
'Gd':5,
'Av':4,
'Mn':3,
'No':2,
'NA':1
}
all_data['BsmtExposure'] = all_data['BsmtExposure'].fillna('NA' ).replace(mapp)
 all_data['BsmtCond'] = all_data['BsmtCond'].fillna('NA' ).replace(maps)
 all_data['BsmtQual'] = all_data['BsmtQual'].fillna('NA' ).replace(maps)
 mapp = {
'GLQ':7,
'ALQ':6,
'BLQ':5,
'Rec':4,
'LwQ':3,
'Unf':2,
'NA':1
}
all_data['BsmtFinType1'] = all_data['BsmtFinType1'].fillna('NA' ).replace(mapp)
 all_data['BsmtFinType2'] = all_data['BsmtFinType2'].fillna('NA' ).replace(mapp)
 all_data[['MasVnrType','MasVnrArea']][all_data['MasVnrType'].isnull() == True]
all_data['MasVnrType'] = all_data['MasVnrType'].fillna('None')
 all_data['MasVnrArea'] = all_data['MasVnrArea'].fillna(0.0)
 all_data.at[2610, 'MasVnrType'] = 'Stone'
all_data['BsmtFinSF1'] = all_data['BsmtFinSF1'].fillna(0.0)
 all_data['BsmtFinSF2'] = all_data['BsmtFinSF2'].fillna(0.0)
 all_data['BsmtUnfSF'] = all_data['BsmtUnfSF'].fillna(0.0)
 all_data['TotalBsmtSF'] = all_data['TotalBsmtSF'].fillna(0.0)
 all_data['SaleType'] = all_data.SaleType.fillna('Oth')
 all_data['KitchenQual'] = all_data.KitchenQual.fillna('Oth')
 for col in all_data.columns:
if(( all_data[col].isnull().sum() > 0)&(col != 'SalePrice')) :
all_data[col] = all_data[col].fillna('NA')
 all_data.head()<sort_values>
target = target.replace([3,4,5,6,7],1 )
linearFeat = []
cor = all_data.corr() ['SalePrice'].sort_values(ascending=False)[all_data.corr() ['SalePrice'] > 0.5]
cor.head(20 )<create_dataframe>
ex = ExtraTreesClassifier(random_state = 123,n_estimators = 1000)
 ex1 = AdaBoostClassifier(base_estimator = ex,n_estimators = 10)
all_Data = all_data.copy()<drop_column>
cross_val_score(ex,predictor,target,cv = 3 ).mean()
all_data = all_data.drop('SalePrice', axis = 1 )<normalization>
ex.fit(predictor,target )
scaler = StandardScaler()
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
for i in numeric_feats:
all_data[i] = scaler.fit_transform(np.array(all_data[i] ).reshape(-1, 1))
all_data.head()<categorify>
prediction = ex.predict(test.drop(["Id","Cover_Type"],axis = 1))
le = LabelEncoder()
le_count = 0
for col in all_data:
if all_data[col].dtype == 'object':
if len(list(all_data[col].unique())) <= 2:
le.fit(all_data[col])
 all_data[col] = le.transform(all_data[col])
 le_count += 1
print('%d columns were label encoded.' % le_count )<categorify>
test["Cover_Type"] = prediction
all_data = pd.get_dummies(all_data)
 all_data.head()<train_model>
data = pd.concat([train[train["Cover_Type"] == 1],train[train["Cover_Type"] == 2]])
model = RandomForestRegressor(random_state=1, max_depth=10)
 train = all_data[:train_df.shape[0]]
test = all_data[train_df.shape[0]:]
x_train, x_valid, y_train, y_valid = train_test_split(train, target_log, test_size=0.3, random_state = 1)
 model.fit(x_train,y_train )<predict_on_test>
predictor1 = data.drop(["Id","Cover_Type"],axis = 1)
 target = data.Cover_Type
y_pred_val = model.predict(x_valid)
 print(math.sqrt(mean_squared_error(y_pred_val, y_valid)) )<prepare_output>
predict = cross_val_predict(ex1,predictor1,target,cv = 3 )
all_data = all_Data.copy()
all_data.head()<feature_engineering>
ex1.fit(predictor1,target )
all_data['totalBathroom'] = 0.5*all_data['HalfBath'] + all_data['FullBath'] + 0.5*all_data['BsmtHalfBath'] + all_data['FullBath']
all_data.head()<prepare_x_and_y>
data_test = test[test["Cover_Type"] == 0 ]
all_data['age'] = all_data['YrSold'] - all_data['YearRemodAdd']
ismodeled = []
deltaYr = all_data['YearRemodAdd'] - all_data['YearBuilt']<feature_engineering>
prediction1 = ex1.predict(data_test.drop(["Id","Cover_Type"],axis = 1))
ismodeled = []
for i in deltaYr:
if i == 0:
ismodeled.append(1)
 else:
ismodeled.append(0)
 all_data['IsModeled'] = ismodeled
all_data.head()<define_variables>
data_test["real"] = prediction1
deltaYr = all_data['YrSold'] - all_data['YearBuilt']
isnew = []
for i in deltaYr:
if i == 0:
isnew.append(1)
 else:
isnew.append(0)
 all_data['IsNew'] = isnew
all_data.head()<feature_engineering>
data1 = train[train["Cover_Type"] != 1]
all_data['totalSquareFeet'] = all_data['GrLivArea'] + all_data['TotalBsmtSF']
all_data.head()<feature_engineering>
data1 = data1[data1["Cover_Type"] != 2]
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )<sort_values>
data1.Cover_Type.value_counts()
cor = all_data.corr() ['SalePrice'].sort_values(ascending=False)[all_data.corr() ['SalePrice'] > 0.5]
cor.head(20 )<drop_column>
predictor2 = data1.drop(["Id","Cover_Type"],axis = 1)
 target2 = data1.Cover_Type
all_data = all_data.drop('SalePrice', axis = 1 )<normalization>
ex = ExtraTreesClassifier(random_state = 123,n_estimators = 500 )
scaler = StandardScaler()
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
for i in numeric_feats:
all_data[i] = scaler.fit_transform(np.array(all_data[i] ).reshape(-1, 1))
all_data.head()<categorify>
predict = cross_val_predict(ex1,predictor2,target2,cv = 3 )
le = LabelEncoder()
le_count = 0
for col in all_data:
if all_data[col].dtype == 'object':
if len(list(all_data[col].unique())) <= 2:
le.fit(all_data[col])
 all_data[col] = le.transform(all_data[col])
 le_count += 1
print('%d columns were label encoded.' % le_count )<categorify>
ex1.fit(predictor2,target2 )
all_data = pd.get_dummies(all_data )<create_dataframe>
test.drop("real",axis = 1,inplace = True )
kpca = KernelPCA(n_components = 80)
 all_data_pca = kpca.fit_transform(all_data)
 all_data_pca = pd.DataFrame(all_data_pca)
 all_data_pca.head()<feature_engineering>
dataset = test[test["Cover_Type"] == 1]
all_data_pca['Id'] = all_data_pca.index
all_data_pca.head()<split>
prediction2 = ex1.predict(dataset.drop(["Id","Cover_Type"],axis = 1))
train = all_data_pca[:train_df.shape[0]]
test = all_data_pca[train_df.shape[0]:]<create_dataframe>
dataset.drop("Cover_Type",axis = 1,inplace = True )
target = pd.DataFrame({'SalePrice': target_log})
 target.shape<split>
dataset["Cover_Type"] = prediction2
score = []
predict_val = pd.DataFrame(test['Id'])
 skf = KFold(n_splits = 5, shuffle=True, random_state=123)
 skf.get_n_splits(train, target_log)
 oof_rfr_df = pd.DataFrame()
predictions = pd.DataFrame(test['Id'])
 x_test = test.drop(['Id'], axis = 1)
 for fold,(trn_idx, val_idx)in enumerate(skf.split(train, target)) :
x_train, y_train = train.iloc[trn_idx], target.iloc[trn_idx]['SalePrice']
x_valid, y_valid = train.iloc[val_idx], target.iloc[val_idx]['SalePrice']
index = x_valid['Id']
yp = 0
x_train = x_train.drop(['Id'], axis = 1)
 x_valid = x_valid.drop(['Id'], axis = 1)
 rfr = RandomForestRegressor(random_state=1, max_depth=10)
 rfr.fit(x_train,y_train)
 score.append(math.sqrt(mean_squared_error(rfr.predict(x_valid), y_valid)))
 yp += rfr.predict(x_test)
 fold_pred = pd.DataFrame({'ID': index,
'label':rfr.predict(x_valid)})
 oof_rfr_df = pd.concat([oof_rfr_df, fold_pred], axis=0)
 predictions['fold{}'.format(fold+1)] = yp<prepare_output>
data_test.drop("Cover_Type",axis = 1,inplace = True )
rfr_predict = pd.DataFrame()
rfr_predict['predict'] =(predictions['fold1']+predictions['fold2']+predictions['fold3']+predictions['fold4']+predictions['fold5'])/5
rfr_predict.head()<statistical_test>
data_test["Cover_Type"] = data_test["real"]
explainer = shap.TreeExplainer(rfr)
 shap_values = explainer.shap_values(x_valid )<prepare_output>
data_test.drop("real",axis = 1,inplace = True )
predict = np.exp(rfr_predict['predict']-1)
 mysubmit = pd.DataFrame({'Id': test['Id']+1,
'SalePrice': predict})
 mysubmit.head()<save_to_csv>
test = pd.concat([dataset,data_test] ).sort_index()
mysubmit.to_csv('mysubmit.csv', index=False )<load_from_csv>
my_submission3 = test[["Id","Cover_Type"]]
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 sub = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv" )<concatenate>
my_submission3.to_csv("my_submisssion3",index = False )