Kaggle competition: Forest Cover Type Prediction
<SOS> metric: CategorizationAccuracy Kaggle data source: forest-cover-type-prediction<compute_test_metric>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.pipeline import Pipeline
from xgboost import XGBClassifier
def mean_absolute_percentage_error(y_true, y_pred):
y_true, y_pred = np.array(y_true), np.array(y_pred)
 return np.mean(np.abs(( y_true - y_pred)/ y_true)) * 100
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
 pd.set_option('display.max_columns', None)
 pd.options.display.float_format = '{:.2f}'.format<load_from_csv>
test = pd.read_csv(".. /input/forest-cover-type-prediction/test.csv")
 train = pd.read_csv(".. /input/forest-cover-type-prediction/train.csv" )
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 df = train.append(test)
 print("Num of Object Variables:", df.select_dtypes(object ).shape[1])
 print("Num of Integer Variables:", df.select_dtypes("integer" ).shape[1])
 print("Num of Float Variables:", df.select_dtypes("float" ).shape[1])
 df.shape, train.shape, test.shape<feature_engineering>
train.isnull().sum()
def missing_values(data, plot = False, target = "SalePrice"):
mst = pd.DataFrame({"Num_Missing":data.isnull().sum() , "Missing_Ratio":data.isnull().sum() / data.shape[0]} ).sort_values("Num_Missing", ascending = False)
 mst["DataTypes"] = data[mst.index].dtypes.values
mst = mst[mst.Num_Missing > 0].reset_index().rename({"index":"Feature"}, axis = 1)
 mst = mst[mst.Feature != target]
print("Number of Variables include Missing Values:", mst.shape[0], "
")
 if mst[mst.Missing_Ratio >= 1.0].shape[0] > 0:
print("Full Missing Variables:",mst[mst.Missing_Ratio >= 1.0].Feature.tolist())
 data.drop(mst[mst.Missing_Ratio >= 1.0].Feature.tolist() , axis = 1, inplace = True)
 print("Full missing variables are deleted!", "
")
 if plot:
plt.figure(figsize =(25, 8))
p = sns.barplot(mst.Feature, mst.Missing_Ratio)
 for rotate in p.get_xticklabels() :
rotate.set_rotation(90)
 print(mst, "
")
 missing_values(df, plot = True, target = "SalePrice" )<feature_engineering>
train.isnull().sum()
cat_missing = ["Alley", 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1','BsmtFinType2','Fence', "MiscFeature",
"FireplaceQu", "GarageFinish","GarageCond", "GarageQual", "GarageType", "MasVnrType"]
for i in cat_missing:
df[i] = np.where(df[i].isnull() , "None", df[i])
 missing_values(df, plot = False, target = "SalePrice" )<feature_engineering>
X_train=train.drop(['Id', 'Cover_Type'],axis=1)
 y_train=train.Cover_Type
X_test=test.drop('Id',axis=1)
 test_id=test.Id
df["GarageYrBlt"]= np.where(( df.GarageYrBlt.isnull() == True)&(df.GarageArea == 0), df.YearBuilt,df.GarageYrBlt )<categorify>
scaler = MinMaxScaler()
Xs_train = scaler.fit_transform(X_train)
 Xs_test = scaler.transform(X_test)
def quick_missing_imp(data, num_method = "median", cat_length = 20, target = "SalePrice"):
variables_with_na = [col for col in data.columns if data[col].isnull().sum() > 0]
temp_target = data[target]
print("
print(data[variables_with_na].isnull().sum() , "
")
 data = data.apply(lambda x: x.fillna(x.mode() [0])if(x.dtype == "O" and len(x.unique())<= cat_length)else x, axis=0)
 if num_method == "mean":
data = data.apply(lambda x: x.fillna(x.mean())if x.dtype != "O" else x, axis=0)
 elif num_method == "median":
data = data.apply(lambda x: x.fillna(x.median())if x.dtype != "O" else x, axis=0)
 data[target] = temp_target
print("
Imputation method is 'MODE' for categorical variables!")
 print(" Imputation method is '" + num_method.upper() + "' for numeric variables!
")
 print(data[variables_with_na].isnull().sum() , "
")
 return data
df = quick_missing_imp(df, num_method = "median", cat_length = 17 )<categorify>
lr_pipe = Pipeline(
 steps = [ 
('scaler', MinMaxScaler()), 
('classifier', LogisticRegression(solver='lbfgs', n_jobs=-1))
] 
)
 lr_param_grid = {
'classifier__C': [1, 10, 100,1000],
}
np.random.seed(1)
 grid_search = GridSearchCV(lr_pipe, lr_param_grid, cv=5, refit='True')
 grid_search.fit(X_train, y_train)
 print(grid_search.best_score_)
 print(grid_search.best_params_)
 0.640079365079365
ordinal_vars = [
"LotShape", "OverallQual", "OverallCond", "ExterQual", "ExterCond", "BsmtQual",
"BsmtCond", "BsmtExposure", "BsmtFinType1", "BsmtFinType2", "HeatingQC", "KitchenQual", "Functional",
"FireplaceQu", "GarageQual", "GarageCond", "Fence", "Electrical"
]
def ordinal(serie, category):
numeric = np.arange(1, len(category)+1, 1)
 zip_iterator = zip(category, numeric)
 mapping = dict(zip_iterator)
 serie = serie.map(mapping)
 return serie
def transform_ordinal(data, ordinal_vars, category):
for i in ordinal_vars:
data[i] = ordinal(data[i], category = category)
 transform_ordinal(
 df,
ordinal_vars = ["ExterQual", "ExterCond", "HeatingQC","KitchenQual"],
category = ["Po", "Fa", "TA", "Gd" ,"Ex"] 
)
 transform_ordinal(
 df,
ordinal_vars = ["BsmtQual", "BsmtCond", "FireplaceQu","GarageCond", "GarageQual"],
category = ["None", "Po", "Fa", "TA", "Gd","Ex"] 
)
 transform_ordinal(
 df,
ordinal_vars = ["BsmtFinType1", "BsmtFinType2"],
category = ["None", "Unf", "LwQ", "Rec", "BLQ", "ALQ","GLQ"] 
)
 df["LotShape"] = ordinal(df["LotShape"], category = ["IR3", "IR2", "IR1","Reg"])
 df["BsmtExposure"] = ordinal(df["BsmtExposure"], category = ["None", "No", "Mn", "Av","Gd"])
 df["Functional"] = ordinal(df["Functional"], category = ["Sal", "Sev", "Maj2", "Maj1", "Mod", "Min2", "Min1","Typ"])
 df["Fence"] = ordinal(df["Fence"], category = ["None", "MnWw", "GdWo", "MnPrv","GdPrv"])
 df["Electrical"] = ordinal(df["Electrical"], category = ['SBrkr', 'FuseA', 'FuseF', 'FuseP', 'Mix'])
 print("Ordinal transformation completed!" )<count_values>
xrf_pipe = Pipeline(
 steps = [ 
('classifier', ExtraTreesClassifier(n_estimators=500,random_state=0))
] 
)
 param_grid = {
'classifier__min_samples_leaf': [1,4,7,8],
'classifier__max_depth': [31,34,36,39],
}
xrf_grid_search = GridSearchCV(xrf_pipe, param_grid, cv=5, refit='True', n_jobs=-1)
 xrf_grid_search.fit(X_train, y_train)
 print(xrf_grid_search.best_score_)
 print(xrf_grid_search.best_params_)
for i in ["Street", "Utilities", "LandSlope", "MiscFeature"]:
print(df[i].value_counts() )<create_dataframe>
xrf_model = xrf_grid_search.best_estimator_
cv_score = cross_val_score(xrf_model, X_train, y_train, cv = 5)
 print(cv_score)
 print(( cv_score.mean() , cv_score.std() * 2))
def cat_eda(data, cat_length, target = "SalePrice"):
dataframe = data.copy()
more_cat_cols = [col for col in dataframe.columns if len(dataframe[col].unique())< cat_length]
for i in more_cat_cols:
print(i, ":", len(dataframe[i].value_counts()), "Unique Category -", str(dataframe[i].dtype))
print(pd.DataFrame({"COUNT": dataframe[i].value_counts() ,
"RATIO": dataframe[i].value_counts() / len(dataframe),
"TARGET_MEDIAN": dataframe.groupby(i)[target].median() ,
"TARGET_COUNT": dataframe.groupby(i)[target].count() ,
"TARGET_STD": dataframe.groupby(i)[target].std() }), end="
")
 print("
print("Object Variables:",dataframe[more_cat_cols].select_dtypes("object" ).columns.tolist() , "
")
 print("Integer Variables:",dataframe[more_cat_cols].select_dtypes("integer" ).columns.tolist() , "
")
 print("Float Variables:",dataframe[more_cat_cols].select_dtypes("float" ).columns.tolist() , "
")
 cat_eda(df, cat_length=17 )<drop_column>
rf_pipe = Pipeline(
 steps = [ 
('classifier', RandomForestClassifier(n_estimators=500,random_state=0))
] 
)
 param_grid = {
'classifier__min_samples_leaf': [1,4,7,8],
'classifier__max_depth': [31,34,36,39],
}
np.random.seed(1)
 rf_grid_search = GridSearchCV(rf_pipe, param_grid, cv=5, refit='True', n_jobs=-1)
 rf_grid_search.fit(X_train, y_train)
 print(rf_grid_search.best_score_)
 print(rf_grid_search.best_params_)
drop_list = ["Street", "Utilities", "LandSlope", "MiscFeature", "PoolArea"]
for col in drop_list:
df.drop(col, axis=1, inplace=True )<feature_engineering>
rf_model = rf_grid_search.best_estimator_
cv_score = cross_val_score(rf_model, X_train, y_train, cv = 5)
 print(cv_score)
 print(( cv_score.mean() , cv_score.std() * 2))
df2["TotalQual"] = df2[["OverallQual", "OverallCond", "ExterQual", "ExterCond", "BsmtCond", "BsmtFinType1",
"BsmtFinType2", "HeatingQC", "KitchenQual", "Functional", "FireplaceQu", "GarageQual", "GarageCond", "Fence"]].sum(axis = 1)
 df2["BsmtQual"] = df2[["BsmtCond", "BsmtFinType1", "BsmtFinType2"]].sum(axis = 1)
 df2["TotalGarageQual"] = df2[["GarageQual", "GarageCond"]].sum(axis = 1)
 df2["Overall"] = df2[["OverallQual", "OverallCond"]].sum(axis = 1)
 df2["Exter"] = df2[["ExterQual", "ExterCond"]].sum(axis = 1)
 df2["ExtraQual"] = df2[["Fence", "FireplaceQu", "Functional", "HeatingQC"]].sum(axis = 1)
 df2["Qual"] = df2[["OverallQual", "ExterQual", "GarageQual", "Fence", "BsmtFinType1", "BsmtFinType2", "KitchenQual", "FireplaceQu"]].sum(axis = 1)
 df2["Cond"] = df2[["OverallCond", "ExterCond", "GarageCond", "BsmtCond", "HeatingQC", "Functional"]].sum(axis = 1 )<concatenate>
xgd_pipe = Pipeline(
 steps = [ 
('classifier', XGBClassifier(n_estimators=50, subsample=0.5))
] 
)
 param_grid = {
'classifier__learning_rate' : [0.45],
'classifier__min_samples_split' : [8, 16, 32],
'classifier__min_samples_leaf' : [2],
'classifier__max_depth': [15]
}
np.random.seed(1)
 xgd_grid_search = GridSearchCV(xgd_pipe, param_grid, cv=5,
refit='True', verbose = 10, n_jobs=-1)
 xgd_grid_search.fit(X_train, y_train)
 print(xgd_grid_search.best_score_)
 print(xgd_grid_search.best_params_)
cat_eda(df2[ordinal_vars+["SalePrice"]], cat_length=17 )<feature_engineering>
xgd_model = xgd_grid_search.best_estimator_
cv_score = cross_val_score(xgd_model, X_train, y_train, cv = 5)
 print(cv_score)
 print(( cv_score.mean() , cv_score.std() * 2))
df2["LotShape"] = np.where(df2.LotShape == 1, 2, df2["LotShape"])
 df2["OverallQual"] = np.where(df2.OverallQual.isin([1,2]), 3, df2["OverallQual"])
 df2["OverallCond"] = np.where(df2.OverallCond.isin([1,2]), 3, df2["OverallCond"])
 df2["OverallCond"] = np.where(df2.OverallCond.isin([7,8]), 6, df2["OverallCond"])
 df2["ExterCond"] = np.where(df2.ExterCond.isin([1,2]), 0, 1)
 df2["BsmtQual"] = np.where(df2.BsmtQual.isin([3,6]), 7, df2["BsmtQual"])
 df2["BsmtQual"] = np.where(df2.BsmtQual.isin([17,18]), 16, df2["BsmtQual"])
 df2["BsmtQual"] = np.where(df2.BsmtQual.isin([9,10,11]), 12, df2["BsmtQual"])
 df2["BsmtCond"] = np.where(df2.BsmtCond == 1, 2, df2["BsmtCond"])
 df2["HeatingQC"] = np.where(df2.HeatingQC == 1, 2, df2["HeatingQC"])
 df2["Functional"] = np.where(df2.Functional == 8, 1, 0)
 df2["GarageQual"] = np.where(df2.GarageQual == 1, 2, df2["GarageQual"])
 df2["GarageQual"] = np.where(df2.GarageQual == 6, 5, df2["GarageQual"])
 df2["GarageCond"] = np.where(df2.GarageCond == 1, 2, df2["GarageCond"])
 df2["GarageCond"] = np.where(df2.GarageCond == 6, 5, df2["GarageCond"])
 df2["Fence"] = np.where(df2.Fence.isin([2,4]), 3, df2["Fence"])
 df2["Electrical"] = np.where(df2["Electrical"].isin([5, 4]), 3, df2["Electrical"] )<concatenate>
test_pred = xrf_grid_search.predict(X_test )
cat_eda(df2[ordinal_vars+["SalePrice"]], cat_length=17 )<feature_engineering>
submission = pd.DataFrame({
'Id':test_id,
'Cover_Type':test_pred
})
 submission.head()
<feature_engineering><EOS> 
submission.to_csv('my_submission.csv', index=False )