Kaggle competition: Forest Cover Type Prediction
<SOS> metric: CategorizationAccuracy Kaggle data source: forest-cover-type-prediction<prepare_x_and_y>
%matplotlib inline
indexes = x_train[(df_train['GrLivArea']>4000)&(df_train['SalePrice']<300000)].index
x_train = x_train.drop(indexes)
 y_train = np.delete(y_train, indexes )<import_modules>
train = pd.read_csv('.. /input/forest-cover-type-prediction/train.csv',
index_col='Id')
 test = pd.read_csv('.. /input/forest-cover-type-prediction/test.csv',
index_col='Id' )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb<choose_model_class>
train['Cover_Type'].value_counts()
model_xgb = xgb.XGBRegressor(n_estimators=2200 )<compute_train_metric>
def write_to_submission_file(predicted_labels, out_file,
target='Cover_Type', index_label="Id", init_index=15121):
predicted_df = pd.DataFrame(predicted_labels,
index = np.arange(init_index,
predicted_labels.shape[0] + init_index),
columns=[target])
 predicted_df.to_csv(out_file, index_label=index_label )
n_folds = 5
def rmsle(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
X_train, X_valid, y_train, y_valid = train_test_split(
 train.drop('Cover_Type', axis=1), train['Cover_Type'],
test_size=0.3, random_state=17 )
model_xgb.fit(x_train, y_train)
 xgb_train_pred = model_xgb.predict(x_train)
 xgb_pred = model_xgb.predict(df_test)
 print(rmsle(y_train, xgb_train_pred))<save_to_csv>
logit = LogisticRegression(C=1, solver='lbfgs', max_iter=500,
random_state=17, n_jobs=4,
multi_class='multinomial')
 logit_pipe = Pipeline([('scaler', StandardScaler()), 
('logit', logit)] )
sub = pd.DataFrame()
sub['Id'] = ids
sub['SalePrice'] = xgb_pred
sub.to_csv('submission.csv',index=False )<choose_model_class>
%%time
logit_pipe.fit(X_train, y_train )
model_xgb = xgb.XGBRegressor(reg_lambda=0.8571, n_estimators=2200, nthread = -1 )<predict_on_test>
logit_val_pred = logit_pipe.predict(X_valid )
model_xgb.fit(x_train, y_train)
 xgb_train_pred = model_xgb.predict(x_train)
 xgb_pred = model_xgb.predict(df_test)
 print(rmsle(y_train, xgb_train_pred))<train_on_grid>
accuracy_score(y_valid, logit_val_pred )
params = {'min_child_weight':[4,5], 'gamma':[i/10.0 for i in range(3,6)], 'subsample':[i/10.0 for i in range(6,11)],
'colsample_bytree':[i/10.0 for i in range(4,11)], 'max_depth': [2,3,4], 'reg_lambda':[i/10.0 for i in range(7,9)], 'reg_alpha':[i/10.0 for i in range(4,7)]}
model = xgb.XGBRegressor(nthread=-1)
 grid = GridSearchCV(model, params)
 grid.fit(x_train, y_train )<predict_on_test>
first_forest = RandomForestClassifier(
 n_estimators=100, random_state=17, n_jobs=4 )
xgb_train_pred = grid.best_estimator_.predict(x_train)
 xgb_pred = grid.best_estimator_.predict(df_test)
 print(rmsle(y_train, xgb_train_pred))<train_model>
%%time
first_forest.fit(X_train, y_train )
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<choose_model_class>
forest_val_pred = first_forest.predict(X_valid )
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, random_state =42)
 ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005,random_state=42))
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=42))
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso )<train_model>
accuracy_score(y_valid, forest_val_pred )
model_xgb = xgb.XGBRegressor(n_estimators=2200, nthread = -1)
 model_xgb.fit(x_train, y_train)
 model_lgb = lgb.LGBMRegressor(objective='regression',n_estimators=720)
 model_lgb.fit(x_train, y_train)
 stacked_averaged_models.fit(x_train.values, y_train )<predict_on_test>
pd.DataFrame(first_forest.feature_importances_,
index=X_train.columns, columns=['Importance'] ).sort_values(
 by='Importance', ascending=False)[:10]
lgb_pred = model_lgb.predict(df_test)
 xgb_pred = model_xgb.predict(df_test)
 stacked_pred = stacked_averaged_models.predict(df_test.values )<define_variables>
lgb_clf = LGBMClassifier(random_state=17 )
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15<save_to_csv>
%%time
lgb_clf.fit(X_train, y_train )
sub = pd.DataFrame()
sub['Id'] = ids
sub['SalePrice'] = xgb_pred
sub.to_csv('submission.csv',index=False )<load_from_csv>
accuracy_score(y_valid, lgb_clf.predict(X_valid))
train = pd.read_csv('.. /input/ames-housing-dataset/AmesHousing.csv')
 train.drop(['PID'], axis=1, inplace=True)
 origin = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.columns = origin.columns
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 print('Train:{} Test:{}'.format(train.shape,test.shape))
missing = test.isnull().sum()
missing = missing[missing>0]
train.drop(missing.index, axis=1, inplace=True)
 train.drop(['Electrical'], axis=1, inplace=True)
 test.dropna(axis=1, inplace=True)
 test.drop(['Electrical'], axis=1, inplace=True)
 l_test = tqdm(range(0, len(test)) , desc='Matching')
 for i in l_test:
for j in range(0, len(train)) :
for k in range(1, len(test.columns)) :
if test.iloc[i,k] == train.iloc[j,k]:
continue
else:
break
else:
submission.iloc[i, 1] = train.iloc[j, -1]
break
l_test.close()
submission.to_csv('result-with-best.csv', index=False )<import_modules>
param_grid = {'num_leaves': [7, 15, 31, 63],
'max_depth': [3, 4, 5, 6, -1]}
warnings.filterwarnings('ignore' )<data_type_conversions>
grid_searcher = GridSearchCV(estimator=lgb_clf, param_grid=param_grid,
cv=5, verbose=1, n_jobs=4 )
def prepare_df(df, cols_raw, cols_drop):
result = df[cols_raw].copy()
if len(cols_drop)> 0:
result.drop(cols_drop, axis=1, inplace=True)
 return result<load_from_csv>
grid_searcher.fit(X_train, y_train )
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv',na_filter=False)
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv',na_filter=False )<define_variables>
grid_searcher.best_params_, grid_searcher.best_score_
object_c = [
'MSZoning', 'Street', 'Alley',
'LotShape', 'LandContour', 'Utilities', 'LotConfig',
'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl',
'Exterior1st', 'Exterior2nd', 'MasVnrType',
'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure',
'BsmtFinType1', 'BsmtFinType2', 'Heating',
'HeatingQC', 'CentralAir', 'Electrical',
'KitchenQual', 'Functional', 'FireplaceQu',
'GarageType', 'GarageFinish', 'GarageQual',
'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature',
'SaleType', 'SaleCondition'
]
numeric_c = [
'LotArea', 'YearBuilt', 'YearRemodAdd',
'BsmtUnfSF', 'TotalBsmtSF', 'LowQualFinSF', 'GrLivArea',
'BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces',
'GarageArea', 'LotFrontage', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', '1stFlrSF', '2ndFlrSF', 'GarageYrBlt', 'GarageCars',
'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal',
'OverallCond', 'OverallQual', 'MSSubClass', 'MoSold', 'YrSold'
]
convert_f = ['LotFrontage', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'GarageYrBlt', 'BsmtFullBath', 'BsmtHalfBath', 'GarageCars', 'GarageArea']<split>
accuracy_score(y_valid, grid_searcher.predict(X_valid))
y_col = 'SalePrice'
cols_raw = object_c.copy()
cols_raw.extend(numeric_c)
 cols_raw_y = cols_raw.copy()
cols_raw_y.extend([y_col])
 categorical_cols_l, _ = split_cat_cols(train, test, object_c)
 categorical_cols_oh = object_c
numerical_cols = numeric_c
count_enc_cols = []
drop = []<categorify>
num_iterations = 200
lgb_clf2 = LGBMClassifier(random_state=17, max_depth=-1,
num_leaves=63, n_estimators=num_iterations,
n_jobs=1)
 param_grid2 = {'learning_rate': np.logspace(-3, 0, 10)}
grid_searcher2 = GridSearchCV(estimator=lgb_clf2, param_grid=param_grid2,
cv=5, verbose=1, n_jobs=4)
 grid_searcher2.fit(X_train, y_train)
 print(grid_searcher2.best_params_, grid_searcher2.best_score_)
 print(accuracy_score(y_valid, grid_searcher2.predict(X_valid)) )
type_converter = FunctionTransformer(func=convert_df_types, kw_args={'target': [(convert_f, 'numeric')]})
 numerical_transformer = SimpleImputer(strategy='mean')
 steps = []
if count_enc_cols:
count_encoder = ce.CountEncoder(cols=count_enc_cols)
 steps.append(( 'count_enc', count_encoder))
steps.append(( 'imputer', SimpleImputer(strategy='most_frequent')))
 steps.append(( 'label', ce.BinaryEncoder()))
categorical_transformer_l = Pipeline(steps=steps)
 steps = []
if count_enc_cols:
count_encoder = ce.CountEncoder(cols=count_enc_cols)
 steps.append(( 'count_enc', count_encoder))
steps.append(( 'imputer', SimpleImputer(strategy='most_frequent')))
 steps.append(( 'onehot', OneHotEncoder(handle_unknown='ignore', sparse=False)))
 categorical_transformer_oh = Pipeline(steps=steps)
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat_l', categorical_transformer_l, categorical_cols_l), 
('cat_oh', categorical_transformer_oh, categorical_cols_oh)
 ])
 drop_cols = []
feature_eng = FunctionTransformer(func=prepare_df, kw_args={'cols_raw': cols_raw, 'cols_drop': drop_cols} )<choose_model_class>
final_lgb = LGBMClassifier(n_estimators=200, num_leaves=63,
learning_rate=0.2, max_depth=-1,
n_jobs=4 )
final_layer = StackingRegressor(
 estimators=[('rf', RandomForestRegressor(random_state=42)) , 
('gbrt', GradientBoostingRegressor(random_state=42)) ],
final_estimator=RidgeCV() 
)
 model = StackingRegressor(
 estimators=[ 
('x', xgb.XGBRegressor(n_estimators=5000, learning_rate=0.01, max_depth=4, n_jobs=4, random_state=0)) , 
('lasso', LassoCV(random_state=42)) , 
('svr', SVR(kernel='rbf')) ],
final_estimator=final_layer 
)
 my_pipeline = Pipeline(
 steps=[ 
('type_conv', type_converter), 
('preprocessor', preprocessor), 
('scaler', PowerTransformer()), 
('model', model)
 ] 
)
 _ = my_pipeline.fit(train[cols_raw], train[y_col] )<save_to_csv>
%%time
final_lgb.fit(train.drop('Cover_Type', axis=1), train['Cover_Type'] )
output = pd.DataFrame({'Id': test.Id, 'SalePrice': my_pipeline.predict(test)})
 output.to_csv('result28.csv', index=False)
 print("Your submission was successfully saved!" )<load_from_csv>
%%time
lgb_final_pred = final_lgb.predict(test )
train_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 print(f'train shape : {train_data.shape} , test data : {test_data.shape}' )<prepare_x_and_y>
write_to_submission_file(lgb_final_pred,
'lgb_forest_cover_type.csv' )
x = train_data.iloc[:,0:80]
y = train_data.iloc[:,-1]
df = pd.concat([x,test_data],axis = 0)
 df = df.drop(['Id'], axis=1 )<drop_column>
pd.DataFrame(final_lgb.feature_importances_,
index=X_train.columns, columns=['Importance'] ).sort_values(
 by='Importance', ascending=False)[:10]