Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<feature_engineering>
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse)
all_df.drop('agesq', axis=1, inplace=True)
 all_df['edjefe'] =(all_df['edjefe'] == all_df['edjefe'].max())* 1
all_df['edjefa'] =(all_df['edjefa'] == all_df['edjefa'].max())* 1
all_df.shape<feature_engineering>
pd.set_option('display.max_columns', 500 )
all_df['age'] = sc.fit_transform(all_df['age'].values.reshape(( -1, 1)))
 all_df['idhogar'] = sc.fit_transform(all_df['idhogar'].values.reshape(( -1, 1)) )<drop_column>
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 sample = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )
cat_cols = [col for col in int_cols if col not in [col for col in int_cols if col[:3]=='SQB']]
cat_cols.remove('agesq')
 cat_cols.remove('age' )<categorify>
all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'],test.loc[:,'MSSubClass':'SaleCondition']))
onehot_cols = [col for col in cat_cols if len(all_df[col].unique())> 2]
onehot_arr = onehot.fit_transform(all_df[onehot_cols].values)
 onehot_arr.shape<drop_column>
train["SalePrice"] = np.log1p(train["SalePrice"])
 numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
all_df.drop(onehot_cols, axis=1, inplace=True)
 all_df.shape<feature_engineering>
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )
train_df['Target'] = train_df['Target'].apply(lambda x: x-1 )<prepare_x_and_y>
all_data = pd.get_dummies(all_data)
 all_data = all_data.fillna(all_data.mean() )
tr_cols = [col for col in train_df.columns if col not in ['Id', 'Target']]
X_train = train_df[tr_cols].values
X_train = np.concatenate(( X_train, onehot_arr[:9557, :]), axis=1)
 y_train = train_df['Target'].values
skf = StratifiedKFold(n_splits=5, shuffle=True )<load_pretrained>
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice
class_weights = compute_class_weight('balanced', np.sort(train_df['Target'].unique()), train_df['Target'].values)
 n_rounds = 1000
learning_rate = 0.2
max_depth = 5
l2_reg = 2
n_classes = 4
early_stopping_rounds = 20<concatenate>
model_ridge = Ridge()
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in [0.05, 0.1, 0.3, 1, 3, 5, 30, 50, 75]]
cv_ridge = pd.Series(cv_ridge, index = [0.05, 0.1, 0.3, 1, 3, 5, 30, 50, 75])
 print('ridge test score ',cv_ridge.min())
 print("alpha: ",cv_ridge.sort_values().index[0] )
test_arr = test_df[[col for col in test_df.columns if col not in ['Id', 'Target']]].values
test_arr = np.concatenate(( test_arr, onehot_arr[9557:, :]), axis=1 )<train_model>
dtrain = xgb.DMatrix(X_train, label = y)
 dtest = xgb.DMatrix(X_test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100 )
xgb_params = [('eta', learning_rate),('max_depth', max_depth),('colsample_bytree', 0.8),('lambda', l2_reg),('objective', 'multi:softprob'),('num_class', n_classes),('eval_metric', 'mlogloss'),('silent', 1)]
for tr_idx, val_idx in skf.split(X_train, y_train):
X_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
X_val, y_val = X_train[val_idx], y_train[val_idx]
dtrain = xgb.DMatrix(X_tr, y_tr, weight=[class_weights[int(y_tr[i])] for i in range(y_tr.shape[0])])
 dval = xgb.DMatrix(X_val, y_val, weight=[class_weights[int(y_val[i])] for i in range(y_val.shape[0])])
 eval_list = [(dtrain, 'train'),(dval, 'validation')]
bst = xgb.train(xgb_params, dtrain, n_rounds, eval_list, early_stopping_rounds=early_stopping_rounds, verbose_eval=True )<save_to_csv>
model_ridge = RandomForestRegressor()
cv_rf = [rmse_cv(RandomForestRegressor(n_estimators = estimators)).mean()
for estimators in [1,10,20,40,100]]
cf_rf = pd.Series(cv_ridge, index = [1,10,20,40,100])
 print('random forest score ',cf_rf.min())
 print("number of estimators: ",cf_rf.sort_values().index[0] )
dtest = xgb.DMatrix(test_arr, weight=class_weights)
 xgb_preds = bst.predict(dtest)
 xgb_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 xgb_sample_subm['Target'] = np.argmax(xgb_preds, axis=1 ).astype(int)
 xgb_sample_subm['Target'] = xgb_sample_subm['Target'].apply(lambda x: x+1)
 xgb_sample_subm.to_csv('xgb_preds.csv', index=False)
 sns.countplot(xgb_sample_subm['Target'])
 plt.show()<split>
rf = RandomForestRegressor(n_estimators=100)
 rf.fit(X_train,y )
lgb_params = {'objective': 'multiclass', 'num_class': n_classes, 'learning_rate': learning_rate, 'num_leaves': 31, 'num_thread': 4, 'max_depth': max_depth, 'feature_fraction': 0.8, 'lambda_l2': l2_reg}
for tr_idx, val_idx in skf.split(X_train, y_train):
X_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
X_val, y_val = X_train[val_idx], y_train[val_idx]
dtrain = lgb.Dataset(X_tr, y_tr, weight=[class_weights[int(y_tr[i])] for i in range(y_tr.shape[0])])
 dval = lgb.Dataset(X_val, y_val, weight=[class_weights[int(y_val[i])] for i in range(y_val.shape[0])])
 bst = lgb.train(lgb_params, dtrain, n_rounds, valid_sets = [dval, dtrain], valid_names = ['validation', 'train'], early_stopping_rounds = early_stopping_rounds, verbose_eval=False )<save_to_csv>
param_grid = {
'max_depth': [80, 90,],
'max_features': [2, 3],
'min_samples_leaf': [3, 4, 5],
'min_samples_split': [8, 10, 12],
'n_estimators': [100, 200]
}
grid_search = GridSearchCV(estimator = GradientBoostingRegressor() , scoring="neg_mean_squared_error", param_grid = param_grid,
cv = 3, n_jobs = -1, verbose = 2)
 grid_search.fit(X_train, y )
lgb_preds = bst.predict(test_arr)
 lgb_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 lgb_sample_subm['Target'] = np.argmax(lgb_preds, axis=1 ).astype(int)
 lgb_sample_subm['Target'] = lgb_sample_subm['Target'].apply(lambda x: x+1)
 lgb_sample_subm.to_csv('lgb_preds.csv', index=False)
 sns.countplot(lgb_sample_subm['Target'])
 plt.show()<train_model>
pd.DataFrame(grid_search.cv_results_)['mean_test_score'].mean() *(-1 )
cat_model = CatBoostClassifier(iterations=n_rounds, learning_rate=learning_rate, depth=max_depth, loss_function='MultiClass', classes_count=n_classes, logging_level='Silent', l2_leaf_reg=l2_reg, thread_count=4, class_weights=class_weights)
 for tr_idx, val_idx in skf.split(X_train, y_train):
X_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
X_val, y_val = X_train[val_idx], y_train[val_idx]
cat_model.fit(X_tr, y_tr, use_best_model=True, eval_set=(X_val, y_val), early_stopping_rounds=early_stopping_rounds )<save_to_csv>
gradient_boost_preds = np.expm1(grid_search.best_estimator_.predict(X_test))
rf_preds = np.expm1(rf.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))
cat_preds = cat_model.predict_proba(test_arr)
 cat_sample_subm = pd.read_csv('.. /input/sample_submission.csv')
 cat_sample_subm['Target'] = np.argmax(cat_preds, axis=1 ).astype(int)
 cat_sample_subm['Target'] = cat_sample_subm['Target'].apply(lambda x: x+1)
 cat_sample_subm.to_csv('cat_preds.csv', index=False)
 sns.countplot(cat_sample_subm['Target'])
 plt.show()<save_to_csv>
preds = 0.45*gradient_boost_preds +0.1*rf_preds + 0.45*lasso_preds
kaggle_solution = pd.DataFrame({"id":test.Id, "SalePrice":preds})
 kaggle_solution.to_csv("test_pred.csv", index = False )