Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<save_to_csv>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
knn=KNeighborsRegressor(n_neighbors=5)
 knn.fit(train_x, train_y)
 predicted_test_values2 = knn.predict(test2)
 submission2 = pd.DataFrame(columns=['Id','Prediction'])
 submission2['Id'] = test['Id']
submission2['Prediction'] = predicted_test_values2
submission2.to_csv('submission_KNN.csv',index=False )<load_from_csv>
df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df.head()
df = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/train.csv.zip')
 df.shape<load_from_csv>
test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test_df.head()
test_df = pd.read_csv('/kaggle/input/restaurant-revenue-prediction/test.csv.zip')
 test_df.shape<count_missing_values>
df.isnull().sum()
df.isnull().sum().sort_index() /len(df )<count_unique_values>
columns=['MSZoning','Street','LotShape','LandContour','Utilities','LotConfig',
'Condition2','BldgType','Condition1','HouseStyle','SaleType',
'SaleCondition','ExterCond','ExterQual','Foundation','BsmtQual',
'BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2',
'RoofStyle','RoofMatl','Exterior1st','Exterior2nd','MasVnrType','Heating','HeatingQC',
'CentralAir', 'Electrical','KitchenQual','Functional','GarageType',
'GarageFinish','GarageQual','GarageCond','PavedDrive','LandSlope','Neighborhood']
len(columns )
( df['City'].nunique() , test_df['City'].nunique() )<feature_engineering>
final_df = pd.concat([df,test_df],axis=0 )
test_df.loc[test_df['Type']=='MB', 'Type'] = 'DT'<drop_column>
final_df = pd.get_dummies(final_df, columns=columns, drop_first=True)
 final_df
df.drop('City', axis=1, inplace=True)
 test_df.drop('City', axis=1, inplace=True )<feature_engineering>
final_df = final_df.loc[:,~final_df.columns.duplicated() ]
df.drop('Id',axis=1,inplace=True)
 df['Open Date'] = pd.to_datetime(df['Open Date'])
 test_df['Open Date'] = pd.to_datetime(test_df['Open Date'])
 launch_date = datetime.datetime(2015, 3, 23)
 df['Days Open'] =(launch_date - df['Open Date'] ).dt.days / 1000
test_df['Days Open'] =(launch_date - test_df['Open Date'] ).dt.days / 1000
df.drop('Open Date', axis=1, inplace=True)
 test_df.drop('Open Date', axis=1, inplace=True )<feature_engineering>
final_df.drop_duplicates(inplace=True )
<feature_engineering>
df=final_df.iloc[:1458,:]
test_df=final_df.iloc[1458:,:]
test_df.drop(['SalePrice'],axis=1,inplace=True )
imp_train = IterativeImputer(max_iter=30, missing_values=0, sample_posterior=True, min_value=1, random_state=37)
 imp_test = IterativeImputer(max_iter=30, missing_values=0, sample_posterior=True, min_value=1, random_state=23)
 p_data = ['P'+str(i)for i in range(1,38)]
df[p_data] = np.round(imp_train.fit_transform(df[p_data]))
test_df[p_data] = np.round(imp_test.fit_transform(test_df[p_data]))<categorify>
df[(np.abs(stats.zscore(df)) <3 ).all(axis=1)]
columnsToEncode = df.select_dtypes(include=[object] ).columns
df = pd.get_dummies(df, columns=columnsToEncode, drop_first=False)
 test_df = pd.get_dummies(test_df, columns=columnsToEncode, drop_first=False )<split>
def getCorrelatedFeature(corrdata, threshold):
feature = []
value = []
for i, index in enumerate(corrdata.index):
if abs(corrdata[index])> threshold:
feature.append(index)
 value.append(corrdata[index])
 corrdf = pd.DataFrame(data = value, index = feature, columns=['corr value'])
 return corrdf
df['revenue'] = np.log1p(df['revenue'])
 X, y = df.drop('revenue', axis=1), df['revenue']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=118 )<import_modules>
corr_value = getCorrelatedFeature(corrmat['SalePrice'], threshold)
 corr_value
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso<train_on_grid>
X = df.drop(labels=['SalePrice'], axis = 1)
 y = df['SalePrice']
y= np.log1p(y )
params_ridge = {
'alpha' : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
'fit_intercept' : [True, False],
'normalize' : [True,False],
'solver' : ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga']
}
ridge_model = Ridge()
ridge_regressor = GridSearchCV(ridge_model, params_ridge, scoring='neg_root_mean_squared_error', cv=5, n_jobs=-1)
 ridge_regressor.fit(X_train, y_train)
 print(f'Optimal alpha: {ridge_regressor.best_params_["alpha"]:.2f}')
 print(f'Optimal fit_intercept: {ridge_regressor.best_params_["fit_intercept"]}')
 print(f'Optimal normalize: {ridge_regressor.best_params_["normalize"]}')
 print(f'Optimal solver: {ridge_regressor.best_params_["solver"]}')
 print(f'Best score: {ridge_regressor.best_score_}' )<compute_train_metric>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42 )
ridge_model = Ridge(alpha=ridge_regressor.best_params_["alpha"], fit_intercept=ridge_regressor.best_params_["fit_intercept"],
normalize=ridge_regressor.best_params_["normalize"], solver=ridge_regressor.best_params_["solver"])
 ridge_model.fit(X_train, y_train)
 y_train_pred = ridge_model.predict(X_train)
 y_pred = ridge_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<train_on_grid>
warnings.filterwarnings('ignore', category=FutureWarning)
 warnings.filterwarnings('ignore', category=DeprecationWarning )
params_lasso = {
'alpha' : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
'fit_intercept' : [True, False],
'normalize' : [True,False],
}
lasso_model = Lasso()
lasso_regressor = GridSearchCV(lasso_model, params_lasso, scoring='neg_root_mean_squared_error', cv=5, n_jobs=-1)
 lasso_regressor.fit(X_train, y_train)
 print(f'Optimal alpha: {lasso_regressor.best_params_["alpha"]:.2f}')
 print(f'Optimal fit_intercept: {lasso_regressor.best_params_["fit_intercept"]}')
 print(f'Optimal normalize: {lasso_regressor.best_params_["normalize"]}')
 print(f'Best score: {lasso_regressor.best_score_}' )<compute_test_metric>
def print_results(results):
print('BEST PARAMS: {}\
'.format(results.best_params_))
means = results.cv_results_['mean_test_score']
stds = results.cv_results_['std_test_score']
for mean, std, params in zip(means, stds, results.cv_results_['params']):
print('{}(+/-{})for {}'.format(round(mean, 3), round(std * 2, 3), params))
lasso_model = Lasso(alpha=lasso_regressor.best_params_["alpha"], fit_intercept=lasso_regressor.best_params_["fit_intercept"],
normalize=lasso_regressor.best_params_["normalize"])
 lasso_model.fit(X_train, y_train)
 y_train_pred = lasso_model.predict(X_train)
 y_pred = lasso_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<train_on_grid>
lr = LinearRegression()
lr.fit(X_train, y_train)
 lr.score(X_train, y_train )
el_model = ElasticNetCV(l1_ratio=[.1,.5,.7,.9,.95,.99, 1], eps=5e-2, cv=10, n_jobs=-1)
 el_model.fit(X_train, y_train)
 print(f'Optimal alpha: {el_model.alpha_:.6f}')
 print(f'Optimal l1_ratio: {el_model.l1_ratio_:.3f}')
 print(f'Number of iterations {el_model.n_iter_}' )<compute_test_metric>
sc = StandardScaler()
sc.fit(X_train)
 X_scaled = sc.transform(X_train )
y_train_pred = el_model.predict(X_train)
 y_pred = el_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<train_on_grid>
ridge = Ridge()
ridge.fit(X_scaled, y_train)
 ridge.score(X_scaled, y_train )
params_knn = {
'n_neighbors' : [3, 5, 7, 9, 11],
}
knn_model = KNeighborsRegressor()
knn_regressor = GridSearchCV(knn_model, params_knn, scoring='neg_root_mean_squared_error', cv=10, n_jobs=-1)
 knn_regressor.fit(X_train, y_train)
 print(f'Optimal neighbors: {knn_regressor.best_params_["n_neighbors"]}')
 print(f'Best score: {knn_regressor.best_score_}' )<compute_test_metric>
parameters = {
'solver': ['cholesky', 'auto'],
'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]
}
cv = GridSearchCV(ridge, parameters, cv=5)
 cv.fit(X_train, y_train)
 print_results(cv)
 print(cv.best_estimator_)
 joblib.dump(cv.best_estimator_, '.. /.. /.. /ridge_model.pkl' )
knn_model = KNeighborsRegressor(n_neighbors=knn_regressor.best_params_["n_neighbors"])
 knn_model.fit(X_train, y_train)
 y_train_pred = knn_model.predict(X_train)
 y_pred = knn_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<train_on_grid>
lasso = Lasso()
parameters = {
'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]
}
cv = GridSearchCV(lasso, parameters, cv=5)
 cv.fit(X_train, y_train)
 print_results(cv)
 print('best estimator:', cv.best_estimator_)
 joblib.dump(cv.best_estimator_, '.. /.. /.. /lasso_model.pkl' )
params_rf = {
'max_depth': [10, 30, 35, 50, 65, 75, 100],
'max_features': [.3,.4,.5,.6],
'min_samples_leaf': [3, 4, 5],
'min_samples_split': [8, 10, 12],
'n_estimators': [30, 50, 100, 200]
}
rf = RandomForestRegressor()
rf_regressor = GridSearchCV(rf, params_rf, scoring='neg_root_mean_squared_error', cv = 10, n_jobs = -1)
 rf_regressor.fit(X_train, y_train)
 print(f'Optimal depth: {rf_regressor.best_params_["max_depth"]}')
 print(f'Optimal max_features: {rf_regressor.best_params_["max_features"]}')
 print(f'Optimal min_sample_leaf: {rf_regressor.best_params_["min_samples_leaf"]}')
 print(f'Optimal min_samples_split: {rf_regressor.best_params_["min_samples_split"]}')
 print(f'Optimal n_estimators: {rf_regressor.best_params_["n_estimators"]}')
 print(f'Best score: {rf_regressor.best_score_}' )<choose_model_class>
en = ElasticNet(l1_ratio=0.5)
 parameters = {
'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]
}
cv = GridSearchCV(en, parameters, cv=5)
 cv.fit(X_train, y_train)
 print_results(cv)
 print('best estimator:', cv.best_estimator_)
 joblib.dump(cv.best_estimator_, '.. /.. /.. /en_model.pkl' )
rf_model = RandomForestRegressor(max_depth=rf_regressor.best_params_["max_depth"],
max_features=rf_regressor.best_params_["max_features"],
min_samples_leaf=rf_regressor.best_params_["min_samples_leaf"],
min_samples_split=rf_regressor.best_params_["min_samples_split"],
n_estimators=rf_regressor.best_params_["n_estimators"],
n_jobs=-1, oob_score=True)
 rf_model.fit(X_train, y_train)
 y_train_pred = rf_model.predict(X_train)
 y_pred = rf_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<train_on_grid>
dtr = DecisionTreeRegressor(max_depth = 2, min_samples_leaf=10)
 dtr.fit(X_scaled, y_train)
 dtr.score(X_scaled, y_train )
params_lgbm = {
'learning_rate': [.01,.1,.5,.7,.9,.95,.99, 1],
'boosting': ['gbdt'],
'metric': ['l1'],
'feature_fraction': [.3,.4,.5, 1],
'num_leaves': [20],
'min_data': [10],
'max_depth': [10],
'n_estimators': [10, 30, 50, 100]
}
lgb = lgbm.LGBMRegressor()
lgb_regressor = GridSearchCV(lgb, params_lgbm, scoring='neg_root_mean_squared_error', cv = 10, n_jobs = -1)
 lgb_regressor.fit(X_train, y_train)
 print(f'Optimal lr: {lgb_regressor.best_params_["learning_rate"]}')
 print(f'Optimal feature_fraction: {lgb_regressor.best_params_["feature_fraction"]}')
 print(f'Optimal n_estimators: {lgb_regressor.best_params_["n_estimators"]}')
 print(f'Best score: {lgb_regressor.best_score_}' )<train_model>
gbr = GradientBoostingRegressor(max_depth=2, n_estimators=3, learning_rate=0.1)
 gbr.fit(X_train, y_train)
 gbr.score(X_train, y_train )
lgb_model = lgbm.LGBMRegressor(learning_rate=lgb_regressor.best_params_["learning_rate"], boosting='gbdt',
metric='l1', feature_fraction=lgb_regressor.best_params_["feature_fraction"],
num_leaves=20, min_data=10, max_depth=10,
n_estimators=lgb_regressor.best_params_["n_estimators"], n_jobs=-1)
 lgb_model.fit(X_train, y_train)
 y_train_pred = lgb_model.predict(X_train)
 y_pred = lgb_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<train_on_grid>
gbrt = GradientBoostingRegressor(max_depth=3, n_estimators=120, random_state=42)
 gbrt.fit(X_train, y_train)
 print(gbrt.score(X_train, y_train))
params_xgb = {
'learning_rate': [.1,.5,.7,.9,.95,.99, 1],
'colsample_bytree': [.3,.4,.5,.6],
'max_depth': [4],
'alpha': [3],
'subsample': [.5],
'n_estimators': [30, 70, 100, 200]
}
xgb_model = XGBRegressor()
xgb_regressor = GridSearchCV(xgb_model, params_xgb, scoring='neg_root_mean_squared_error', cv = 10, n_jobs = -1)
 xgb_regressor.fit(X_train, y_train)
 print(f'Optimal lr: {xgb_regressor.best_params_["learning_rate"]}')
 print(f'Optimal colsample_bytree: {xgb_regressor.best_params_["colsample_bytree"]}')
 print(f'Optimal n_estimators: {xgb_regressor.best_params_["n_estimators"]}')
 print(f'Best score: {xgb_regressor.best_score_}' )<train_model>
xgb_reg = xgboost.XGBRegressor(n_estimators = 550, random_state=42)
 xgb_reg.fit(X_train, y_train)
 print(xgb_reg.score(X_train, y_train))
xgb_model = XGBRegressor(learning_rate=xgb_regressor.best_params_["learning_rate"],
colsample_bytree=xgb_regressor.best_params_["colsample_bytree"],
max_depth=4, alpha=3, subsample=.5,
n_estimators=xgb_regressor.best_params_["n_estimators"], n_jobs=-1)
 xgb_model.fit(X_train, y_train)
 y_train_pred = xgb_model.predict(X_train)
 y_pred = xgb_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<compute_test_metric>
y_pred=xgb_reg.predict(test_df)
 y_pred
xgb_model.fit(X_train, y_train, early_stopping_rounds=4,
eval_set=[(X_test, y_test)], verbose=False)
 y_train_pred = xgb_model.predict(X_train)
 y_pred = xgb_model.predict(X_test)
 print('Train r2 score: ', r2_score(y_train_pred, y_train))
print('Test r2 score: ', r2_score(y_test, y_pred))
train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))
test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'Train RMSE: {train_rmse:.4f}')
 print(f'Test RMSE: {test_rmse:.4f}' )<compute_train_metric>
y_pred[y_pred<0] = 0
y_pred = np.expm1(y_pred)
 y_pred
<choose_model_class><EOS> 
prediction = pd.DataFrame(y_pred)
 submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 prediction_df = pd.concat([submission['Id'], prediction], axis=1)
 prediction_df.columns=['Id','SalePrice']
prediction_df.to_csv('sample_submission.csv',index=False )