Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<groupby>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.ensemble import RandomForestRegressor
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import mean_squared_log_error
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
import math
times = op.groupby(['user_id', 'product_id'])[['order_id']].count()
times.columns = ['Times_Bought_N']
times.head()<prepare_output>
def root_mean_squared_log_error(y_validations, y_predicted):
if len(y_predicted)!= len(y_validations):
return 'error: mismatch in number of data points between x and y'
y_predict_modified = [math.log(i)for i in y_predicted]
y_validations_modified = [math.log(i)for i in y_validations]
return mean_squared_error(y_validations_modified, y_predict_modified, squared=False )
total_orders = op.groupby('user_id')['order_number'].max().to_frame('total_orders')
 total_orders.head()<merge>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
span = pd.merge(total_orders, first_order_no, on='user_id', how='right')
 span.head()<feature_engineering>
year_feature = [feature for feature in numeric_features if 'Yr' in feature or 'Year' in feature]
year_feature
span['Order_Range_D'] = span.total_orders - span.first_order_number + 1
span.head()<merge>
top_correlations = df_train.corr()
top_feature_columns = top_correlations['SalePrice'][top_correlations['SalePrice'].values > 0.2].index.values
top_feature_columns
uxp_ratio = pd.merge(times, span, on=['user_id', 'product_id'], how='left')
 uxp_ratio.head()<feature_engineering>
total = df_train.isnull().sum().sort_values(ascending=False)
 percent =(df_train.isnull().sum() / df_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
uxp_ratio['uxp_reorder_ratio'] = uxp_ratio.Times_Bought_N / uxp_ratio.Order_Range_D
uxp_ratio.head()<drop_column>
def loss_func(y_predicted, y_actual):
squared_error =(y_predicted - y_actual)** 2
sum_squared_error = squared_error.sum()
size = len(y_actual)
 return 1/(2*size)* sum_squared_error
uxp_ratio = uxp_ratio.drop(['Times_Bought_N', 'total_orders', 'first_order_number', 'Order_Range_D'], axis=1)
 uxp_ratio.head()<drop_column>
x = df_train['GrLivArea']
y = df_train['SalePrice']
x =(x - x.mean())/ x.std()
x = np.c_[np.ones(x.shape[0]), x]
x.shape
del [times, first_order_no, span]<merge>
x = df_train['GrLivArea']
y = df_train['SalePrice']
uxp = uxp.merge(uxp_ratio, on=['user_id', 'product_id'], how='left')
 del uxp_ratio
uxp.head()<groupby>
x =(x - x.mean())/ x.std()
print(x.shape)
 x = np.c_[np.ones(x.shape[0]), x]
print(x.shape)
op['order_number_back'] = op.groupby('user_id')['order_number'].transform(max)- op.order_number +1
op.head(15 )<groupby>
class SimpleLinearRegression:
def get_predictions(self, X):
return np.dot(X, self._W)
 def _get_gradient_descent_step(self, X, targets, learning_rate):
predictions = self.get_predictions(X)
 error = predictions - targets
gradient = np.dot(X.T, error)/len(x)
 self._W -= learning_rate * gradient
def fit(self, X, y, iterations_num=1000, learning_rate=0.01):
self._W = np.zeros(X.shape[1])
 self._history_of_cost = []
self._w_history = [self._W]
for i in range(iterations_num):
predictions = self.get_predictions(X)
 cost = loss_func(predictions, y)
 self._history_of_cost.append(cost)
 self._get_gradient_descent_step(x, y, learning_rate)
 self._w_history.append(self._W.copy())
 return self
last_five = op5.groupby(['user_id','product_id'])[['order_id']].count()
last_five.columns = ['times_last5']
last_five.head(10 )<merge>
house_price_linear_result = SimpleLinearRegression()
house_price_linear_result.fit(x, y, iterations_num=2000, learning_rate=0.01)
 print(house_price_linear_result._W )
uxp = uxp.merge(last_five, on=['user_id', 'product_id'], how='left')
 del last_five
uxp.head()<correct_missing_values>
x = df_train[['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF',
'YearBuilt', 'FullBath', 'OpenPorchSF', 'WoodDeckSF', '2ndFlrSF', 'YearRemodAdd',
'MasVnrArea', 'LotFrontage', 'LotArea']]
x =(x - x.mean())/ x.std()
x = np.c_[np.ones(x.shape[0]), x]
house_price_multi_variate_linear_result = SimpleLinearRegression()
house_price_multi_variate_linear_result.fit(x, y, iterations_num = 2000, learning_rate=0.01)
 print(house_price_multi_variate_linear_result._W )
uxp = uxp.fillna(0)
 uxp.head()<merge>
train_target_label = df_train['SalePrice']
top_feature_columns_modified = ['OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'YearBuilt', 'FullBath', 'OpenPorchSF',
'WoodDeckSF', '2ndFlrSF', 'YearRemodAdd', 'MasVnrArea', 'LotFrontage', 'LotArea']
training_sample_df = df_train[top_feature_columns_modified]
test_sample_df = df_test[top_feature_columns_modified]
training_sample_df.head()
data = uxp.merge(user, on='user_id', how='left')
 data.head()<merge>
imputer = SimpleImputer(strategy = 'median')
 imputer.fit(training_sample_df)
 imputer.fit(test_sample_df)
 training_sample_df = imputer.transform(training_sample_df)
 test_sample_df = imputer.transform(test_sample_df )
data = data.merge(prd, on='product_id', how='left')
 data.head()<set_options>
scaler = StandardScaler()
scaler.fit(training_sample_df)
 scaler.fit(test_sample_df)
 training_sample_df = scaler.transform(training_sample_df)
 test_sample_df = scaler.transform(test_sample_df )
del op, user, prd, uxp
gc.collect()<filter>
X_train_split, X_test_split, y_train, y_test = train_test_split(training_sample_df, train_target_label, random_state=42, train_size = 0.8)
 X_train_split.shape
orders_future = orders[(( orders.eval_set=='train')|(orders.eval_set=='test')) ]
orders_future = orders_future[ ['user_id', 'eval_set', 'order_id'] ]
orders_future.head(10)
<merge>
model = LinearRegression()
model.fit(training_sample_df, train_target_label )
data = data.merge(orders_future, on='user_id', how='left')
 data.head(10 )<filter>
print('Intercept is ', model.intercept_)
 print("Slope i.e.coefficient of x is ", model.coef_ )
data_train = data[data.eval_set=='train']
data_train.head()<merge>
print("Regression score is", model.score(training_sample_df, train_target_label))
print('train_target_label is ', train_target_label )
data_train = data_train.merge(order_products_train[['product_id','order_id', 'reordered']], on=['product_id','order_id'], how='left')
 data_train.head(15 )<feature_engineering>
y_predict = model.predict(X_test_split )
data_train['reordered'] = data_train['reordered'].fillna(0)
 data_train.head(15 )<drop_column>
rmse_simple_linear = np.sqrt(metrics.mean_squared_error(y_test, y_predict))
print('Root Mean Square Error is ', rmse_simple_linear)
 rmse_log_simple_linear = np.sqrt(mean_squared_log_error(y_test, y_predict))
print('Root Mean Square Log Error is ', rmse_log_simple_linear)
data_train = data_train.set_index(['user_id', 'product_id'])
 data_train.head(15 )<drop_column>
prediction_on_test_data = model.predict(test_sample_df)
 testID = df_test['Id']
predict_submission = pd.DataFrame()
predict_submission['ID'] = testID
predict_submission['SalePrice'] = prediction_on_test_data
predict_submission
data_train = data_train.drop(['eval_set', 'order_id'], axis=1)
 data_train.head(15 )<filter>
model_ridge = Ridge(alpha=0.5)
 model_ridge.fit(training_sample_df, train_target_label)
 y_predict_ridge = model_ridge.predict(X_test_split)
 mse_linear_ridge = np.sqrt(metrics.mean_squared_error(y_test, y_predict_ridge))
print('MSE of Linear Ridge is ', mse_linear_ridge)
 mse_log_linear_ridge = np.sqrt(mean_squared_log_error(y_test, y_predict_ridge))
print('MSE Log of Linear Ridge is ', mse_log_linear_ridge)
data_test = data[data.eval_set=='test']
data_test.head()<drop_column>
model_lasso = Lasso(alpha=33)
 model_lasso.fit(training_sample_df, train_target_label)
 y_predict_lasso = model_lasso.predict(X_test_split)
 mse_linear_lasso = np.sqrt(metrics.mean_squared_error(y_test, y_predict_lasso))
print('MSE of Lasso Regression is ', mse_linear_lasso)
 mse_log_linear_lasso = np.sqrt(mean_squared_log_error(y_test, y_predict_lasso))
print('MSE Log of Lasso Regression is ', mse_log_linear_lasso )
data_test = data_test.set_index(['user_id', 'product_id'])
 data_test.head()<drop_column>
RFR = RandomForestRegressor(max_depth=50)
 RFR.fit(training_sample_df, train_target_label)
 y_predict_random_forest = RFR.predict(X_test_split)
 mse_random_forest = np.sqrt(metrics.mean_squared_error(y_test, y_predict_random_forest))
print('MSE Random Forest is ', mse_random_forest)
 mse_log_random_forest = np.sqrt(mean_squared_log_error(y_test, y_predict_random_forest))
print('MSE Log Random Forest is ', mse_log_random_forest )
data_test = data_test.drop(['eval_set','order_id'], axis=1)
 data_test.head()<train_model>
Y_test_predicted_for_submission = RFR.predict(test_sample_df)
 indexes = np.arange(df_test.shape[0]+2, 2*df_test.shape[0]+2)
 print('Indexex ', indexes)
X_train, y_train = data_train.drop('reordered', axis=1), data_train.reordered
rfc = RandomForestClassifier(n_estimators=20,max_depth=6, n_jobs=-1 ,random_state=50)
 model = rfc.fit(X_train, y_train)
 feature_importances_df = pd.DataFrame(model.feature_importances_, index = X_train.columns, columns=['importance'] ).sort_values('importance',ascending=False)
 print(feature_importances_df)
 feat_importances = pd.Series(model.feature_importances_, index=X_train.columns ).sort_values()
feat_importances.plot(kind='barh')
 gc.collect() 
<find_best_params>
all_features_except_target = [i for i in df_train.columns if i not in ['SalePrice'] ]
features_X = df_train[all_features_except_target]
y = df_train['SalePrice']
X_feature_engineering = features_X.copy()
X_feature_engineering['years_since_update'] = X_feature_engineering['YearRemodAdd'] - X_feature_engineering['YearBuilt']
X_feature_engineering['garage_value'] = X_feature_engineering['YearBuilt'] * X_feature_engineering['GarageCars']
X_feature_engineering = X_feature_engineering.drop(columns=['GarageCars'])
 feature_numerical_columns = [col_name for col_name in X_feature_engineering.columns if
X_feature_engineering[col_name].dtype in ['int64', 'float64']]
model.get_params()<choose_model_class>
feature_categorical_cols = [col_name for col_name in X_feature_engineering.columns if
X_feature_engineering[col_name].nunique() < 50 and
X_feature_engineering[col_name].dtype in ['object', 'bool']]
feature_numerical_transformer = SimpleImputer(strategy='constant')
 feature_categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 feature_preprocessor = ColumnTransformer(
 transformers=[ 
('num', feature_numerical_transformer, feature_numerical_columns), 
('cat', feature_categorical_transformer, feature_categorical_cols)
 ] )
param_grid = {
'max_depth': [4,5,6],
'max_features': [3, 4, 5],
'n_estimators': [14, 16, 20]
}
rfc = RandomForestClassifier()
grid_search = GridSearchCV(estimator = rfc, n_jobs=1, param_grid = param_grid,
cv = 3,)
 grid_search.fit(X_train, y_train)
 print("The best parameters are: /n", grid_search.best_params_)
 model = grid_search.best_estimator_
del [X_train, y_train, rfc]<find_best_params>
feature_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0.0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.02, max_delta_step=0, max_depth=4,
min_child_weight=0.0, monotone_constraints='() ',
n_estimators=1250, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.8,
tree_method='exact', validate_parameters=1, verbosity=None)
 feature_classifier = Pipeline(steps=[('feature_preprocessor', feature_preprocessor), 
('feature_model', feature_model)
 ])
 feature_X_train, feature_X_valid, feature_y_train, feature_y_test = train_test_split(X_feature_engineering, y, random_state=0)
 feature_classifier.fit(feature_X_train, feature_y_train, feature_model__verbose=False)
 feature_predictions = feature_classifier.predict(feature_X_valid)
 print('RMSLE from custom-defined function', root_mean_squared_log_error(feature_y_test, feature_predictions))
print('RMSLE using scikit-learn function', mean_squared_log_error(feature_y_test, feature_predictions))
<train_model><EOS> 
X_test_original = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 X_test_original['years_since_update'] = X_test_original['YearRemodAdd'] - X_feature_engineering['YearBuilt']
X_test_original['garage_value'] = X_test_original['YearBuilt'] * X_test_original['GarageCars']
X_test_original = X_test_original.drop(columns=['GarageCars'])
 feature_classifier.fit(X_feature_engineering, y, feature_model__verbose=False)
 model_predictions = feature_classifier.predict(X_test_original)
 output = pd.DataFrame({'Id': X_test_original.Id,
'SalePrice': model_predictions})
 output.to_csv('submission.csv', index=False )