Kaggle competition: House Prices - Advanced Regression Techniques
null_counts = object_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<data_type_conversions>
train_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv',index_col=0)
 test_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col=0)
 train_df.head()
columns_None = ['FireplaceQu']
object_columns_df[columns_None]= object_columns_df[columns_None].fillna('None' )<data_type_conversions>
missing_data = train_df.isnull().sum().sort_values(ascending=False ).head(20)
 percentage_missing_data = train_df.isnull().sum() /train_df.isnull().count().sort_values(ascending = False)* 100
missing_values = pd.concat([missing_data, percentage_missing_data], axis=1, keys=['missing_data', 'percentage'])
 missing_values.head(20 )
columns_with_lowNA = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','GarageType','GarageFinish','GarageQual','GarageCond','MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','Electrical','KitchenQual','Functional','SaleType']
object_columns_df[columns_with_lowNA] = object_columns_df[columns_with_lowNA].fillna(object_columns_df.mode().iloc[0] )<count_missing_values>
train_df.drop(['MiscFeature', 'PoolQC', 'Alley', 'Fence'], axis=1, inplace=True)
 train_df.shape
null_counts = numerical_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<feature_engineering>
train_df['LotFrontage'] = train_df['LotFrontage'].fillna(train_df['LotFrontage'].mean() )
numerical_columns_df['GarageYrBlt'] = numerical_columns_df['GarageYrBlt'].fillna(numerical_columns_df['YrSold']-35)
 numerical_columns_df['LotFrontage'] = numerical_columns_df['LotFrontage'].fillna(0)
 numerical_columns_df['MasVnrArea'] = numerical_columns_df['MasVnrArea'].fillna(0)
 numerical_columns_df['BsmtFinSF1'] = numerical_columns_df['BsmtFinSF1'].fillna(368.5)
 numerical_columns_df['BsmtFinSF2'] = numerical_columns_df['BsmtFinSF2'].fillna(0)
 numerical_columns_df['BsmtUnfSF'] = numerical_columns_df['BsmtUnfSF'].fillna(467)
 numerical_columns_df['TotalBsmtSF'] = numerical_columns_df['TotalBsmtSF'].fillna(989.5)
 numerical_columns_df['BsmtFullBath'] = numerical_columns_df['BsmtFullBath'].fillna(0)
 numerical_columns_df['BsmtHalfBath'] = numerical_columns_df['BsmtHalfBath'].fillna(0)
 numerical_columns_df['GarageCars'] = numerical_columns_df['GarageCars'].fillna(2)
 numerical_columns_df['GarageArea'] = numerical_columns_df['GarageArea'].fillna(480 )<count_missing_values>
train_df['GarageCond'] = train_df['GarageCond'].fillna(train_df['GarageCond'].mode() [0])
 train_df['FireplaceQu'] = train_df['FireplaceQu'].fillna(train_df['FireplaceQu'].mode() [0])
 train_df['GarageType'] = train_df['GarageType'].fillna(train_df['GarageType'].mode() [0])
 train_df['GarageYrBlt'] = train_df['GarageYrBlt'].fillna(train_df['GarageYrBlt'].mode() [0])
 train_df['GarageFinish'] = train_df['GarageFinish'].fillna(train_df['GarageFinish'].mode() [0])
 train_df['GarageQual'] = train_df['GarageQual'].fillna(train_df['GarageQual'].mode() [0])
 train_df['BsmtExposure'] = train_df['BsmtExposure'].fillna(train_df['BsmtExposure'].mode() [0])
 train_df['BsmtFinType2'] = train_df['BsmtFinType2'].fillna(train_df['BsmtFinType2'].mode() [0])
 train_df['BsmtFinType1'] = train_df['BsmtFinType1'].fillna(train_df['BsmtFinType1'].mode() [0])
 train_df['BsmtCond'] = train_df['BsmtCond'].fillna(train_df['BsmtCond'].mode() [0])
 train_df['BsmtQual'] = train_df['BsmtQual'].fillna(train_df['BsmtQual'].mode() [0])
 train_df['MasVnrArea'] = train_df['MasVnrArea'].fillna(train_df['MasVnrArea'].mode() [0])
 train_df['MasVnrType'] = train_df['MasVnrType'].fillna(train_df['MasVnrType'].mode() [0])
 train_df['Electrical'] = train_df['Electrical'].fillna(train_df['Electrical'].mode() [0] )
null_counts = numerical_columns_df.isnull().sum()
print("Number of null values in each column:
{}".format(null_counts))<drop_column>
missing_test_data = test_df.isnull().sum().sort_values(ascending=False)
 percentage_mtd = test_df.isnull().sum() /test_df.isnull().count().sort_values(ascending=False)*100
missing_test_values = pd.concat([missing_test_data, percentage_mtd], axis=1, keys=['missing_test_data', 'percentage'])
 missing_test_values.head(34 )
object_columns_df = object_columns_df.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1 )<filter>
test_df.drop(['PoolQC', 'Fence', 'Alley', 'MiscFeature'], axis=1, inplace=True )
Negatif = numerical_columns_df[numerical_columns_df['Age_House'] < 0]
Negatif<feature_engineering>
test_df['LotFrontage'] = test_df['LotFrontage'].fillna(test_df['LotFrontage'].mean() )
numerical_columns_df['TotalBsmtBath'] = numerical_columns_df['BsmtFullBath'] + numerical_columns_df['BsmtFullBath']*0.5
numerical_columns_df['TotalBath'] = numerical_columns_df['FullBath'] + numerical_columns_df['HalfBath']*0.5
numerical_columns_df['TotalSA']=numerical_columns_df['TotalBsmtSF'] + numerical_columns_df['1stFlrSF'] + numerical_columns_df['2ndFlrSF']<normalization>
test_df['FireplaceQu'] = test_df['FireplaceQu'].fillna(test_df['FireplaceQu'].mode() [0])
 test_df['GarageCond'] = test_df['GarageCond'].fillna(test_df['GarageCond'].mode() [0])
 test_df['GarageFinish'] = test_df['GarageFinish'].fillna(test_df['GarageFinish'].mode() [0])
 test_df['GarageYrBlt'] = test_df['GarageYrBlt'].fillna(test_df['GarageYrBlt'].mode() [0])
 test_df['GarageQual'] = test_df['GarageQual'].fillna(test_df['GarageQual'].mode() [0])
 test_df['GarageType'] = test_df['GarageType'].fillna(test_df['GarageType'].mode() [0])
 test_df['BsmtCond'] = test_df['BsmtCond'].fillna(test_df['BsmtCond'].mode() [0])
 test_df['BsmtExposure'] = test_df['BsmtExposure'].fillna(test_df['BsmtExposure'].mode() [0])
 test_df['BsmtQual'] = test_df['BsmtQual'].fillna(test_df['BsmtQual'].mode() [0])
 test_df['BsmtFinType1'] = test_df['BsmtFinType1'].fillna(test_df['BsmtFinType1'].mode() [0])
 test_df['BsmtFinType2'] = test_df['BsmtFinType2'].fillna(test_df['BsmtFinType2'].mode() [0])
 test_df['MasVnrType'] = test_df['MasVnrType'].fillna(test_df['MasVnrType'].mode() [0])
 test_df['MasVnrArea'] = test_df['MasVnrArea'].fillna(test_df['MasVnrArea'].mode() [0])
 test_df['MSZoning'] = test_df['MSZoning'].fillna(test_df['MSZoning'].mode() [0])
 test_df['BsmtHalfBath'] = test_df['BsmtHalfBath'].fillna(test_df['BsmtHalfBath'].mode() [0])
 test_df['Utilities'] = test_df['Utilities'].fillna(test_df['Utilities'].mode() [0])
 test_df['Functional'] = test_df['Functional'].fillna(test_df['Functional'].mode() [0])
 test_df['BsmtFullBath'] = test_df['BsmtFullBath'].fillna(test_df['BsmtFullBath'].mode() [0])
 test_df['BsmtFinSF2'] = test_df['BsmtFinSF2'].fillna(test_df['BsmtFinSF2'].mode() [0])
 test_df['BsmtFinSF1'] = test_df['BsmtFinSF1'].fillna(test_df['BsmtFinSF1'].mode() [0])
 test_df['BsmtUnfSF'] = test_df['BsmtUnfSF'].fillna(test_df['BsmtUnfSF'].mode() [0])
 test_df['TotalBsmtSF'] = test_df['TotalBsmtSF'].fillna(test_df['TotalBsmtSF'].mode() [0])
 test_df['Exterior2nd'] = test_df['Exterior2nd'].fillna(test_df['Exterior2nd'].mode() [0])
 test_df['SaleType'] = test_df['SaleType'].fillna(test_df['SaleType'].mode() [0])
 test_df['Exterior1st'] = test_df['Exterior1st'].fillna(test_df['Exterior1st'].mode() [0])
 test_df['KitchenQual'] = test_df['KitchenQual'].fillna(test_df['KitchenQual'].mode() [0])
 test_df['GarageArea'] = test_df['GarageArea'].fillna(test_df['GarageArea'].mode() [0])
 test_df['GarageCars'] = test_df['GarageCars'].fillna(test_df['GarageCars'].mode() [0] )
scaler = MinMaxScaler()
numerical_columns_df[['MSSubClass', 'LotFrontage', 'LotArea',
'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd',
'Fireplaces', 'GarageYrBlt', 'GarageCars', 'TotalSA','OverallQual','GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'MoSold', 'YrSold', 'Age_House', 'TotalBsmtBath',
'TotalBath' ]] = scaler.fit_transform(numerical_columns_df[['MSSubClass', 'LotFrontage', 'LotArea',
'OverallCond', 'YearBuilt', 'YearRemodAdd','TotalSA','OverallQual', 'MasVnrArea', 'BsmtFinSF1',
'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',
'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd',
'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF',
'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'MoSold', 'YrSold', 'Age_House', 'TotalBsmtBath',
'TotalBath']])
<categorify>
test_df.isnull().sum().max()
object_columns_df = pd.get_dummies(object_columns_df, columns=object_columns_df.columns )<concatenate>
frames = [train_df, test_df]
full_data = pd.concat(frames, axis=0)
 full_data['SalePrice']
df_final = pd.concat([object_columns_df, numerical_columns_df], axis=1,sort=False)
 df_final.head()<drop_column>
full_data = pd.get_dummies(full_data, drop_first=True )
df_final = df_final.drop(['Id',],axis=1)
 df_train = df_final[df_final['train'] == 1]
df_train = df_train.drop(['train',],axis=1)
 df_test = df_final[df_final['train'] == 0]
df_test = df_test.drop(['train','SalePrice'],axis=1)
 df_test.head() 
<prepare_x_and_y>
full_data['GrLivArea'] = np.log(full_data['GrLivArea'] )
target= df_train['SalePrice']
df_train = df_train.drop(['SalePrice'],axis=1 )<split>
full_data['HasBsmt'] = pd.Series(len(full_data['TotalBsmtSF']), index=full_data.index)
 full_data['HasBsmt'] = 0
full_data.loc[full_data['TotalBsmtSF'] > 0, 'HasBsmt'] = 1
full_data.loc[full_data['HasBsmt']==1, 'TotalBsmtSF'] = np.log(full_data['TotalBsmtSF'])
 full_data.drop(['HasBsmt'], axis=1, inplace=True )
x_train,x_test,y_train,y_test = train_test_split(df_train,target,test_size=0.33,random_state=0 )<choose_model_class>
df_train = full_data.iloc[:1460, :]
df_test = full_data.iloc[1460:,:]
xgb =XGBRegressor(booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.6, gamma=0,num_parallel_tree=1,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=4, min_child_weight=2, n_estimators=2550,tree_method='exact',
n_jobs=1, nthread=None, objective='reg:squarederror',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.6, verbosity=1)
 lgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=11000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.38,
)<split>
df_test['SalePrice'].isnull().sum()
y = target
feature_names = df_train.columns
X = df_train[feature_names]
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1)
<train_model>
df_test.drop(['SalePrice'], axis=1, inplace=True )
lgbm.fit(train_X, train_y )<train_model>
y_train = df_train['SalePrice']
y_train.head()
y_train.shape
xgb.fit(train_X, train_y )<predict_on_test>
x_train = df_train.drop(['SalePrice'], axis=1)
 x_train.head()
predict1 = xgb.predict(x_test)
 predict = lgbm.predict(x_test)
<compute_test_metric>
import xgboost as xgb
from sklearn.metrics import mean_squared_error
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict1))))
print('Root Mean Square Error test = ' + str(math.sqrt(metrics.mean_squared_error(y_test, predict)))) 
<predict_on_test>
xg_reg = xgb.XGBRegressor()
xg_reg
predict4 = lgbm.predict(df_test)
 predict3 = xgb.predict(df_test)
 predict_y =(predict3*.55+predict4*.45 )<save_to_csv>
from sklearn.model_selection import RandomizedSearchCV
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": predict_y })
 submission.to_csv('submission.csv', index=False )<import_modules>
max_depth = [1,3,5,7,9,11,13,15,17,19,21]
n_estimators = [100, 200, 300, 400, 500, 600, 700]
booster = ['gbtree', 'gblinear']
learning_rate = [0.01, 0.05, 0.1, 0.2,0.3,0.4,0.5,0.6,0.7,0.8]
min_child_weight = [1,3,5,7,9,11,13,15,17,19,21]
hyperparameter_grid = {
"max_depth" : max_depth,
"n_estimators" : n_estimators,
"booster" : booster,
"learning_rate" : learning_rate,
"min_child_weight" : min_child_weight
}
import numpy as np
import lightgbm as lgb
import pandas as pd
from kaggle.competitions import twosigmanews
import matplotlib.pyplot as plt
import random
from datetime import datetime, date
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time<set_options>
random_cv = RandomizedSearchCV(estimator = xg_reg, param_distributions=hyperparameter_grid,
cv=5, n_iter=50,
scoring = 'neg_mean_absolute_error',n_jobs = 4,
verbose = 5,
return_train_score = True,
random_state=42 )
env = twosigmanews.make_env()
print('Done!' )<split>
random_cv.fit(x_train, y_train )
( market_train_df, news_train_df)= env.get_training_data()<feature_engineering>
random_cv.best_estimator_
market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]<categorify>
xg_reg = xgb.XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,
importance_type='gain', interaction_constraints='',
learning_rate=0.05, max_delta_step=0, max_depth=5,
min_child_weight=5, missing=None, monotone_constraints='() ',
n_estimators=700, n_jobs=0, num_parallel_tree=1, random_state=0,
reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,
tree_method='exact', validate_parameters=1, verbosity=None )
def create_lag(df_code,n_lag=[3,7,14,],shift_size=1):
code = df_code['assetCode'].unique()
for col in return_features:
for window in n_lag:
rolled = df_code[col].shift(shift_size ).rolling(window=window)
 lag_mean = rolled.mean()
lag_max = rolled.max()
lag_min = rolled.min()
lag_std = rolled.std()
df_code['%s_lag_%s_mean'%(col,window)] = lag_mean
df_code['%s_lag_%s_max'%(col,window)] = lag_max
df_code['%s_lag_%s_min'%(col,window)] = lag_min
return df_code.fillna(-1)
 def generate_lag_features(df,n_lag = [3,7,14]):
features = ['time', 'assetCode', 'assetName', 'volume', 'close', 'open',
'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10',
'returnsOpenNextMktres10', 'universe']
assetCodes = df['assetCode'].unique()
print(assetCodes)
 all_df = []
df_codes = df.groupby('assetCode')
 df_codes = [df_code[1][['time','assetCode']+return_features] for df_code in df_codes]
print('total %s df'%len(df_codes))
pool = Pool(4)
 all_df = pool.map(create_lag, df_codes)
 new_df = pd.concat(all_df)
 new_df.drop(return_features,axis=1,inplace=True)
 pool.close()
return new_df<merge>
xg_reg.fit(x_train, y_train )
<define_variables>
preds = xg_reg.predict(df_test)
 preds
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag )<merge>
pred=pd.DataFrame(preds)
 subdf = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv')
 datasets=pd.concat([subdf['Id'],pred],axis=1)
 datasets.columns=['Id','SalePrice']
datasets.to_csv('sample_submission2.csv',index=False )
market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'] )<categorify>
foo = pd.read_csv('sample_submission2.csv')
 foo