Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import OneHotEncoder, LabelEncoder, StandardScaler, MinMaxScaler, RobustScaler
from sklearn.neighbors import KNeighborsRegressor
from sklearn.feature_selection import RFE, SelectPercentile, f_regression
from sklearn.model_selection import GridSearchCV, KFold, cross_val_score
from sklearn.pipeline import Pipeline
from sklearn.linear_model import Ridge, Lasso, ElasticNet
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
import catboost
from xgboost import XGBRegressor
import lightgbm as lgb
from mlxtend.regressor import StackingCVRegressor
from scipy import stats
df_s = pd.read_csv('.. /input/gendered-pronoun-resolution/sample_submission_stage_2.csv' )<feature_engineering>
random_state = 831
df['A'] = y[:,0]<feature_engineering>
data_train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv" ).set_index("Id")
 data_test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv" ).set_index("Id")
 data = pd.concat([data_train, data_test] )
df['B'] = y[:,1]
df['NEITHER'] = y[:,2]<save_to_csv>
def expand_categorical(data, features):
d = data[features]
values = np.unique(d.values.flatten() ).tolist()
result = pd.DataFrame(index=data.index, columns=values)
 for i, row in d.iterrows() :
v = np.unique(row.values.flatten() ).tolist()
result.loc[i,v] = 1
result.fillna(0, inplace=True)
 result.columns = [features[0]+"_"+col for col in result.columns]
data.drop(features, axis=1, inplace=True)
 return pd.concat([data, result], axis=1 )
df.to_csv('submission.csv', index=False )<set_options>
analyse_numeric("SalePrice" )
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 warnings.filterwarnings('ignore')
<load_from_csv>
data["SalePrice"] = np.log(data["SalePrice"] )
train = pd.read_csv(".. /input/allstate-claims-severity/train.csv")
 test = pd.read_csv(".. /input/allstate-claims-severity/test.csv")
 pd.set_option('display.max_rows', None)
 pd.set_option('display.max_columns', None )<drop_column>
data["MSSubClass"] = data["MSSubClass"].astype(str )
data = train.copy()
valid = test.copy()
del train
del test<define_variables>
data["LotGeometry"] = data["LotArea"] / data["LotFrontage"]
analyse_numeric("LotGeometry" )
s =(data.dtypes == 'object')
 object_cols = list(s[s].index)
 print("Categorical variables:")
 print(len(object_cols))
print(object_cols )<categorify>
data['LotGeometry'].fillna(0, inplace=True)
 data['LotGeometry'] = make_quantile_bins('LotGeometry', 6 )
mergedata = data.append(valid )<drop_column>
analyse_numeric("LotFrontage" )
mergedata = mergedata.drop(setOfDroppedFeatures, axis=1 )<define_variables>
data['LotFrontage'].fillna(0, inplace=True)
 data['LotFrontage'] = make_quantile_bins('LotFrontage', 6 )
s =(data.dtypes != 'object')
 non_object_cols = list(s[s].index)
 non_object_cols = [ col for col in non_object_cols if col not in setOfDroppedFeatures ]
print(len(non_object_cols))<drop_column>
analyse_numeric("LotArea" )
mergedata = mergedata.drop(nonCorrelatedWithOutput, axis=1 )<define_variables>
data.drop("Utilities", axis=1, inplace=True )
unbalanced_cols = ['cat15','cat18','cat20','cat21','cat22','cat33','cat35','cat48','cat56','cat58',
'cat59','cat60','cat62','cat63','cat64','cat67','cat68','cat69','cat70','cat77']<categorify>
data["LandTopology"] = data["LandContour"] + data["LandSlope"]
data.drop("LandSlope", axis=1, inplace=True)
 analyse_categorical("LandTopology" )
mergedata_ohe = pd.get_dummies(mergedata)
 mergedata_ohe.shape<train_model>
data = expand_categorical(data, ["Condition1", "Condition2"] )
print(test_ohe.loss.isnull().sum())
 test_ohe = test_ohe.drop("loss", axis = 1 )<choose_model_class>
data["YearsLastRemod"] = data["YearRemodAdd"] - data["YearBuilt"]
data["YearsLastRemod"] = make_quantile_bins("YearsLastRemod", 3)
 analyse_categorical("YearsLastRemod" )
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0007, random_state=1))
ridge = make_pipeline(RobustScaler() , Ridge(alpha =20, random_state=42))<train_model>
analyse_numeric("YearBuilt" )
X_train = train_ohe.drop("loss", axis = 1)
 Y_train = train_ohe["loss"]
print(X_train.shape)
 print(Y_train.shape)
 del train_ohe<split>
data["YearBuilt"] = make_quantile_bins("YearBuilt", 10)
 analyse_categorical("YearBuilt" )
x_train, x_valid, y_train, y_valid = train_test_split(X_train, Y_train, train_size=.90,random_state=2016)
 RANDOM_STATE = 2016
params = {
'min_child_weight': 1,
'eta': 0.03,
'colsample_bytree': 0.5,
'max_depth': 12,
'subsample': 0.8,
'alpha': 1,
'seed': RANDOM_STATE,
'gamma': 1.5,
'silent': 1,
'verbose_eval': True,
'nthread':7,
'base_score':7.76
}
xgtrain = xgb.DMatrix(x_train, label=y_train)
 xgval = xgb.DMatrix(x_valid, label=y_valid)
 watchlist = [(xgtrain,'train'),(xgval,'eval')]
model = xgb.train(params, xgtrain,300, watchlist,verbose_eval=True)
 del xgtrain
del xgval
xgtest = xgb.DMatrix(test_ohe)
 test_y_log = model.predict(xgtest)
 test_y = np.exp(1)**test_y_log<drop_column>
analyse_numeric("YearRemodAdd" )
del X_train
del y_train<save_to_csv>
data["YearRemodAdd"] = make_quantile_bins("YearRemodAdd", 8)
 analyse_categorical("YearRemodAdd" )
submission = pd.DataFrame()
submission["id"] = test_id
print(test_y.shape)
 print(test_id.shape)
 submission["loss"] = test_y
submission.to_csv("submission_svm_l.csv", index=False )<set_options>
data["Exterior1st"].replace(["WdShing"], "Wd Shng", inplace=True)
 data["Exterior2nd"].replace(["CmentBd"], "CemntBd", inplace=True)
 data["Exterior2nd"].replace(["Brk Cmn"], "BrkComm", inplace=True)
 data = expand_categorical(data, ["Exterior1st", "Exterior2nd"] )
%matplotlib inline<set_options>
analyse_numeric("MasVnrArea" )
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 500 )<define_variables>
data.drop("MasVnrArea", axis=1, inplace=True )
PATH = '.. /input/allstate-claims-severity/'<load_from_csv>
data = expand_categorical(data, ["BsmtFinType1", "BsmtFinType2"] )
variables = [x.replace('.csv', '')for x in files]
for v, f in zip(variables, files):
exec("%s=pd.read_csv(PATH+f)" % v )<drop_column>
analyse_numeric("BsmtFinSF1")
 data["BsmtFinSF1"].fillna(0, inplace=True )
tr, ts = train.drop(['id'], 1), test.drop(['id'], 1 )<count_missing_values>
data['BsmtFinSF1'] = make_quantile_bins('BsmtFinSF1', 3)
 analyse_categorical("BsmtFinSF1" )
print(f'No.of missing values in Train: {tr.isnull().sum().sum() }')
 print(f'No.of missing values in Test: {ts.isnull().sum().sum() }')
<count_values>
data["BsmtFinSF2"].fillna(0, inplace=True)
 data['BsmtFinSF2'] = np.where(data["BsmtFinSF2"] == 0, 0, 1)
 analyse_categorical("BsmtFinSF2" )
pd.DataFrame({'Train': tr.dtypes.value_counts() , 'Test': ts.dtypes.value_counts() })
<feature_engineering>
data.drop('BsmtFinSF2', axis=1, inplace=True )
tr['loss'] = np.log1p(tr['loss'])
 sns.kdeplot(tr['loss'], shade=True)
<define_variables>
data["BsmtUnfSF"].fillna(0, inplace=True)
 data['BsmtUnfSF'] = np.where(data["BsmtUnfSF"] == 0, 0, 1)
 analyse_categorical("BsmtUnfSF" )
SEED = 10<create_dataframe>
data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
analyse_numeric("TotalSF" )
dist_percent = np.divide(len(tr), np.multiply(categories.loc['unique'].values, categories.loc['freq'].values))
percentages = pd.DataFrame({'column': categories.columns, 'percent': dist_percent})
 cat_cols = list(percentages[percentages['percent'] > 0.1]['column'].values)
 print(f"Filtered categories: {len(cat_cols)}")
<data_type_conversions>
analyse_numeric("1stFlrSF" )
corr = tr.drop('loss', 1 ).corr().abs()
upper = corr.where(np.triu(np.ones(corr.shape), k=1 ).astype(np.bool))
to_drop_num = [column for column in upper.columns if any(upper[column] > 0.6)]
num_cols = list(set(tr.drop('loss', 1 ).select_dtypes(['float64'])) - set(to_drop_num))<prepare_x_and_y>
analyse_numeric("2ndFlrSF" )
features = final_tr.drop('loss', 1)
 target = final_tr['loss']<categorify>
data['has2ndFlr'] = np.where(data["2ndFlrSF"] > 0, 1, 0)
 analyse_categorical("has2ndFlr" )
features = pd.get_dummies(features, columns=features.select_dtypes('object' ).columns)
 features.shape<categorify>
data["LowQualFinSF"].fillna(0, inplace=True)
 data['LowQualFinSF'] = np.where(data["LowQualFinSF"] == 0, 0, 1)
 analyse_categorical("LowQualFinSF" )
payload = pd.get_dummies(final_ts, columns=final_ts.select_dtypes('object' ).columns)
 payload.shape<concatenate>
data.drop("LowQualFinSF", axis=1, inplace=True )
features, payload = features.align(payload, join='outer', axis=1, fill_value=0)
 features.shape, payload.shape<define_variables>
analyse_numeric("GrLivArea" )
SIZE = 0.2<split>
data["BathroomsBasement"] = data["BsmtFullBath"] + 0.5 * data["BsmtHalfBath"]
analyse_categorical("BathroomsBasement")
 data.drop(["BsmtFullBath", "BsmtHalfBath"], axis=1, inplace=True )
x, x_test, y, y_test = train_test_split(features, target, random_state=SEED, test_size=SIZE)
 x_train, x_val, y_train, y_val = train_test_split(x, y, random_state=SEED, test_size=SIZE )<import_modules>
data["Bathrooms"] = data["FullBath"] + 0.5 * data["HalfBath"]
analyse_categorical("Bathrooms")
 data.drop(["FullBath", "HalfBath"], axis=1, inplace=True )
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
from sklearn.model_selection import KFold
from sklearn.metrics import mean_absolute_error, accuracy_score<choose_model_class>
data.drop("KitchenAbvGr", axis=1, inplace=True )
eval_set = [(x_val,y_val)]
model = CatBoostRegressor(random_state=SEED,
n_estimators=6000,
max_depth=3,
learning_rate=0.1,
loss_function='MAE',
eval_metric='MAE',
reg_lambda=3,
verbose=100)
 %time model.fit(x_train, y_train, eval_set=eval_set, early_stopping_rounds=30 )<predict_on_test>
analyse_numeric("GarageYrBlt" )
y_train_pred = model.predict(x_train)
 y_val_pred = model.predict(x_val)
 y_test_pred = model.predict(x_test )<compute_test_metric>
data['GarageYrBlt'] = make_quantile_bins('GarageYrBlt', 10)
 analyse_categorical("GarageYrBlt", fillnan=5.0 )
print(f"MAE Train: {mean_absolute_error(np.expm1(y_train_pred), np.expm1(y_train)) }")
 print(f"MAE Val: {mean_absolute_error(np.expm1(y_val_pred), np.expm1(y_val)) }")
 print(f"MAE Test: {mean_absolute_error(np.expm1(y_test_pred), np.expm1(y_test)) }" )<save_to_csv>
analyse_numeric("GarageArea" )
predictions = np.round(np.expm1(model.predict(payload)) , 2)
 submission = pd.DataFrame({'id': test['id'], 'loss': pd.Series(predictions)})
 submission.to_csv('submission.csv', index=False)
<load_from_csv>
data['GarageArea_Bin'] = make_quantile_bins('GarageArea', 10)
 analyse_categorical('GarageArea_Bin' )
train_data = pd.read_csv('.. /input/allstate-claims-severity/train.csv')
 test_data = pd.read_csv('.. /input/allstate-claims-severity/test.csv')
 print("Finished reading train and test data..." )<prepare_x_and_y>
analyse_numeric('WoodDeckSF' )
X_train = train_data.drop(['id', 'loss'], axis = 1)
 y_train = train_data["loss"]
print("Building and training neural network on {} data points...".format(len(train_data)))
 claimNet= MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,
beta_2=0.999, early_stopping=True, epsilon=1e-08,
hidden_layer_sizes=(1500,1500,1500), learning_rate='constant',
learning_rate_init=0.001, max_iter=500, momentum=0.9,
n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,
random_state=None, shuffle=True, solver='adam', tol=0.0001,
validation_fraction=0.1, verbose=True, warm_start=False)
 claimNet.fit(X_train,y_train)
 print("Finished training..." )<train_model>
data["WoodDeckSF"] = np.where(data["WoodDeckSF"] == 0, 0, 1)
 analyse_categorical("WoodDeckSF" )
print("Loading test data...")
<save_to_csv>
analyse_numeric('OpenPorchSF' )
x_test = test_data.drop(['id'], axis = 1)
 print("Starting predictions on {} data points".format(len(x_test)))
 predictions = claimNet.predict(x_test)
 output = pd.DataFrame({'id': test_data.id,'loss': np.exp(predictions)-200})
 sns.distplot(np.exp(predictions)-200, hist = True, rug = True)
 output.to_csv('submission.csv', index=False)
 print("Finished test and I/O to csv file" )<load_from_csv>
data["OpenPorchSF"] = np.where(data["OpenPorchSF"] == 0, 0, 1)
 analyse_categorical("OpenPorchSF" )
train=pd.read_csv('/kaggle/input/allstate-claims-severity/train.csv', index_col='id')
 test=pd.read_csv('/kaggle/input/allstate-claims-severity/test.csv', index_col='id')
 submission=pd.read_csv('/kaggle/input/allstate-claims-severity/sample_submission.csv', index_col='id')
 print(train.shape, test.shape, submission.shape )<drop_column>
data["EnclosedPorch"] = np.where(data["EnclosedPorch"] == 0, 0, 1)
 analyse_categorical("EnclosedPorch" )
train=train.drop(train.loc[train['loss']>40000].index )<feature_engineering>
data["3SsnPorch"] = np.where(data["3SsnPorch"] == 0, 0, 1)
 analyse_categorical("3SsnPorch" )
train['loss']=np.log(train['loss']+1)
 Ytrain=train['loss']<concatenate>
data["ScreenPorch"] = np.where(data["ScreenPorch"] == 0, 0, 1)
 analyse_categorical("ScreenPorch" )
data=train
train=train[list(test)]
all_data=pd.concat(( train, test))
all_data.shape<define_variables>
data["PoolArea"] = np.where(data["PoolArea"] == 0, 0, 1)
 analyse_categorical("PoolArea" )
cat_features=list(np.where(all_data.dtypes==np.object)[0])
 print(cat_features )<categorify>
data.drop("PoolQC", axis=1, inplace=True )
le=LabelEncoder()
non_numeric=list(all_data.select_dtypes(np.object))
for cols in non_numeric:
le.fit(all_data[cols])
 all_data[cols]=le.transform(all_data[cols] )<import_modules>
data.drop("MiscVal", axis=1, inplace=True )
from catboost import CatBoostRegressor
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.model_selection import cross_val_score<choose_model_class>
data.drop("MoSold", axis=1, inplace=True )
model_xgb=XGBRegressor(tree_method='gpu_hist', seed=18, objective='reg:linear', n_jobs=-1, verbosity=0,
colsample_bylevel=0.764115402027029, colsample_bynode=0.29243734009596956,
colsample_bytree= 0.7095719673041723, gamma= 4.127534050725986, learning_rate= 0.02387231810322894,
max_depth=14, min_child_weight=135, n_estimators=828,reg_alpha=0.3170105723222332,
reg_lambda= 0.3660379465131937, subsample=0.611471430211575)
 model_xgb<choose_model_class>
data.drop("YrSold", axis=1, inplace=True )
model_LGB=LGBMRegressor(objective='regression_l1', random_state=18, subsample_freq=1,
colsample_bytree=0.3261853512759363, min_child_samples=221, n_estimators=2151, num_leaves= 45,
reg_alpha=0.9113713668943361, reg_lambda=0.8220990333713991, subsample=0.49969995651550947,
max_bin=202, learning_rate=0.02959820893211799)
 model_LGB<choose_model_class>
def generate_dummies(data):
features = data.columns[data.dtypes == "object"].to_list()
data = pd.concat([data, pd.get_dummies(data[features])], axis=1)
 data.drop(features, axis=1, inplace=True)
 return data
model_Cat=CatBoostRegressor(loss_function='MAE', random_seed=18, task_type='GPU', cat_features=cat_features, verbose=False,
iterations=2681, learning_rate=0.2127106032536721, depth=7, l2_leaf_reg=5.266150673910493,
random_strength=7.3001140226199315, bagging_temperature=0.26098669708900213)
 model_Cat<predict_on_test>
data = generate_dummies(data)
 data.head(1 )
model_Cat.fit(Xtrain, Ytrain)
 model_LGB.fit(Xtrain, Ytrain)
 model_xgb.fit(Xtrain, Ytrain)
 lgb_predictions=model_LGB.predict(Xtest)
 cat_predictions=model_Cat.predict(Xtest)
 xgb_predictions=model_xgb.predict(Xtest )<save_to_csv>
X_train = data.loc[data_train.index].drop("SalePrice", axis=1)
 y_train = data.loc[data_train.index, "SalePrice"]
X_test = data.loc[data_test.index].drop("SalePrice", axis=1)
 cv=KFold(10, shuffle=True, random_state=random_state )
predictions=(lgb_predictions + cat_predictions + xgb_predictions)/3
predictions=np.exp(predictions)-1
submission['loss']=predictions
submission.to_csv('Result.csv')
 submission.head()<load_from_csv>
model = KNeighborsRegressor()
pipe = Pipeline([("scaler", RobustScaler()),("knn", model)])
 params = {
'knn__n_neighbors': [3, 5, 7, 8, 9, 10, 11, 15, 20,],
'knn__weights': ['uniform', 'distance'],
}
knn_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, n_jobs=-1, scoring='neg_root_mean_squared_error')
 knn_grid.fit(X_train, y_train)
 train_score = knn_grid.score(X_train, y_train )
df=pd.read_csv('.. /input/train.csv')
 testdf=pd.read_csv('.. /input/test.csv')
 print('-------Training data info------------>')
 print(df.info())
 print('-------Testing data info------------>')
 print(testdf.info() )<count_missing_values>
MODEL_0 = knn_grid.best_estimator_
print(f"Model train / CV score is {train_score:.5f} / {knn_grid.best_score_:.5f}")
print(df.isnull().sum().max())
 print(testdf.isnull().sum().max() )<define_variables>
prediction_0 = MODEL_0.predict(X_test)
 prediction_0 = np.e ** prediction_0
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_0})
 submit.to_csv("MODEL_0.csv",index=False )
colNames=df.select_dtypes(include='object' ).columns
colNames<categorify>
model = Ridge()
pipe = Pipeline([("scaler", RobustScaler()),("ridge", model)])
 params = {
'ridge__alpha': [0.01, 0.1, 1, 10, 100, 1000]
}
ridge_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, n_jobs=-1, scoring='neg_root_mean_squared_error')
 ridge_grid.fit(X_train, y_train)
 train_score = ridge_grid.score(X_train, y_train )
df=pd.get_dummies(df,columns=colNames,drop_first=True)
 testdf=pd.get_dummies(testdf,columns=colNames,drop_first=True )<prepare_x_and_y>
MODEL_1 = ridge_grid.best_estimator_
print(f"Model CV score is {train_score:.5f} / {ridge_grid.best_score_:.5f}")
y=df['loss']
testdf=testdf.drop(columns=c1, axis=1)
 df=df.drop(columns=c2,axis=1 )<prepare_x_and_y>
prediction_1 = MODEL_1.predict(X_test)
 prediction_1 = np.e ** prediction_1
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_1})
 submit.to_csv("MODEL_1.csv",index=False )
testdf=testdf.drop(list(col_corr)+['id'],axis=1)
 X=df.drop(list(col_corr)+['id'],axis=1)
<split>
model = Lasso(max_iter=100000)
 pipe = Pipeline([("scaler", MinMaxScaler()),("lasso", model)])
 params = {
'lasso__alpha': [0.00001, 0.0001, 0.001, 0.01, 0.1]
}
lasso_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, n_jobs=-1, scoring='neg_root_mean_squared_error')
 lasso_grid.fit(X_train, y_train)
 train_score = lasso_grid.score(X_train, y_train )
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=.1,random_state=1)
 y_train=np.log1p(y_train )<train_model>
MODEL_2 = lasso_grid.best_estimator_
print(f"Model CV score is {train_score:.5f} / {lasso_grid.best_score_:.5f}")
ss=StandardScaler()
ss.fit(X_train )<categorify>
prediction_2 = MODEL_2.predict(X_test)
 prediction_2 = np.e ** prediction_2
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_2})
 submit.to_csv("MODEL_2.csv",index=False )
<predict_on_test>
model = ElasticNet(max_iter=100000)
 pipe = Pipeline([("scaler", MinMaxScaler()),("elastic", model)])
 params = {
'elastic__alpha': [0.0001, 0.001, 0.01, 0.1],
'elastic__l1_ratio': [0.001, 0.01, 0.1, 0.5, 1, 10]
}
elastic_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, n_jobs=-1, scoring='neg_root_mean_squared_error')
 elastic_grid.fit(X_train, y_train)
 train_score = elastic_grid.score(X_train, y_train )
xg=XGBRegressor(n_estimators=1000,learning_rate=.1,n_jobs=-1)
 xg.fit(ss.transform(X_train),y_train)
 pred=xg.predict(ss.transform(X_test))
print(mean_absolute_error(y_test,np.expm1(pred)))
<predict_on_test>
MODEL_3 = elastic_grid.best_estimator_
print(f"Model CV score is {train_score:.5f} / {elastic_grid.best_score_:.5f}")
testPredict=xg.predict(ss.transform(testdf))<load_from_csv>
prediction_3 = MODEL_3.predict(X_test)
 prediction_3 = np.e ** prediction_3
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_3})
 submit.to_csv("MODEL_3.csv",index=False )
sample=pd.read_csv('.. /input/sample_submission.csv' )<feature_engineering>
model = SVR()
pipe = Pipeline([("scaler", StandardScaler()),("select", SelectPercentile(score_func=f_regression)) ,("svr", model)])
 params = {
'svr__C': [0.01, 0.1, 1, 10, 1000],
'svr__epsilon': [0.0001, 0.001, 0.01, 0.1],
'select__percentile': [10, 20, 25, 27, 28, 29, 30, 50, 100]
}
svr_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, n_jobs=-1, scoring='neg_root_mean_squared_error')
 svr_grid.fit(X_train, y_train)
 train_score = elastic_grid.score(X_train, y_train )
sample['loss']=np.expm1(testPredict )<save_to_csv>
MODEL_4 = svr_grid.best_estimator_
print(f"Model CV score is {train_score:.5f} / {svr_grid.best_score_:.5f}")
sample.to_csv('Submission.csv',index=False )<set_options>
prediction_4 = MODEL_4.predict(X_test)
 prediction_4 = np.e ** prediction_4
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_4})
 submit.to_csv("MODEL_4.csv",index=False )
init_notebook_mode(connected=True)
 cf.go_offline()
%matplotlib inline 
<define_variables>
model = catboost.CatBoostRegressor(random_seed=random_state, verbose=False)
 pipe = Pipeline([("catboost", model)])
 params = {
'catboost__learning_rate':[0.01, 0.1, 0.2],
'catboost__depth': [5, 6]
}
catboost_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, scoring='neg_root_mean_squared_error')
 catboost_grid.fit(X_train, y_train)
 train_score = catboost_grid.score(X_train, y_train )
data_desc = '.. /input/house-prices-advanced-regression-techniques/data_description.txt'
train_path = '.. /input/house-prices-advanced-regression-techniques/train.csv'
test_path = '.. /input/house-prices-advanced-regression-techniques/test.csv'<load_from_csv>
MODEL_5 = catboost_grid.best_estimator_
print(f"Model CV score is {train_score:.5f} / {catboost_grid.best_score_:.5f}")
data_desc = open(data_desc, mode='r')
 for line in data_desc.readlines() :
print(line[:-1] )<load_from_csv>
prediction_5 = MODEL_5.predict(X_test)
 prediction_5 = np.e ** prediction_5
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_5})
 submit.to_csv("MODEL_5.csv",index=False )
train_data = pd.read_csv(train_path, index_col=0)
 test_data = pd.read_csv(test_path, index_col=0 )<count_missing_values>
model = XGBRegressor(n_estimators=1000)
 pipe = Pipeline([("xgb", model)])
 params = {
'xgb__learning_rate': [.01,.02,.03,.05,.07],
'xgb__max_depth': [3, 4, 5]
}
xgb_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, scoring='neg_root_mean_squared_error')
 xgb_grid.fit(X_train, y_train)
 train_score = xgb_grid.score(X_train, y_train )
print('Missing Values in Train Data:
')
 print(train_data.isna().sum() [train_data.isna().sum() != 0] )<count_missing_values>
MODEL_6 = xgb_grid.best_estimator_
print(f"Model CV score is {train_score:.5f} / {xgb_grid.best_score_:.5f}")
print('Missing Values in Test Data :
')
 print(test_data.isna().sum() [test_data.isna().sum() != 0] )<prepare_x_and_y>
prediction_6 = MODEL_6.predict(X_test)
 prediction_6 = np.e ** prediction_6
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_6})
 submit.to_csv("MODEL_6.csv",index=False )
y = train_data['SalePrice']
data = pd.concat([train_data.drop('SalePrice', axis=1), test_data])
 data.head(3 )<sort_values>
model = lgb.LGBMRegressor(n_estimators=1000)
 pipe = Pipeline([("lgb", model)])
 params = {
'lgb__num_leaves': [2, 3, 4, 5],
'lgb__max_depth': [3, 4, 5]
}
lgb_grid = GridSearchCV(pipe, param_grid=params, cv=cv, verbose=1, scoring='neg_root_mean_squared_error')
 lgb_grid.fit(X_train, y_train)
 train_score = lgb_grid.score(X_train, y_train )
def missing_data_func(feature_type):
if feature_type.lower() [0] == 'c':
data_type = ['object']
elif feature_type.lower() [0] == 'n':
data_type = ['integer', 'float']
else :
data_type = ['integer', 'float', 'object']
missing_value = data.select_dtypes(include=data_type ).isna().sum() [data.select_dtypes(include=data_type ).isna().sum() != 0].sort_values(ascending=False)
 missing_percentage =(missing_value/2919)*100
missing_data = pd.DataFrame(data=[missing_value, missing_percentage], index=['Count', 'Percentage'])
 return missing_data
def visualize_missing_data(Data, title):
sns.set(font_scale=1.3)
 plt.figure(figsize=(20, 5))
sns.barplot(x=Data.T.index, y=Data.T['Percentage'], palette="rocket")
 plt.xticks(rotation=90)
 plt.title(label=title, fontdict={'fontsize':18})
 plt.show()<drop_column>
MODEL_7 = lgb_grid.best_estimator_
print(f"Model CV score is {train_score:.5f} / {lgb_grid.best_score_:.5f}")
data.drop(labels=['YearRemodAdd', 'LotArea', '1stFlrSF', 'TotRmsAbvGrd', 'GarageYrBlt', 'GarageCars'], axis=1, inplace=True)
 data.drop(labels=['MSSubClass', 'OverallCond', 'BsmtFinSF2', 'BsmtUnfSF', 'BsmtFullBath', 'LowQualFinSF', 'BsmtHalfBath',
'BedroomAbvGr', 'KitchenAbvGr', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',
'MiscVal', 'MoSold', 'YrSold'], axis=1, inplace=True )<drop_column>
prediction_7 = MODEL_7.predict(X_test)
 prediction_7 = np.e ** prediction_7
submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction_7})
 submit.to_csv("MODEL_7.csv",index=False )
data.drop(labels=['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu'], axis=1, inplace=True )<count_values>
prediction = np.mean([prediction_1, prediction_2, prediction_5, prediction_6, prediction_7], axis=0)
 submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction})
 submit.to_csv("MODEL_8.csv",index=False )
categorical = dict()
for column in data.select_dtypes(include=['object'] ).columns:
categorical[column] =(train_data[column].value_counts() [0].sum() , 
(train_data[column].value_counts() [0].sum() /train_data[column].value_counts().sum())*100,
train_data[column].value_counts().shape[0])
 dominant_features = pd.DataFrame(data=categorical.values() , index=categorical.keys() , columns=['Count', 'Percentage',
'variable type'] ).sort_values(by='Percentage')
 dominant_features<drop_column>
prediction = np.mean([prediction_2, prediction_3, prediction_5, prediction_6, prediction_7], axis=0)
 submit = pd.DataFrame({"Id": data_test.index,"SalePrice": prediction})
 submit.to_csv("MODEL_9.csv",index=False )