Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
import numpy as np
import pandas as pd
import statsmodels.formula.api as sm
import sklearn.preprocessing as sk
from sklearn.metrics import mean_squared_error,mean_squared_log_error,r2_score
from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor,AdaBoostRegressor
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import weight_boosting
from sklearn.ensemble import BaggingRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from tensorflow import keras
df=pd.get_dummies(df_full, dtype=float )<split>
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 y_train=train_data.pop('SalePrice')
 test_id=test_data['Id'].values.reshape(-1,1 )
numTrain=df_train.shape[0]
train = df[:numTrain]
test = df[numTrain:]<split>
y_train = np.log(y_train)
X_train, X_test, y_train, y_test = train_test_split(train,y,test_size=0.3,random_state=42)
 print("Shapes: ", X_train.shape, X_test.shape, y_train.shape, y_test.shape )<import_modules>
train_data.isna().sum() [train_data.isna().sum() >0]
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Lasso, Ridge, ElasticNet
from sklearn.ensemble import AdaBoostRegressor
from xgboost import XGBRegressor 
<define_variables>
train_data=train_data.drop(columns=['PoolQC','Fence','MiscFeature','Alley'],axis=0)
 test_data=test_data.drop(columns=['PoolQC','Fence','MiscFeature','Alley'],axis=0 )
best_estimators=[]<train_on_grid>
cat_list = list(train_data.dtypes[train_data.dtypes == 'object'].index )
params = {
"alpha" : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
"fit_intercept" : [True, False],
"normalize" : [True,False],
"solver" : ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'],
"tol" : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
ridge = Ridge()
ridge_grid = GridSearchCV(ridge, params, scoring='r2', cv=5, n_jobs=-1)
 ridge_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(ridge_grid.best_params_))
print("Best score: {}".format(ridge_grid.best_score_))
best_estimators.append(["Ridge",ridge_grid.best_estimator_] )<train_on_grid>
train_data.isna().sum() [train_data.isna().sum() >0]
params = {
'alpha' : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
'fit_intercept' : [True, False],
'normalize' : [True,False],
'tol' : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
lasso = Lasso()
lasso_grid = GridSearchCV(lasso, params, scoring='r2', cv=5, n_jobs=-1)
 lasso_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(lasso_grid.best_params_))
print("Best score: {}".format(lasso_grid.best_score_))
best_estimators.append(["Lasso",lasso_grid.best_estimator_] )<train_on_grid>
test_data.isna().sum() [test_data.isna().sum() >0]
params = {
"alpha" : [.01,.1,.5,.7,.9,.95,.99, 1, 5, 10, 20],
"fit_intercept" : [True, False],
"normalize" : [True,False],
"tol" : [0.0001, 0.001, 0.01, 0.1],
"random_state" : [42]
}
EL = ElasticNet()
EL_grid = GridSearchCV(EL, params, scoring='r2', cv=5, n_jobs=-1)
 EL_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(EL_grid.best_params_))
print("Best score: {}".format(EL_grid.best_score_))
best_estimators.append(["ElasticNet",EL_grid.best_estimator_] )<train_on_grid>
train_data.isna().sum() [train_data.isna().sum() >0]
params = {
"learning_rate": [.1,.5,.7,.9,.95,.99, 1],
"colsample_bytree": [.3,.4,.5,.6],
"max_depth": [2, 4],
"alpha": [1, 3, 5],
"subsample": [.5],
"n_estimators": [30, 70, 100, 200],
"random_state" : [42]
}
XGBR = XGBRegressor()
XGBR_grid = GridSearchCV(XGBR, params, scoring='r2', cv=5, n_jobs=-1)
 XGBR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(XGBR_grid.best_params_))
print("Best score: {}".format(XGBR_grid.best_score_))
best_estimators.append(["XGBoostR",XGBR_grid.best_estimator_] )<train_on_grid>
test_data.isna().sum() [train_data.isna().sum() >0]
params = {
"n_estimators": [10, 30, 50, 100],
"learning_rate": [.01, 0.1, 0.5, 0.9, 0.95, 1],
"random_state" : [42]
}
AdaBoostR = AdaBoostRegressor()
AdaBoostR_grid = GridSearchCV(AdaBoostR, params, scoring='r2', cv=5, n_jobs=-1)
 AdaBoostR_grid.fit(X_train, y_train)
 print("Best parameters: {}:".format(AdaBoostR_grid.best_params_))
print("Best score: {}".format(AdaBoostR_grid.best_score_))
best_estimators.append(["AdaBoostR",AdaBoostR_grid.best_estimator_] )<choose_model_class>
train_data=pd.DataFrame(data=train_data,columns=keys)
 test_data=pd.DataFrame(data=test_data,columns=keys)
pipelines = []
for name,model in best_estimators:
pipeline = Pipeline([("Scaler",StandardScaler()), 
(name,model)
 ])
 pipelines.append(["Scaled_"+name,pipeline] )<feature_engineering>
total_data=pd.concat([train_data,test_data],axis=0, )
evaluate = pd.DataFrame(columns=["model","cv","std"])
 for name,model in pipelines:
kfold = KFold(n_splits=10,random_state=42)
 cv = cross_val_score(model, X_train, y_train, cv=kfold, n_jobs=-1, scoring="neg_root_mean_squared_error")
 row = evaluate.shape[0]
evaluate.loc[row,"model"] = name
evaluate.loc[row,"cv"] = round(cv.mean() ,3)
 evaluate.loc[row,"std"] = "+/- {}".format(round(cv.std() ,4))
evaluate = evaluate.sort_values("cv",ascending=False)
 evaluate<import_modules>
total_data=pd.get_dummies(data=total_data,drop_first=True)
 train_data=total_data.iloc[:1460,:]
test_data=total_data.iloc[1460:,:]
XX=train_data.copy()
y=y_train.copy()
from sklearn.ensemble import VotingRegressor<define_search_model>
history=pd.DataFrame(columns=['itreation','will_deleted','pvalue','R_adjusted','rsquared'])
 for x in range(len(XX.columns)) :
regressor_osl=sm.OLS(endog=y,exog=XX ).fit()
pvalues=regressor_osl.pvalues
pvalues=pd.DataFrame(pvalues)
 p=pd.DataFrame(pvalues.reset_index())
 out=pd.DataFrame()
out=p.loc[p[0]== max(p[0])]
out=pd.Series.tolist(out)
 w1=out[0][0]
R=regressor_osl.rsquared_adj
r=regressor_osl.rsquared
XX.pop(w1)
 history=history.append([{'itreation':x+1,'will_deleted':out[0][0],'pvalue':out[0][1],'R_adjusted':R,'rsquared':r}] )
votings = []
votings.append(( "Scaled_all_models",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_), 
("Ridge",ridge_grid.best_estimator_)
 ])
)])))
 votings.append(( "Scaled_XGBR_AB",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])) ])
))
votings.append(( "Scaled_XGBR_R",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("Ridge",ridge_grid.best_estimator_)
 ])) ])))
 votings.append(( "Scaled_AB_R",Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("AdaBoostR", AdaBoostR_grid.best_estimator_), 
("Ridge",ridge_grid.best_estimator_)
 ])) ])) )<feature_engineering>
drop=history[history['pvalue']>.05]['will_deleted']
drop=list(drop)
 drop
evaluate_vote = pd.DataFrame(columns=["model","cv","std"])
 for name, model in votings:
kfold = KFold(n_splits=10,random_state=42)
 cv = cross_val_score(model,X_train,y_train, cv=kfold, scoring="neg_root_mean_squared_error",n_jobs=-1)
 row = evaluate_vote.shape[0]
evaluate_vote.loc[row,"model"] = name
evaluate_vote.loc[row,"cv"] = round(cv.mean() ,4)
 evaluate_vote.loc[row,"std"] = "+- {}".format(round(cv.std() ,5))
evaluate_vote = evaluate_vote.sort_values("cv",ascending=False)
 evaluate_vote<train_model>
train_data.drop(columns=drop,inplace=True)
 test_data.drop(columns=drop,inplace=True )
best_model = Pipeline([("Scaler",StandardScaler()), 
("Votings",VotingRegressor([("XGBoostR",XGBR_grid.best_estimator_), 
("AdaBoostR", AdaBoostR_grid.best_estimator_)
 ])) ])
 best_model = best_model.fit(train,y)
 submission = pd.read_csv("/kaggle/input/restaurant-revenue-prediction/sampleSubmission.csv")
 submission.iloc[:,1] = best_model.predict(test)
 submission.to_csv('submission', index=False)
 submission<set_options>
history[history['pvalue']<.05]
import numpy as np
import pandas as pd
from sklearn.neighbors import KNeighborsRegressor
import seaborn as sns; sns.set(style="ticks", color_codes=True )<load_from_csv>
sc=MinMaxScaler()
train_data=sc.fit_transform(train_data)
 test_data=sc.transform(test_data)
 train_data.shape[1]
file_list = [
'/kaggle/input/restaurant-revenue-prediction/test.csv.zip',
'/kaggle/input/restaurant-revenue-prediction/train.csv.zip']
for file_name in file_list:
with zipfile.ZipFile(file=file_name)as target_zip:
target_zip.extractall()<load_from_csv>
model=keras.Sequential([keras.layers.Dense(500,input_shape=[train_data.shape[1]])
 ,keras.layers.Dense(400,activation='relu')
 ,keras.layers.Dense(300,activation='relu')
 ,keras.layers.Dense(200,activation='relu')
 ,keras.layers.Dense(100,activation='relu')
 ,keras.layers.Dense(1,activation='linear')
 ] )
train = pd.read_csv("/kaggle/working/train.csv")
 test = pd.read_csv("/kaggle/working/test.csv")
<categorify>
model.compile(optimizer='adam',loss='mean_squared_error',metrics=['mean_squared_error'])
 model.fit(train_data,y_train,batch_size=32,verbose=1,epochs=1000)
train = pd.get_dummies(train, columns=["City Group","Type"])
 test = pd.get_dummies(test, columns=["City Group","Type"])
<compute_test_metric>
y_pre=model.predict(test_data)
 test_id=test_id.astype(np.int32 )
train2 = train.copy()
for i in range(36):
train2['P'+str(i+1)] = scipy.stats.zscore(train2['P'+str(i+1)])
 train2.head() 
<split>
cols = ['Id', 'SalePrice']
submit_df = pd.DataFrame(np.hstack(( test_id.reshape(-1,1),np.exp(y_pre.reshape(-1,1)))) ,columns=cols,dtype='Int32' )
<save_to_csv><EOS> 
submit_df.to_csv('submission.csv', index=False )