Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
pd.set_option('display.max_columns', None )
class BERTModel(tf.keras.Model):

 def __init__(self,
batch_size=64,
units=40,
embed_dim=100,
sequence_length=len(word_cols), 
):

 super(BERTModel, self ).__init__()
self.inps = [ 
(None, sequence_length), 
(None, sequence_length), 
(None, sequence_length),
]
self.bs = batch_size
out_dim = 2
self.max_seq_length = sequence_length
self.bert_layer = bert_layer
self.input_word_ids = tf.keras.Input(
 shape=(self.max_seq_length,),
dtype=tf.int32,
name='input_word_ids', 
)
 self.input_m = tf.keras.Input(
 shape=(self.max_seq_length,),
dtype=tf.int32,
name='input_mask', 
)
 self.segment_ids = tf.keras.Input(
 shape=(self.max_seq_length,),
dtype=tf.int32,
name='segment_ids', 
)
 self._dense0 = tf.keras.layers.Dense(units, activation=tf.nn.relu)
 self._do0 = tf.keras.layers.Dropout(0.5)
 self._dense2 = tf.keras.layers.Dense(
 out_dim,
activation=tf.nn.softmax,
name="final", 
)
 self._optimizer = tf.keras.optimizers.Adam(learning_rate=0.00001)
 self._metrics = [tf.keras.metrics.BinaryAccuracy() ]
self._loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
 self.compile(
 optimizer=self._optimizer,
loss=self._loss,
metrics=self._metrics, 
)
 self.inputs =(self.input_word_ids, self.input_m, self.segment_ids)
 self.outputs = self._dense2
def call(self, inputs, training=False):
self.input_word_ids, self.input_m, self.segment_ids = inputs
pooled_output, sequence_output = self.bert_layer(
 [
self.input_word_ids,
self.input_m,
self.segment_ids,
] 
)
 clf_output = sequence_output[:, 0, :]
z = self._dense0(clf_output)
 if training:
z = self._do0(z, training=training)
 out = self._dense2(z)
 return out<choose_model_class>
mydata = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 mydata.drop(mydata[(mydata['OverallQual']<5)&(mydata['SalePrice']>200000)].index, inplace=True)
 mydata.drop(mydata[(mydata['GrLivArea']>4000)&(mydata['SalePrice']<300000)].index, inplace=True)
 mydata.reset_index(drop=True, inplace=True)
 mydata['MSSubClass'] = mydata['MSSubClass'].apply(str)
 mydata['YrSold'] = mydata['YrSold'].astype(str)
 mydata['MoSold'] = mydata['MoSold'].astype(str)
tfboard_dir = "logs"
if not os.path.exists(tfboard_dir):
os.mkdir(tfboard_dir)
 tensorboard_callback = tf.keras.callbacks.TensorBoard(
 log_dir=tfboard_dir,
histogram_freq=1,
write_graph=True,
write_images=True, 
)
 early_stopping = tf.keras.callbacks.EarlyStopping(
 monitor="val_binary_accuracy",
min_delta=1e-5,
patience=10,
baseline=0.5,
restore_best_weights=True,
)<data_type_conversions>
model = BERTModel(batch_size=32)
 df_test = df_full.iloc[train_pts:]
df_train = df_full.iloc[:train_pts]
log.info(f"Dataset size: {df_train.shape[0]}")
 remainder = df_train.shape[0] % model.bs
pad_size = model.bs - remainder if remainder !=0 else 0
log.info(f"Remainder from batch size: {remainder}
"
f"Padding {pad_size} elements." 
)
 X1 = df_train[word_cols].values.astype(int)
 X2 = df_train[mask_cols].values.astype(int)
 X3 = df_train[type_cols].values.astype(int)
 X_unpad =(
 tf.convert_to_tensor(X1, dtype=tf.dtypes.int32, name='input_word_ids'),
tf.convert_to_tensor(X2, dtype=tf.dtypes.int32, name='input_mask'),
tf.convert_to_tensor(X3, dtype=tf.dtypes.int32, name='input_type_ids'), 
)
 if pad_size > 0:
X1 = np.vstack([X1, np.zeros(( pad_size, X1.shape[1])).astype(int)])
 X2 = np.vstack([X2, np.zeros(( pad_size, X1.shape[1])).astype(int)])
 X3 = np.vstack([X3, np.zeros(( pad_size, X1.shape[1])).astype(int)])
 X1 = tf.convert_to_tensor(X1, dtype=tf.dtypes.int32, name='input_word_ids')
 X2 = tf.convert_to_tensor(X2, dtype=tf.dtypes.int32, name='input_mask')
 X3 = tf.convert_to_tensor(X3, dtype=tf.dtypes.int32, name='input_type_ids')
 df_train[y_cols] = pd.get_dummies(df_train[target], prefix=target)
 Y = df_train[y_cols].values
Y_add = np.zeros(( pad_size, 2))
Y_add[:, 0] = 1.0
if pad_size > 0:
Y = np.vstack([Y, Y_add])
 X_val =(X1, X2, X3)
 X =(
 tf.data.Dataset.from_tensor_slices(( X_val, Y)) 
.batch(model.bs, drop_remainder=True)
)
 X = X.shuffle(buffer_size=10000)
 q = 5
p = 1
select = lambda x, y:(x % q <= p)
 nselect = lambda x, y: ~(x % q <= p)
 take = lambda x, y: y
X_train = X.enumerate().filter(nselect ).map(take)
 X_valid = X.enumerate().filter(select ).map(take)
 log.info(f"Test dataset size: {df_test.shape}")
 Z1 = df_test[word_cols].values.astype(int)
 Z2 = df_test[mask_cols].values.astype(int)
 Z3 = df_test[type_cols].values.astype(int)
 Z1 = tf.convert_to_tensor(Z1, dtype=tf.int32, name='input_word_ids')
 Z2 = tf.convert_to_tensor(Z2, dtype=tf.int32, name='input_mask')
 Z3 = tf.convert_to_tensor(Z3, dtype=tf.int32, name='input_type_ids')
 X_test =(Z1, Z2, Z3 )<train_model>
def fill_missings(res):
res['Alley'] = res['Alley'].fillna('missing')
 res['PoolQC'] = res['PoolQC'].fillna(res['PoolQC'].mode() [0])
 res['MasVnrType'] = res['MasVnrType'].fillna('None')
 res['BsmtQual'] = res['BsmtQual'].fillna(res['BsmtQual'].mode() [0])
 res['BsmtCond'] = res['BsmtCond'].fillna(res['BsmtCond'].mode() [0])
 res['FireplaceQu'] = res['FireplaceQu'].fillna(res['FireplaceQu'].mode() [0])
 res['GarageType'] = res['GarageType'].fillna('missing')
 res['GarageFinish'] = res['GarageFinish'].fillna(res['GarageFinish'].mode() [0])
 res['GarageQual'] = res['GarageQual'].fillna(res['GarageQual'].mode() [0])
 res['GarageCond'] = res['GarageCond'].fillna('missing')
 res['Fence'] = res['Fence'].fillna('missing')
 res['Street'] = res['Street'].fillna('missing')
 res['LotShape'] = res['LotShape'].fillna('missing')
 res['LandContour'] = res['LandContour'].fillna('missing')
 res['BsmtExposure'] = res['BsmtExposure'].fillna(res['BsmtExposure'].mode() [0])
 res['BsmtFinType1'] = res['BsmtFinType1'].fillna('missing')
 res['BsmtFinType2'] = res['BsmtFinType2'].fillna('missing')
 res['CentralAir'] = res['CentralAir'].fillna('missing')
 res['Electrical'] = res['Electrical'].fillna(res['Electrical'].mode() [0])
 res['MiscFeature'] = res['MiscFeature'].fillna('missing')
 res['MSZoning'] = res['MSZoning'].fillna(res['MSZoning'].mode() [0])
 res['Utilities'] = res['Utilities'].fillna('missing')
 res['Exterior1st'] = res['Exterior1st'].fillna(res['Exterior1st'].mode() [0])
 res['Exterior2nd'] = res['Exterior2nd'].fillna(res['Exterior2nd'].mode() [0])
 res['KitchenQual'] = res['KitchenQual'].fillna(res['KitchenQual'].mode() [0])
 res["Functional"] = res["Functional"].fillna("Typ")
 res['SaleType'] = res['SaleType'].fillna(res['SaleType'].mode() [0])
 res['SaleCondition'] = res['SaleCondition'].fillna('missing')
 flist = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal']
for fl in flist:
res[fl] = res[fl].fillna(0)
 res['TotalBsmtSF'] = res['TotalBsmtSF'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 res['2ndFlrSF'] = res['2ndFlrSF'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 res['GarageArea'] = res['GarageArea'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 res['GarageCars'] = res['GarageCars'].apply(lambda x: 0 if x <= 0.0 else x)
 res['LotFrontage'] = res['LotFrontage'].apply(lambda x: np.exp(4.2)if x <= 0.0 else x)
 res['MasVnrArea'] = res['MasVnrArea'].apply(lambda x: np.exp(4)if x <= 0.0 else x)
 res['BsmtFinSF1'] = res['BsmtFinSF1'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 return res
hist = model.fit(
 X_train,
epochs=8,
validation_data=X_valid,
callbacks=[
early_stopping,
],
)<predict_on_test>
mydata = fill_missings(mydata)
 mydata['TotalSF'] = mydata['TotalBsmtSF'] + mydata['1stFlrSF'] + mydata['2ndFlrSF']
def QualToInt(x):
if(x=='Ex'):
r = 0
elif(x=='Gd'):
r = 1
elif(x=='TA'):
r = 2
elif(x=='Fa'):
r = 3
elif(x=='missing'):
r = 4
else:
r = 5
return r
mydata['ExterQual'] = mydata['ExterQual'].apply(QualToInt)
 mydata['ExterCond'] = mydata['ExterCond'].apply(QualToInt)
 mydata['KitchenQual'] = mydata['KitchenQual'].apply(QualToInt)
 mydata['HeatingQC'] = mydata['HeatingQC'].apply(QualToInt)
 mydata['BsmtQual'] = mydata['BsmtQual'].apply(QualToInt)
 mydata['BsmtCond'] = mydata['BsmtCond'].apply(QualToInt)
 mydata['FireplaceQu'] = mydata['FireplaceQu'].apply(QualToInt)
 mydata['GarageQual'] = mydata['GarageQual'].apply(QualToInt)
 mydata['PoolQC'] = mydata['PoolQC'].apply(QualToInt)
 def SlopeToInt(x):
if(x=='Gtl'):
r = 0
elif(x=='Mod'):
r = 1
elif(x=='Sev'):
r = 2
else:
r = 3
return r
mydata['LandSlope'] = mydata['LandSlope'].apply(SlopeToInt)
 mydata['CentralAir'] = mydata['CentralAir'].apply(lambda x: 0 if x == 'N' else 1)
 mydata['Street'] = mydata['Street'].apply(lambda x: 0 if x == 'Pave' else 1)
 mydata['PavedDrive'] = mydata['PavedDrive'].apply(lambda x: 0 if x == 'Y' else 1)
 def GFinishToInt(x):
if(x=='Fin'):
r = 0
elif(x=='RFn'):
r = 1
elif(x=='Unf'):
r = 2
else:
r = 3
return r
mydata['GarageFinish'] = mydata['GarageFinish'].apply(GFinishToInt)
 def BsmtExposureToInt(x):
if(x=='Gd'):
r = 0
elif(x=='Av'):
r = 1
elif(x=='Mn'):
r = 2
elif(x=='No'):
r = 3
else:
r = 4
return r
mydata['BsmtExposure'] = mydata['BsmtExposure'].apply(BsmtExposureToInt)
 def FunctionalToInt(x):
if(x=='Typ'):
r = 0
elif(x=='Min1'):
r = 1
elif(x=='Min2'):
r = 1
else:
r = 2
return r
mydata['Functional_int'] = mydata['Functional'].apply(FunctionalToInt)
 def HouseStyleToInt(x):
if(x=='1.5Unf'):
r = 0
elif(x=='SFoyer'):
r = 1
elif(x=='1.5Fin'):
r = 2
elif(x=='2.5Unf'):
r = 3
elif(x=='SLvl'):
r = 4
elif(x=='1Story'):
r = 5
elif(x=='2Story'):
r = 6
elif(x==' 2.5Fin'):
r = 7
else:
r = 8
return r
mydata['HouseStyle_int'] = mydata['HouseStyle'].apply(HouseStyleToInt)
 mydata['HouseStyle_1st'] = 1*(mydata['HouseStyle'] == '1Story')
 mydata['HouseStyle_2st'] = 1*(mydata['HouseStyle'] == '2Story')
 mydata['HouseStyle_15st'] = 1*(mydata['HouseStyle'] == '1.5Fin')
 def FoundationToInt(x):
if(x=='PConc'):
r = 3
elif(x=='CBlock'):
r = 2
elif(x=='BrkTil'):
r = 1
else:
r = 0
return r
mydata['Foundation_int'] = mydata['Foundation'].apply(FoundationToInt)
 def MasVnrTypeToInt(x):
if(x=='Stone'):
r = 3
elif(x=='BrkFace'):
r = 2
elif(x=='BrkCmn'):
r = 1
else:
r = 0
return r
mydata['MasVnrType_int'] = mydata['MasVnrType'].apply(MasVnrTypeToInt)
 def BsmtFinType1ToInt(x):
if(x=='GLQ'):
r = 6
elif(x=='ALQ'):
r = 5
elif(x=='BLQ'):
r = 4
elif(x=='Rec'):
r = 3
elif(x=='LwQ'):
r = 2
elif(x=='Unf'):
r = 1
else:
r = 0
return r
mydata['BsmtFinType1_int'] = mydata['BsmtFinType1'].apply(BsmtFinType1ToInt)
 mydata['BsmtFinType1_Unf'] = 1*(mydata['BsmtFinType1'] == 'Unf')
 mydata['HasWoodDeck'] =(mydata['WoodDeckSF'] == 0)* 1
mydata['HasOpenPorch'] =(mydata['OpenPorchSF'] == 0)* 1
mydata['HasEnclosedPorch'] =(mydata['EnclosedPorch'] == 0)* 1
mydata['Has3SsnPorch'] =(mydata['3SsnPorch'] == 0)* 1
mydata['HasScreenPorch'] =(mydata['ScreenPorch'] == 0)* 1
mydata['YearsSinceRemodel'] = mydata['YrSold'].astype(int)- mydata['YearRemodAdd'].astype(int)
 mydata['Total_Home_Quality'] = mydata['OverallQual'] + mydata['OverallCond']
Y_train_pred = model.predict(X_unpad)
 Y_test_pred = model.predict(X_test )<prepare_output>
def addlogs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
loglist = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YearRemodAdd','TotalSF']
mydata = addlogs(mydata, loglist )
df_train_pred = pd.DataFrame(Y_train_pred, columns=y_cols)
 df_train_pred = df_train_pred.apply(np.round ).astype({x: int for x in y_cols})
 df_train_pred[target] = df_train_pred["target_1"]
df_train_pred.drop(y_cols, inplace=True, axis=1)
 df_train_pred["id"] = df_train["id"].values
df_train_pred = df_train_pred[["id", target]]<data_type_conversions>
df_test_pred = pd.DataFrame(Y_test_pred, columns=y_cols)
 df_test_pred = df_test_pred.apply(np.round ).astype({x: int for x in y_cols})
 df_test_pred[target] = df_test_pred["target_1"]
df_test_pred.drop(y_cols, inplace=True, axis=1)
 df_test_pred.drop(list(df_test_pred.index[df_train.shape[0]:]), inplace=True, axis=0)
 df_test_pred["id"] = df_test["id"].values
df_test_pred = df_test_pred[["id", target]]<compute_train_metric>
def getdummies(res, ls):
def encode(encode_df):
encode_df = np.array(encode_df)
 enc = OneHotEncoder()
le = LabelEncoder()
le.fit(encode_df)
 res1 = le.transform(encode_df ).reshape(-1, 1)
 enc.fit(res1)
 return pd.DataFrame(enc.transform(res1 ).toarray()), le, enc
decoder = []
outres = pd.DataFrame({'A' : []})
 for l in ls:
cat, le, enc = encode(res[l])
 cat.columns = [l+str(x)for x in cat.columns]
outres.reset_index(drop=True, inplace=True)
 outres = pd.concat([outres, cat], axis = 1)
 decoder.append([le,enc])
 return(outres, decoder)
 catpredlist = ['MSSubClass','MSZoning','LotShape','LandContour','LotConfig',
'Neighborhood','Condition1','Condition2','BldgType',
'RoofStyle','RoofMatl','Exterior1st','Exterior2nd',
'BsmtFinType2','Heating','HouseStyle','Foundation','MasVnrType','BsmtFinType1',
'Electrical','Functional','GarageType','Alley','Utilities',
'GarageCond','Fence','MiscFeature','SaleType','SaleCondition','LandSlope','CentralAir',
'GarageFinish','BsmtExposure','Street']
res = getdummies(mydata[catpredlist],catpredlist)
 df = res[0]
decoder = res[1]
floatpredlist = ['LotFrontage_log',
'LotArea_log',
'MasVnrArea_log','BsmtFinSF1_log','BsmtFinSF2_log','BsmtUnfSF_log',
'TotalBsmtSF_log','1stFlrSF_log','2ndFlrSF_log','LowQualFinSF_log','GrLivArea_log',
'BsmtFullBath_log','BsmtHalfBath_log','FullBath_log','HalfBath_log','BedroomAbvGr_log','KitchenAbvGr_log',
'TotRmsAbvGrd_log','Fireplaces_log','GarageCars_log','GarageArea_log',
'PoolArea_log','MiscVal_log',
'YearRemodAdd','TotalSF_log','OverallQual','OverallCond','ExterQual','ExterCond','KitchenQual',
'HeatingQC','BsmtQual','BsmtCond','FireplaceQu','GarageQual','PoolQC','PavedDrive',
'HasWoodDeck', 'HasOpenPorch','HasEnclosedPorch', 'Has3SsnPorch', 'HasScreenPorch']
df = pd.concat([df,mydata[floatpredlist]],axis=1)
log.info("
" +
sklearn.metrics.classification_report(
 df_train[target],
df_train_pred[target],
target_names=["Not disaster", "Disaster"] 
)
 )<compute_test_metric>
log.info("Training accuracy score {}.".format(
 sklearn.metrics.accuracy_score(df_train[target], df_train_pred[target])
)
 )<save_to_csv>
def addSquared(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(res[l]*res[l] ).values)
 res.columns.values[m] = l + '_sq'
m += 1
return res
sqpredlist = ['YearRemodAdd', 'LotFrontage_log',
'TotalBsmtSF_log', '1stFlrSF_log', '2ndFlrSF_log', 'GrLivArea_log',
'GarageCars_log', 'GarageArea_log',
'OverallQual','ExterQual','BsmtQual','GarageQual','FireplaceQu','KitchenQual']
df = addSquared(df, sqpredlist)
output_dir = "./"
results_bn = "results.csv"
results_fn = os.path.join(output_dir, results_bn)
 df_test_pred.to_csv(results_fn, index=False )<import_modules>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
import spacy
import nltk
import pickle
from nltk.tokenize import RegexpTokenizer
from scipy import sparse
from sklearn.naive_bayes import BernoulliNB
from sklearn.linear_model import LogisticRegressionCV
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import cross_val_score
from sklearn.metrics import f1_score
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer<load_from_csv>
X = np.array(df)
 X = np.delete(X, 0, axis=1)
 y = np.log(1+np.array(mydata['SalePrice']))
train = pd.read_csv(r'.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv(r'.. /input/nlp-getting-started/test.csv')
 train.head()<count_values>
nF = 20
kf = KFold(n_splits=nF, random_state=241, shuffle=True)
 test_errors_l2 = []
train_errors_l2 = []
test_errors_l1 = []
train_errors_l1 = []
test_errors_GBR = []
train_errors_GBR = []
test_errors_ENet = []
test_errors_LGB = []
test_errors_stack = []
test_errors_ens = []
train_errors_ens = []
models = []
pred_all = []
ifold = 1
for train_index, test_index in kf.split(X):
print('fold: ',ifold)
 ifold = ifold + 1
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
l2Regr = Ridge(alpha=9.0, fit_intercept = True)
 l2Regr.fit(X_train, y_train)
 pred_train_l2 = l2Regr.predict(X_train)
 pred_test_l2 = l2Regr.predict(X_test)
 l1Regr = make_pipeline(RobustScaler() , Lasso(alpha = 0.0003, random_state=1, max_iter=50000))
l1Regr.fit(X_train, y_train)
 pred_train_l1 = l1Regr.predict(X_train)
 pred_test_l1 = l1Regr.predict(X_test)
 myGBR = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.02,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=50,
loss='huber', random_state = 5)
 myGBR.fit(X_train,y_train)
 pred_train_GBR = myGBR.predict(X_train)
 pred_test_GBR = myGBR.predict(X_test)
 ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=4.0, l1_ratio=0.005, random_state=3))
ENet.fit(X_train, y_train)
 pred_train_ENet = ENet.predict(X_train)
 pred_test_ENet = ENet.predict(X_test)
 myLGB = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=600,
max_bin = 50, bagging_fraction = 0.6,
bagging_freq = 5, feature_fraction = 0.25,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf = 6, min_sum_hessian_in_leaf = 11)
 myLGB.fit(X_train, y_train)
 pred_train_LGB = myLGB.predict(X_train)
 pred_test_LGB = myLGB.predict(X_test)
 stackedset = pd.DataFrame({'A' : []})
 stackedset = pd.concat([stackedset,pd.DataFrame(pred_test_l2)],axis=1)
 stackedset = pd.concat([stackedset,pd.DataFrame(pred_test_l1)],axis=1)
 stackedset = pd.concat([stackedset,pd.DataFrame(pred_test_GBR)],axis=1)
 stackedset = pd.concat([stackedset,pd.DataFrame(pred_test_ENet)],axis=1)
 stackedset = pd.concat([stackedset,pd.DataFrame(pred_test_LGB)],axis=1)
 prod =(pred_test_l2*pred_test_l1*pred_test_GBR*pred_test_ENet*pred_test_LGB)**(1.0/5.0)
 stackedset = pd.concat([stackedset,pd.DataFrame(prod)],axis=1)
 Xstack = np.array(stackedset)
 Xstack = np.delete(Xstack, 0, axis=1)
 l1_staked = Lasso(alpha = 0.0001,fit_intercept = True)
 l1_staked.fit(Xstack, y_test)
 pred_test_stack = l1_staked.predict(Xstack)
 models.append([l2Regr,l1Regr,myGBR,ENet,myLGB,l1_staked])
 test_errors_l2.append(np.square(pred_test_l2 - y_test ).mean() ** 0.5)
 test_errors_l1.append(np.square(pred_test_l1 - y_test ).mean() ** 0.5)
 test_errors_GBR.append(np.square(pred_test_GBR - y_test ).mean() ** 0.5)
 test_errors_ENet.append(np.square(pred_test_ENet - y_test ).mean() ** 0.5)
 test_errors_LGB.append(np.square(pred_test_LGB - y_test ).mean() ** 0.5)
 test_errors_stack.append(np.square(pred_test_stack - y_test ).mean() ** 0.5)
plt.bar(train['target'].value_counts().index, train['target'].value_counts().values )<count_values>
print(np.mean(test_errors_l2))
print(np.mean(test_errors_l1))
print(np.mean(test_errors_GBR))
print(np.mean(test_errors_ENet))
print(np.mean(test_errors_LGB))
print(np.mean(test_errors_stack))
train['location'].value_counts(dropna=False )<count_values>
scoredata = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 scoredata['MSSubClass'] = scoredata['MSSubClass'].apply(str)
 scoredata['YrSold'] = scoredata['YrSold'].astype(str)
 scoredata['MoSold'] = scoredata['MoSold'].astype(str)
train_for_plot = train.fillna('NOINFO')
 locations = train_for_plot['location'].value_counts(dropna=False)
 freq_locations = list(locations.index )<concatenate>
scoredata = fill_missings(scoredata)
 scoredata['ExterQual'] = scoredata['ExterQual'].apply(QualToInt)
 scoredata['ExterCond'] = scoredata['ExterCond'].apply(QualToInt)
 scoredata['KitchenQual'] = scoredata['KitchenQual'].apply(QualToInt)
 scoredata['HeatingQC'] = scoredata['HeatingQC'].apply(QualToInt)
 scoredata['BsmtQual'] = scoredata['BsmtQual'].apply(QualToInt)
 scoredata['BsmtCond'] = scoredata['BsmtCond'].apply(QualToInt)
 scoredata['FireplaceQu'] = scoredata['FireplaceQu'].apply(QualToInt)
 scoredata['GarageQual'] = scoredata['GarageQual'].apply(QualToInt)
 scoredata['PoolQC'] = scoredata['PoolQC'].apply(QualToInt)
 scoredata['LandSlope'] = scoredata['LandSlope'].apply(SlopeToInt)
 scoredata['CentralAir'] = scoredata['CentralAir'].apply(lambda x: 0 if x == 'N' else 1)
 scoredata['Street'] = scoredata['Street'].apply(lambda x: 0 if x == 'Grvl' else 1)
 scoredata['GarageFinish'] = scoredata['GarageFinish'].apply(GFinishToInt)
 scoredata['BsmtExposure'] = scoredata['BsmtExposure'].apply(BsmtExposureToInt)
 scoredata['TotalSF'] = scoredata['TotalBsmtSF'] + scoredata['1stFlrSF'] + scoredata['2ndFlrSF']
scoredata['TotalSF'] = scoredata['TotalSF'].fillna(0)
 scoredata['Functional_int'] = scoredata['Functional'].apply(FunctionalToInt)
 scoredata['HouseStyle_int'] = scoredata['HouseStyle'].apply(HouseStyleToInt)
 scoredata['HouseStyle_1st'] = 1*(scoredata['HouseStyle'] == '1Story')
 scoredata['HouseStyle_2st'] = 1*(scoredata['HouseStyle'] == '2Story')
 scoredata['HouseStyle_15st'] = 1*(scoredata['HouseStyle'] == '1.5Fin')
 scoredata['Foundation_int'] = scoredata['Foundation'].apply(FoundationToInt)
 scoredata['MasVnrType_int'] = scoredata['MasVnrType'].apply(MasVnrTypeToInt)
 scoredata['BsmtFinType1_int'] = scoredata['BsmtFinType1'].apply(BsmtFinType1ToInt)
 scoredata['BsmtFinType1_Unf'] = 1*(scoredata['BsmtFinType1'] == 'Unf')
 scoredata['PavedDrive'] = scoredata['PavedDrive'].apply(lambda x: 0 if x == 'Y' else 1)
 scoredata['HasWoodDeck'] =(scoredata['WoodDeckSF'] == 0)* 1
scoredata['HasOpenPorch'] =(scoredata['OpenPorchSF'] == 0)* 1
scoredata['HasEnclosedPorch'] =(scoredata['EnclosedPorch'] == 0)* 1
scoredata['Has3SsnPorch'] =(scoredata['3SsnPorch'] == 0)* 1
scoredata['HasScreenPorch'] =(scoredata['ScreenPorch'] == 0)* 1
scoredata['Total_Home_Quality'] = scoredata['OverallQual'] + scoredata['OverallCond']
frames = [train, test]
full_data = pd.concat(frames )<count_values>
scoredata['MSSubClass'] = scoredata['MSSubClass'].apply(lambda x: '20' if x == '150' else x)
 scoredata['MSZoning'] = scoredata['MSZoning'].apply(lambda x: 'RL' if x == 'missing' else x)
 scoredata['Utilities'] = scoredata['Utilities'].apply(lambda x: 'AllPub' if x == 'missing' else x)
 scoredata['Exterior1st'] = scoredata['Exterior1st'].apply(lambda x: 'VinylSd' if x == 'missing' else x)
 scoredata['Exterior2nd'] = scoredata['Exterior2nd'].apply(lambda x: 'VinylSd' if x == 'missing' else x)
 scoredata['Functional'] = scoredata['Functional'].apply(lambda x: 'Typ' if x == 'missing' else x)
 scoredata['SaleType'] = scoredata['SaleType'].apply(lambda x: 'WD' if x == 'missing' else x)
 scoredata['SaleCondition'] = scoredata['SaleCondition'].apply(lambda x: 'Normal' if x == 'missing' else x)
locs = full_data['location'].value_counts(dropna=True )<filter>
scoredata = addlogs(scoredata, loglist )
locations_nan = locs[locs > 8]
print(locations_nan.index )<feature_engineering>
def change_location(dataset, name='dataset'):
dataset['location'] = dataset['location'].replace('United States', 'USA')
 dataset['location'] = dataset['location'].replace('US', 'USA')
 dataset['location'] = dataset['location'].replace('Worldwide', 'Anywhere')
 dataset['location'] = dataset['location'].replace('worldwide', 'Anywhere')
 dataset['location'] = dataset['location'].replace('Everywhere', 'Anywhere')
 dataset['location'] = dataset['location'].replace('California, USA', 'California')
 dataset['location'] = dataset['location'].replace('United Kingdom', 'UK')
 dataset['location'] = dataset['location'].replace('San Francisco, CA', 'San Francisco')
 dataset['location'] = dataset['location'].replace('Chicago, IL', 'Chicago')
 dataset['location'] = dataset['location'].replace('New York, NY', 'New York')
 dataset['location'] = dataset['location'].replace('Los Angeles, CA', 'Los Angeles')
 dataset['location'] = dataset['location'].replace('NYC', 'New York')
 dataset['location'] = dataset['location'].replace('New York City', 'New York')
 dataset['location'] = dataset['location'].replace('Earth', 'Anywhere')
 dataset['location'] = dataset['location'].replace('ss', 'Anywhere')
 dataset['location'] = dataset['location'].replace('304', 'Anywhere')
 dataset['location'] = dataset['location'].replace('World', 'Anywhere')
 dataset['location'] = dataset['location'].replace('??????', 'Anywhere')
 dataset['location'] = dataset['location'].replace('Planet Earth', 'Anywhere')
 dataset['location'] = dataset['location'].replace('Washington, D.C.', 'Washington, DC')
 dataset['location'] = dataset['location'].replace('Denver, Colorado', 'Denver')
 dataset['location'] = dataset['location'].replace('Denver, CO', 'Denver')
 dataset['location'] = dataset['location'].replace(' Road to the Billionaires Club', 'Anywhere')
 dataset['location'] = dataset['location'].replace('Pennsylvania, USA', 'Pennsylvania')
 dataset['location'] = dataset['location'].replace('Paterson, New Jersey', 'Paterson')
 locations = dataset['location'].value_counts(dropna=True)
 anywhere_locations = locations[locations <= 8]
for loc in list(anywhere_locations.index):
dataset['location'] = dataset['location'].replace(loc, 'noinfo')
 dataset['location'].fillna('noinfo', inplace=True)
 change_location(train, 'train')
 change_location(test, 'test')
 frames = [train, test]
full_data = pd.concat(frames)
 print(full_data['location'].value_counts() )<count_values>
def getdummies_transform(res, ls, decoder):
def encode(encode_df, le_df, enc_df):
encode_df = np.array(encode_df)
 res1 = le_df.transform(encode_df ).reshape(-1, 1)
 return pd.DataFrame(enc_df.transform(res1 ).toarray())
 L = len(ls)
 outres = pd.DataFrame({'A' : []})
 for j in range(L):
l = ls[j]
le = decoder[j][0]
enc = decoder[j][1]
cat = encode(res[l], le, enc)
 cat.columns = [l+str(x)for x in cat.columns]
outres.reset_index(drop=True, inplace=True)
 outres = pd.concat([outres, cat], axis = 1)
 return outres
df_scores = getdummies_transform(scoredata, catpredlist, decoder)
 df_scores = pd.concat([df_scores,scoredata[floatpredlist]],axis=1)
 df_scores = addSquared(df_scores, sqpredlist)
keywords = train_for_plot['keyword'].value_counts(dropna=False)
 keywords<count_values>
train['target'].value_counts() [1] / train['target'].value_counts() [0]<train_on_grid>
X_score = np.array(df_scores)
 X_score = np.delete(X_score, 0, axis=1 )
np.random.seed(144)
 train.fillna('noinfo', inplace=True)
 test.fillna('noinfo', inplace=True)
 shuffled_train = train.iloc[np.random.permutation(len(train)) ]
max_iter = 100
cv = 5
clf = LogisticRegressionCV(cv = cv, n_jobs = -1, max_iter = max_iter, scoring='f1')
 X = shuffled_train[['location', 'keyword']]
Y = shuffled_train['target']
enc = OneHotEncoder()
X = enc.fit_transform(X)
 clf.fit(X, Y)
 print('Best mean score from 5 cross validations: ')
 print(clf.scores_[1].mean(axis=0 ).max() )<compute_train_metric>
M = X_score.shape[0]
scores_fin = 1+np.zeros(M)
 for md in models:
l2 = md[0]
l1 = md[1]
GBR = md[2]
ENet = md[3]
LGB = md[4]
l1_stacked = md[5]
l2_scores = l2.predict(X_score)
 l1_scores = l1.predict(X_score)
 GBR_scores = GBR.predict(X_score)
 ENet_scores = ENet.predict(X_score)
 LGB_scores = LGB.predict(X_score)
 stackedsets = pd.DataFrame({'A' : []})
 stackedsets = pd.concat([stackedsets,pd.DataFrame(l2_scores)],axis=1)
 stackedsets = pd.concat([stackedsets,pd.DataFrame(l1_scores)],axis=1)
 stackedsets = pd.concat([stackedsets,pd.DataFrame(GBR_scores)],axis=1)
 stackedsets = pd.concat([stackedsets,pd.DataFrame(ENet_scores)],axis=1)
 stackedsets = pd.concat([stackedsets,pd.DataFrame(LGB_scores)],axis=1)
 prod =(l2_scores*l1_scores*GBR_scores*ENet_scores*LGB_scores)**(1.0/5.0)
 stackedsets = pd.concat([stackedsets,pd.DataFrame(prod)],axis=1)
 Xstacks = np.array(stackedsets)
 Xstacks = np.delete(Xstacks, 0, axis=1)
 scores_fin = scores_fin * l1_stacked.predict(Xstacks)
 scores_fin = scores_fin **(1/nF)
np.random.seed(144)
 shuffled_train = train.iloc[np.random.permutation(len(train)) ]
clf = BernoulliNB(fit_prior = False)
 X = shuffled_train[['location', 'keyword']]
Y = shuffled_train['target']
enc = OneHotEncoder()
X = enc.fit_transform(X)
 score = cross_val_score(estimator=clf, X=X, y=Y, scoring='f1')
 print('Best mean score from 5 cross validations: ')
 print(score.mean() )<feature_engineering>
train['text'] = train['text'].str.lower()
test['text'] = test['text'].str.lower()<feature_engineering>
train['text'] = train['text'].apply(lambda x:
re.sub(r'https?://\S+|www\.\S+','', x))
test['text'] = test['text'].apply(lambda x:
re.sub(r'https?://\S+|www\.\S+','', x))<feature_engineering>
svm_solution = pd.read_csv('.. /input/svm-solution-32/svm_solution_32.csv')
 svm_solution_ln = np.log(svm_solution['SalePrice'])
 fin_score = np.sqrt(scores_fin * svm_solution_ln)
 Id = scoredata['Id']
fin_score = pd.DataFrame({'SalePrice': np.exp(fin_score)-1})
 fin_data = pd.concat([Id,fin_score],axis=1 )
train['text'] = train['text'].apply(lambda x:
re.sub(r'@[A-Za-z0-9]+','', x))
test['text'] = test['text'].apply(lambda x:
re.sub(r'@[A-Za-z0-9]+','', x))
train['text'] = train['text'].apply(lambda x:
re.sub(r'
test['text'] = test['text'].apply(lambda x:
re.sub(r'<feature_engineering>
q1 = fin_data['SalePrice'].quantile(0.0042)
 q2 = fin_data['SalePrice'].quantile(0.99)
 fin_data['SalePrice'] = fin_data['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 fin_data['SalePrice'] = fin_data['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
train['text'] = train['text'].apply(lambda x:
re.sub(r'[^\w\s]','', x))
test['text'] = test['text'].apply(lambda x:
re.sub(r'[^\w\s]','', x))<feature_engineering>
w_tokenizer = nltk.tokenize.WhitespaceTokenizer()
lemmatizer = nltk.stem.WordNetLemmatizer()
def lemmatize_text(text):
return ' '.join([lemmatizer.lemmatize(w)for w in w_tokenizer.tokenize(text)])
 train['text'] = train.text.apply(lambda x: lemmatize_text(x))
test['text'] = test.text.apply(lambda x: lemmatize_text(x))<string_transform>
fin_data.to_csv('House_Prices_submit.csv', sep=',', index = False )