Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<set_options>
warnings.filterwarnings('ignore' )
random.seed(42)
 init_notebook_mode(connected=True)
<load_from_csv>
train = pd.read_csv('.. /input/train.csv', index_col='Id' )
df_train = pd.read_csv('.. /input/train.csv')
 df_comp = pd.read_csv('.. /input/test.csv' )<count_missing_values>
no_features = ['Alley', 'BsmtQual', 'BsmtCond',
'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'FireplaceQu', 'GarageType', 'GarageFinish',
'GarageQual', 'GarageYrBlt', 'GarageCond', 'PoolQC',
'Fence', 'MiscFeature']
train[no_features] = train[no_features].fillna('No')
 train.loc[:, "LotFrontage_NA"] = train.LotFrontage.isnull() * 1
train.loc[:, "LotFrontage_NA"] = train.loc[:, "LotFrontage_NA"].astype("object" )
df_train.isnull().sum().sum()<import_modules>
def HasFeat(var):
hasf = var
hasf[hasf != 'No'] = 1
hasf[hasf == 'No'] = 0
return hasf
def feat_eng(tr):
tr.GarageQual[tr['GarageQual'] == 'No'] = 0
tr.GarageQual[tr['GarageQual'] == 'Po'] = 1
tr.GarageQual[tr['GarageQual'] == 'Fa'] = 2
tr.GarageQual[tr['GarageQual'] == 'TA'] = 3
tr.GarageQual[tr['GarageQual'] == 'Gd'] = 4
tr.GarageQual[tr['GarageQual'] == 'Ex'] = 5
tr.GarageQual = tr.GarageQual.astype('float32')
 tr.GarageCond[tr['GarageCond'] == 'No'] = 0
tr.GarageCond[tr['GarageCond'] == 'Po'] = 1
tr.GarageCond[tr['GarageCond'] == 'Fa'] = 2
tr.GarageCond[tr['GarageCond'] == 'TA'] = 3
tr.GarageCond[tr['GarageCond'] == 'Gd'] = 4
tr.GarageCond[tr['GarageCond'] == 'Ex'] = 5
tr.GarageCond = tr.GarageCond.astype('float32')
 tr.GarageYrBlt[tr['GarageYrBlt'] == 'No'] = 0
tr.GarageYrBlt = tr.GarageYrBlt.astype('float32')
 tr.Functional[tr['Functional'] == 'Typ'] = 7
tr.Functional[tr['Functional'] == 'Min1'] = 6
tr.Functional[tr['Functional'] == 'Min2'] = 5
tr.Functional[tr['Functional'] == 'Mod'] = 4
tr.Functional[tr['Functional'] == 'Maj1'] = 3
tr.Functional[tr['Functional'] == 'Maj2'] = 2
tr.Functional[tr['Functional'] == 'Sev'] = 1
tr.Functional[tr['Functional'] == 'Sal'] = 0
tr.Functional = tr.Functional.astype('float32')
 tr.Fence[tr['Fence'] == 'No'] = 0
tr.Fence[tr['Fence'] == 'MnWw'] = 1
tr.Fence[tr['Fence'] == 'GdWo'] = 2
tr.Fence[tr['Fence'] == 'MnPrv'] = 3
tr.Fence[tr['Fence'] == 'GdPrv'] = 4
tr.Fence = tr.Fence.astype('float32')
 tr.KitchenQual[tr['KitchenQual'] == 'Po'] = 1
tr.KitchenQual[tr['KitchenQual'] == 'Fa'] = 2
tr.KitchenQual[tr['KitchenQual'] == 'TA'] = 3
tr.KitchenQual[tr['KitchenQual'] == 'Gd'] = 4
tr.KitchenQual[tr['KitchenQual'] == 'Ex'] = 5
tr.KitchenQual = tr.KitchenQual.astype('float32')
 tr.HeatingQC[tr['HeatingQC'] == 'Po'] = 1
tr.HeatingQC[tr['HeatingQC'] == 'Fa'] = 2
tr.HeatingQC[tr['HeatingQC'] == 'TA'] = 3
tr.HeatingQC[tr['HeatingQC'] == 'Gd'] = 4
tr.HeatingQC[tr['HeatingQC'] == 'Ex'] = 5
tr.HeatingQC = tr.HeatingQC.astype('float32')
 tr.ExterQual[tr['ExterQual'] == 'Po'] = 1
tr.ExterQual[tr['ExterQual'] == 'Fa'] = 2
tr.ExterQual[tr['ExterQual'] == 'TA'] = 3
tr.ExterQual[tr['ExterQual'] == 'Gd'] = 4
tr.ExterQual[tr['ExterQual'] == 'Ex'] = 5
tr.ExterQual = tr.ExterQual.astype('float32')
 tr.BsmtQual[tr['BsmtQual'] == 'No'] = 0
tr.BsmtQual[tr['BsmtQual'] == 'Po'] = 1
tr.BsmtQual[tr['BsmtQual'] == 'Fa'] = 2
tr.BsmtQual[tr['BsmtQual'] == 'TA'] = 3
tr.BsmtQual[tr['BsmtQual'] == 'Gd'] = 4
tr.BsmtQual[tr['BsmtQual'] == 'Ex'] = 5
tr.BsmtQual = tr.BsmtQual.astype('float32')
 tr.BsmtFinType1[tr['BsmtFinType1'] == 'No'] = 0
tr.BsmtFinType1[tr['BsmtFinType1'] == 'Unf'] = 1
tr.BsmtFinType1[tr['BsmtFinType1'] == 'LwQ'] = 2
tr.BsmtFinType1[tr['BsmtFinType1'] == 'Rec'] = 3
tr.BsmtFinType1[tr['BsmtFinType1'] == 'BLQ'] = 4
tr.BsmtFinType1[tr['BsmtFinType1'] == 'ALQ'] = 5
tr.BsmtFinType1[tr['BsmtFinType1'] == 'GLQ'] = 6
tr.BsmtFinType1 = tr.BsmtFinType1.astype('float32')
 tr.BsmtFinType2[tr['BsmtFinType2'] == 'No'] = 0
tr.BsmtFinType2[tr['BsmtFinType2'] == 'Unf'] = 1
tr.BsmtFinType2[tr['BsmtFinType2'] == 'LwQ'] = 2
tr.BsmtFinType2[tr['BsmtFinType2'] == 'Rec'] = 3
tr.BsmtFinType2[tr['BsmtFinType2'] == 'BLQ'] = 4
tr.BsmtFinType2[tr['BsmtFinType2'] == 'ALQ'] = 5
tr.BsmtFinType2[tr['BsmtFinType2'] == 'GLQ'] = 6
tr.BsmtFinType2 = tr.BsmtFinType2.astype('float32')
 tr['TotalArea'] = tr['TotalBsmtSF'] + tr['1stFlrSF'] + tr['2ndFlrSF']
tr['BsmtFinArea'] =(tr['BsmtFinSF1'] + tr['BsmtFinSF2'])/(tr['BsmtFinSF1'] + tr['BsmtFinSF2'] + tr['BsmtUnfSF'])
 tr['MSSubClass'] = tr['MSSubClass'].astype('object')
 return tr
train = feat_eng(train )
from sklearn.model_selection import train_test_split<import_modules>
def boxcox(a, f):
if sp.skew(a)not in range(-1,1)or sp.kurtosis(a)not in range(-1,1):
a = boxcox1p(a, 0.05)
 print(f, 'transformed.')
 return a
transformed_feats = []
for f in train.keys() :
if train[f].dtype == object:
train[f].fillna(value=train[f].value_counts().idxmax())
 train[f] = train[f].astype('category')
 else:
train[f] = train[f].fillna(value=np.mean(train[f]))
if f != 'SalePrice':
train[f] = boxcox(train[f], f)
 train[f + 'Sq'] =(train[f] ** 2)
 train[f + 'Cub'] =(train[f] ** 3)
 transformed_feats.append(f )
from sklearn.model_selection import train_test_split<split>
cols = train.columns
train = pd.get_dummies(train, drop_first=True )
Y = df_train.label
X = df_train.drop('label', axis=1)
 X = X / 255
X_comp = df_comp / 255
X_train, X_cross, Y_train, Y_cross = train_test_split(X, Y,test_size=0.1, random_state=42)
 X_valid, X_test, Y_valid, Y_test = train_test_split(X_cross, Y_cross, test_size=0.5, random_state=42 )<import_modules>
all_features = train.keys()
train = train.drop(train.loc[:,(train==0 ).sum() >=1444],axis=1)
 train = train.drop(train.loc[:,(train==1 ).sum() >=1444],axis=1)
 remain_features = train.keys()
remov_features = [st for st in all_features if st not in remain_features]
print(len(remov_features), 'features were removed:', remov_features )
from keras.models import Sequential, load_model
from keras.layers import Dense, Conv2D, MaxPool2D, Dropout, Flatten
from keras.utils import plot_model, to_categorical
from keras.utils.vis_utils import model_to_dot
from keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import confusion_matrix, accuracy_score<categorify>
X = train.drop('SalePrice', axis=1 )
X_train = X_train.values.reshape(X_train.shape[0],28,28,1)
 X_valid = X_valid.values.reshape(X_valid.shape[0],28,28,1)
 X_test = X_test.values.reshape(X_test.shape[0],28,28,1)
 X_comp = X_comp.values.reshape(X_comp.shape[0],28,28,1)
 Y_train = to_categorical(Y_train)
 Y_valid = to_categorical(Y_valid)
 Y_test = to_categorical(Y_test )<choose_model_class>
def scorer(estimator, X, y):
y_new = estimator.predict(X)
 return np.sqrt(mt.mean_squared_error(y, y_new))
est = sk.ElasticNet(l1_ratio=0, alpha=0.017, random_state=1)
 fsel = RFECV(est, step=1, cv=15, n_jobs=-1, scoring=scorer)
 fsel = fsel.fit(X, y)
 important_feat = list(X.loc[:, fsel.ranking_<=130].columns)
 X = train.loc[:, important_feat].values
print("Most important features:", important_feat )
datagen = ImageDataGenerator(height_shift_range=0.1,
width_shift_range=0.1,
rotation_range=10,
zoom_range=0.1,
fill_mode='constant',
cval=0 
)
 datagen.fit(X_train )<choose_model_class>
def cv_train(X, y, k):
param = {'learning_rate' : [0.1],
'n_estimators' : [600],
'max_depth' : [2]}
gbm = GradientBoostingRegressor(loss='huber', random_state=1)
 cv1 = GridSearchCV(gbm, param, cv=k, scoring='neg_mean_squared_error')
 cv1.fit(X, y)
 lasso = sk.ElasticNet(random_state=1)
 param = {'l1_ratio' : [0],
'alpha' : [0.017]}
cv2 = GridSearchCV(lasso, param, cv=k, scoring='neg_mean_squared_error')
 cv2.fit(X, y)
 print('GBM:', np.sqrt(cv1.best_score_*-1),
'Lasso:', np.sqrt(cv2.best_score_*-1),
'Mean:', np.sqrt(((cv1.best_score_+cv2.best_score_)/2)*-1),
'Pond Mean(0.8):', np.sqrt(((cv1.best_score_*0.2)+(cv2.best_score_*(0.8)))*-1))
return cv1, cv2
model = Sequential()
droprate = 0.175
model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu', input_shape=(28,28,1)))
 model.add(Conv2D(kernel_size=(2,2), filters=128, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(Conv2D(kernel_size=(2,2), filters=64, strides=(1,1), padding='same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(droprate))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(droprate))
model.add(Dense(10, activation='softmax'))
model1 = model
model1.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )<train_model>
cv1, cv2 = cv_train(X, y, 20 )
epochsN = 25
batch_sizeN = 63
history1 = model1.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_sizeN), validation_data=(X_valid, Y_valid), steps_per_epoch=len(X_train)/batch_sizeN, epochs=epochsN, verbose=2 )<compute_test_metric>
test = pd.read_csv('.. /input/test.csv')
 Id = test.loc[:, "Id"]
model1.evaluate(X_test, Y_test, verbose=0 )<save_model>
test[no_features] = test[no_features].fillna('No')
 test.loc[:, "LotFrontage_NA"] = test.LotFrontage.isnull() * 1
test.loc[:, "LotFrontage_NA"] = test.loc[:, "LotFrontage_NA"].astype("object")
 test = feat_eng(test)
 for f in test.keys() :
if test[f].dtype == object:
test[f].fillna(value=test[f].value_counts().idxmax())
 test[f] = test[f].astype('category')
 else:
test[f] = test[f].fillna(value=np.mean(test[f]))
if f in transformed_feats and f != 'SalePrice':
test[f] = boxcox(test[f], f)
 test[f + 'Sq'] =(test[f] ** 2)
 test[f + 'Cub'] =(test[f] ** 3)
 test = pd.get_dummies(test, drop_first=True)
 test = test.loc[:, important_feat]
print(test.loc[:, np.sum(test.isnull())== len(test)].columns)
 for c in list(test.loc[:, np.sum(test.isnull())== len(test)].columns):
test.loc[:, c] = 0
print("NAs:", np.sum(np.sum(test.isnull())) )
model1.save('model_1.h5' )<choose_model_class>
pred =(( math.e**cv2.predict(test)) *(0.8)) +(( math.e**cv1.predict(test)) *0.2)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 pd.DataFrame({"Id": Id.values, "SalePrice": pred} ).to_csv("submission.csv", index=False )