Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<groupby>
warnings.filterwarnings('ignore')
 %config InlineBackend.figure_format = 'retina'
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
input_dir = "/kaggle/input/house-prices-advanced-regression-techniques/"
test_groups = test_df.groupby("Id")
 test_size = len(test_groups)
 X_test = np.zeros(( test_size, MAX_SEQ_LEN, N_FEATURES), dtype=np.float32)
 i = 0
for _, group in test_groups:
X = group.values
seq_len = X.shape[0]
X_test[i,:seq_len,:] = X[:,1:23]
i += 1
del X
del test_groups
X_test.shape<import_modules>
train = pd.read_csv(f"{input_dir}/train.csv")
 test = pd.read_csv(f"{input_dir}/test.csv")
 print("Train shape:{}".format(train.shape))
print("Test shape:{}
".format(test.shape))
train.head()
Input,
Dense,
LSTM,
AveragePooling1D,
TimeDistributed,
Flatten,
Bidirectional,
Dropout 
)
<choose_model_class>
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 train.head()
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=5)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, min_delta=0.01 )<define_variables>
features = pd.concat([train_feat, test_feat] ).reset_index(drop=True)
 print(features.shape)
 features.head()
BATCH_SIZE = 1024
N_EPOCHS = 30<choose_model_class>
fill_with_none =('PoolQC', 'Alley', 'MiscFeature', 'Fence', 'FireplaceQu', 'GarageFinish', 'GarageQual', 'GarageCond', 'GarageType', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'MasVnrType')
 fill_with_mode =('Utilities', 'Functional')
 fill_with_0 =('GarageYrBlt', 'MasVnrArea', 'BsmtFullBath', 'BsmtHalfBath')
 for col in fill_with_none:
features[col]=features[col].fillna('None')
 for col in fill_with_mode:
features[col]=features[col].fillna(features[col].mode() [0])
 for col in fill_with_0:
features[col]=features[col].fillna(0)
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0] )
def get_model_deep(shape=(19, 22)) :
inp = Input(shape)
 x = Dense(16 )(inp)
 x = Bidirectional(LSTM(64, return_sequences=True))(x)
 x = TimeDistributed(Dense(64))(x)
 x = Bidirectional(LSTM(128, return_sequences=True))(x)
 x = TimeDistributed(Dense(1))(x)
 x = AveragePooling1D()(x)
 x = Flatten()(x)
 x = Dropout(0.5 )(x)
 x = Dense(1 )(x)
 model = Model(inp, x)
 return model<choose_model_class>
features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
model = get_model_deep(( 19,22))
model.compile(optimizer='adam', loss='mae',)
 model.summary()<train_model>
features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerics = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numerics.append(i)
 features.update(features[numerics].fillna(0))
numerics[1:10]
model.fit(X_train, y_train,
batch_size=BATCH_SIZE, epochs=N_EPOCHS,
validation_split=0.2, callbacks=[early_stopping, reduce_lr] )<save_to_csv>
features['MSSubClass'] = features['MSSubClass'].apply(str)
 features['YrSold'] = features['YrSold'].apply(str)
 features['MoSold'] = features['MoSold'].apply(str )
y_pred = model.predict(X_test, batch_size=BATCH_SIZE)
 submission = pd.DataFrame({'Id': test_ids, 'Expected': y_pred.reshape(-1)})
 submission.to_csv('submission.csv', index=False )<define_variables>
numeric_types = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric_features = []
for i in features.columns:
if features[i].dtypes in numeric_types:
numeric_features.append(i)
 skewed_feats = features[numeric_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skewed_feats[skewed_feats > 0.5]
skew_ind = high_skew.index
for i in skew_ind:
features[i] = boxcox1p(features[i], boxcox_normmax(features[i] + 1))
INPUT_WIDTH = 19
N_FEATURES = 22
THRESHOLD = 73<load_from_csv>
features = features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 features['TotalSF'] = features['TotalBsmtSF'] + features['1stFlrSF'] + features['2ndFlrSF']
features['TotalFinSF'] = features['BsmtFinSF1'] + features['BsmtFinSF2'] + features['1stFlrSF'] + features['2ndFlrSF']
features['TotalBathrooms'] = features['FullBath'] + features['BsmtFullBath'] +(0.5 * features['HalfBath'])+(0.5 * features['BsmtHalfBath'])
train_df = pd.read_csv(".. /input/train.csv")
 train_df[train_df.columns[1:]] = train_df[train_df.columns[1:]].astype(np.float32)
 train_df.head(20 )<categorify>
features['HasPool'] = features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 features['Has2ndLvl'] = features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasBsmt'] = features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 features['HasGarage'] = features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 features.shape
train_new = train_new.fillna(0.0)
 train_new = train_new.reset_index(drop=True)
 train_new.head()<create_dataframe>
final_feat = pd.get_dummies(features ).reset_index(drop=True)
 final_feat.head()
df_temp = pd.DataFrame(train_new.groupby('Id')['Expected'].mean())
 meaningful_ids = np.array(df_temp[df_temp['Expected'] < THRESHOLD].index)
 del df_temp
train_final = train_new[np.in1d(train_new.Id, meaningful_ids)]
del train_new, meaningful_ids
train_final.shape<prepare_x_and_y>
X = final_feat.iloc[:len(y), :]
X_test = final_feat.iloc[len(y):, :]
X.shape, y.shape, X_test.shape
test_types = X_test.dtypes
test_missing = round(( X_test.isnull().sum() /X_test.shape[0]),3)*100
test_explo = pd.DataFrame({'Types': test_types, 'Missing': test_missing} ).sort_values(by=['Missing', 'Types'], ascending=False)
 print(test_explo[test_explo.Missing > 0])
 train_types = X.dtypes
train_missing = round(( X.isnull().sum() /X.shape[0]),3)*100
train_explo = pd.DataFrame({'Types': train_types, 'Missing': train_missing} ).sort_values(by=['Missing', 'Types'], ascending=False)
 print(test_explo[test_explo.Missing > 0] )
train_gp = train_final.groupby("Id")
 train_size = len(train_gp)
 del train_final
X_train = np.zeros(( train_size, INPUT_WIDTH, N_FEATURES), dtype=np.float32)
 y_train = np.zeros(train_size, dtype=np.float32)
 seq_len_train = np.zeros(train_size, dtype=np.float32)
 i = 0
for _, group in train_gp:
X = group.values
seq_len = X.shape[0]
X_train[i,:seq_len,:] = X[:,1:23]
y_train[i] = X[0,23]
seq_len_train[i] = seq_len
i += 1
del X
del train_gp
X_train.shape, y_train.shape<load_from_csv>
overfit = []
for i in X.columns:
counts = X[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(X)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 X = X.drop(overfit, axis=1)
 X_test = X_test.drop(overfit, axis=1)
 overfit
test_df = pd.read_csv(".. /input/test.csv")
 test_df[test_df.columns[1:]] = test_df[test_df.columns[1:]].astype(np.float32)
 test_ids = np.array(test_df.Id.unique())
 test_final = test_df.fillna(0.0)
 test_final = test_final.reset_index(drop=True)
 del test_df<categorify>
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )
test_gp = test_final.groupby("Id")
 test_size = len(test_gp)
 del test_final
X_test = np.zeros(( test_size, INPUT_WIDTH, N_FEATURES), dtype=np.float32)
 seq_len_test = np.zeros(test_size, dtype=np.float32)
 i = 0
for _, group in test_gp:
X = group.values
seq_len = X.shape[0]
X_test[i,:seq_len,:] = X[:,1:23]
seq_len_test[i] = seq_len
i += 1
del X
del test_gp
X_test.shape<import_modules>
alphas_ridge = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas_lasso = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_ridge, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas_lasso, random_state=42, cv=kfolds))
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:squarederror', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
TimeDistributed, Flatten, Bidirectional 
<choose_model_class>
stack = StackingCVRegressor(regressors=(ridge, lasso, xgboost), meta_regressor=xgboost, use_features_in_secondary=True)
es_callback = EarlyStopping(monitor='val_loss', min_delta=0, patience=5 )<define_variables>
print('stack')
 stack_gen_model = stack.fit(np.array(X), np.array(y))
print('ridge')
 ridge_model = ridge.fit(X, y)
 print('lasso')
 lasso_model = lasso.fit(X, y)
 print('XGBoost')
 xgb_model = xgboost.fit(X, y )
BATCH_SIZE = 1024
N_EPOCHS = 32<choose_model_class>
print("RMSLE score on train data: ")
 print(rmsle(y, stack.predict(np.array(X))))
def get_model_simple(shape=(19,22)) :
inp = Input(shape)
 x = CuDNNLSTM(64, return_sequences=False )(inp)
 x = Dense(1 )(x)
 model = Model(inp, x)
 return model<train_model>
print('Predict Submission')
 submission = pd.read_csv(f"{input_dir}/sample_submission.csv")
 submission.iloc[:,1] = np.expm1(stack.predict(X_test))
model_0.fit(X_train, y_train,
batch_size=BATCH_SIZE, epochs=N_EPOCHS,
validation_split=0.2, callbacks=[es_callback] )<save_to_csv>
submission.to_csv("submission.csv", index=False )
<choose_model_class><EOS> 
submission.to_csv("submission.csv", index=False )