Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<compute_train_metric>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 warnings.simplefilter(action='ignore', category=FutureWarning)
 print(os.listdir(".. /input"))
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]<define_search_model>
new_train = train.copy()
new_test = test.copy()
new_train.drop('Id', axis = 1, inplace = True)
 new_test.drop('Id', axis = 1, inplace = True )
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))<choose_model_class>
lotfrontage_by_neighborhood = new_train.groupby(['Neighborhood'])['LotFrontage'].agg({
'median': np.median,
'count': np.size,
'std': np.std,
} ).reset_index()
lotfrontage_by_neighborhood.T
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )<choose_model_class>
def fill_lot_frontage(row):
return lotfrontage_by_neighborhood[lotfrontage_by_neighborhood['Neighborhood'] == row['Neighborhood']]['median'].values[0]
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1,
)<choose_model_class>
new_train['LotFrontage'] = new_train.apply(lambda row: fill_lot_frontage(row)if np.isnan(row['LotFrontage'])else row['LotFrontage'], axis=1)
 new_test['LotFrontage'] = new_test.apply(lambda row: fill_lot_frontage(row)if np.isnan(row['LotFrontage'])else row['LotFrontage'], axis=1 )
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )<choose_model_class>
new_train["PoolQC"] = new_train["PoolQC"].fillna("None")
 new_test["PoolQC"] = new_test["PoolQC"].fillna("None" )
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )<compute_test_metric>
new_train["MiscFeature"] = new_train["MiscFeature"].fillna("None")
 new_test["MiscFeature"] = new_test["MiscFeature"].fillna("None" )
score = cv_rmse(ridge , X)
 score = cv_rmse(lasso , X)
 print("LASSO: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )<train_model>
new_train["Alley"] = new_train["Alley"].fillna("None")
 new_test["Alley"] = new_test["Alley"].fillna("None" )
print('START Fit')
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print('elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print('Lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print('Ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print('Svr')
 svr_model_full_data = svr.fit(X, y)
 print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, y)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )<predict_on_test>
new_train["Fence"] = new_train["Fence"].fillna("None")
 new_test["Fence"] = new_test["Fence"].fillna("None" )
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.05 * lasso_model_full_data.predict(X)) + \ 
(0.1 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
new_train["FireplaceQu"] = new_train["FireplaceQu"].fillna("None")
 new_test["FireplaceQu"] = new_test["FireplaceQu"].fillna("None" )
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )<load_from_csv>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
new_train[col] = new_train[col].fillna('None')
 new_test[col] = new_test[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
new_train[col] = new_train[col].fillna(0)
 new_test[col] = new_test[col].fillna(0 )
print('Predict submission')
 submission = pd.read_csv(".. /input/sample_submission.csv")
 submission.iloc[:,1] =(np.expm1(blend_models_predict(X_sub)) )<load_from_csv>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
new_train[col] = new_train[col].fillna('None')
 new_test[col] = new_test[col].fillna('None')
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
new_train[col] = new_train[col].fillna(0)
 new_test[col] = new_test[col].fillna(0 )
<save_to_csv>
new_train["MasVnrType"] = new_train["MasVnrType"].fillna("None")
 new_test["MasVnrType"] = new_test["MasVnrType"].fillna("None")
 new_train["MasVnrArea"] = new_train["MasVnrArea"].fillna(0)
 new_test["MasVnrArea"] = new_test["MasVnrArea"].fillna(0 )
q1 = submission['SalePrice'].quantile(0.0042)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission.csv", index=False )<import_modules>
new_test['MSZoning'] = new_test['MSZoning'].fillna(train['MSZoning'].mode() [0] )
print(K.image_data_format() )<split>
cols_to_drop = []
cols_to_clean = []
( x_train, y_train),(x_test, y_test)= mnist.load_data()<categorify>
def handle_na(train, test, cols_to_clean, cols_to_drop):
if(cols_to_drop):
train.drop(cols_to_drop, axis = 1, inplace = True)
 test.drop(cols_to_drop, axis = 1, inplace = True)
 if(cols_to_clean):
for col in cols_to_clean:
train[col+'_is_na'] = train[col].isnull()
test[col+'_is_na'] = test[col].isnull()
for col in test.columns:
if np.issubdtype(train[col].dtype, np.number):
train.loc[train[col].isnull() == True, col] = train[col].median()
test.loc[test[col].isnull() == True, col] = train[col].median()
if(train[col].dtype == 'O'):
train.loc[train[col].isnull() == True, col] = train[col].mode().iloc[0]
test.loc[test[col].isnull() == True, col] = train[col].mode().iloc[0]
print(f'Shape of Train Set: {train.shape}')
 print(f'Shape of Test Set : {test.shape}')
 return train, test
y_train = to_categorical(y_train, num_classes = 10)
 y_test = to_categorical(y_test, num_classes=10 )<data_type_conversions>
new_train, new_test = handle_na(new_train, new_test, cols_to_clean, cols_to_drop )
x_train.astype('float32')
 x_test.astype('float32' )<feature_engineering>
def label_encode(train, test):
combined = train.append(test, sort = False)
 for col in test.columns:
if(train[col].dtype == 'O'):
le = LabelEncoder()
le.fit(combined[col])
 train[col] = le.transform(train[col])
 test[col] = le.transform(test[col])
 return train, test
x_train = x_train/255
x_test = x_test/255<choose_model_class>
new_train, new_test = label_encode(new_train, new_test )
input_shape =(28,28,1)
 model = Sequential()
model.add(Conv2D(96,(3, 3),
padding='Same',
activation='relu',
input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(96,(3, 3),
padding='Same',
activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(192,(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(256,(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Conv2D(256,(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(256,(3,3), activation='relu', padding='same'))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(units=10, activation='softmax'))<choose_model_class>
new_train['TotalSF'] = new_train['TotalBsmtSF'] + new_train['1stFlrSF'] + new_train['2ndFlrSF']
new_test['TotalSF'] = new_test['TotalBsmtSF'] + new_test['1stFlrSF'] + new_test['2ndFlrSF']
call_back = keras.callbacks.EarlyStopping(monitor='val_acc', min_delta=0, patience=5, verbose=0, restore_best_weights=True)
 reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_acc',
factor=0.25,
verbose=1,
patience=2,
min_lr=0.000001 )<choose_model_class>
new_train['SalePrice'] = np.log(new_train['SalePrice'] )
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(lr=0.001),
metrics=['accuracy'])
 train_datagen = ImageDataGenerator(shear_range=0.2,
zoom_range=0.2,
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
height_shift_range=.1,
rotation_range=10,
width_shift_range=.1)
 train_datagen.fit(x_train)
 history = model.fit_generator(
 train_datagen.flow(x_train,y_train, batch_size=128),
steps_per_epoch=x_train.shape[0] // 128,
epochs=32,
validation_data=(x_test,y_test),callbacks=[call_back, reduce_lr] )<load_from_csv>
def split(df, target, test_size = 0.1):
X_train, X_valid, y_train, y_valid = train_test_split(df.drop(target, axis = 1), df[target], test_size = test_size, random_state = 42)
 print(f'Shape of Train Set: {X_train.shape}')
 print(f'Shape of Valid Set: {X_valid.shape}')
 return X_train, X_valid, y_train, y_valid
xtest = pd.read_csv(".. /input/test.csv" )<save_to_csv>
X_train, X_valid, y_train, y_valid = split(new_train, 'SalePrice' )
submission = model.predict(xtest)
 submission = np.argmax(submission, axis = 1)
 submission = pd.Series(submission,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),submission],axis = 1)
 submission.to_csv("submission.csv",index=False )<set_options>
def xgtrain(X_train, X_valid, y_train, y_valid):
regressor = XGBRegressor(
 n_estimators = 50000,
learning_rate = 0.01,
max_depth = 3,
subsample = 0.5,
colsample_bytree = 0.2 
)
 regressor_ = regressor.fit(
 X_train.values, y_train.values,
eval_metric = 'rmse',
eval_set = [ 
(X_train.values, y_train.values), 
(X_valid.values, y_valid.values)
 ],
verbose = 1000,
early_stopping_rounds = 500, 
)
 return regressor_
tf.set_random_seed(42 )<define_variables>
%%time
regressor_ = xgtrain(X_train, X_valid, y_train, y_valid )
train_path = os.path.join('.. ', 'input', 'train.csv')
 test_path = os.path.join('.. ', 'input', 'test.csv')
 size = 28
lr = 0.001
num_classes = 10
epochs = 30
batch_size = 128<load_from_csv>
def lgbtrain(X_train, y_train, X_valid, y_valid):
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_valid, y_valid, reference=lgb_train)
 params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'l2_root'},
'max_depth': 3,
'learning_rate': 0.01,
'feature_fraction': 0.2,
'bagging_fraction': 0.8,
'bagging_freq': 5,
'num_leaves': 5
}
gbm = lgb.train(
 params,
lgb_train,
valid_sets = lgb_eval,
num_boost_round=50000,
early_stopping_rounds=100,
verbose_eval = 250 
)
 return gbm
raw_train_df = pd.read_csv(train_path)
 raw_test_df = pd.read_csv(test_path )<feature_engineering>
%%time
gbm = lgbtrain(X_train, y_train, X_valid, y_valid )
def parse_train_df(_train_df):
labels = _train_df.iloc[:,0].values
imgs = _train_df.iloc[:,1:].values
imgs_2d = np.array([[[[float(imgs[index][i*28 + j])/ 255] for j in range(28)] for i in range(28)] for index in range(len(imgs)) ])
 processed_labels = [[0 for _ in range(10)] for i in range(len(labels)) ]
for i in range(len(labels)) :
processed_labels[i][labels[i]] = 1
return np.array(processed_labels), imgs_2d
def parse_test_df(test_df):
imgs = test_df.iloc[:, 0:].values
imgs_2d = np.array([[[[float(imgs[index][i * 28 + j])/ 255] for j in range(28)] for i in range(28)] for index in
range(len(imgs)) ])
 return imgs_2d<split>
def prepare_submission(comment = 'latest'):
xgb_preds = np.exp(regressor_.predict(new_test.values))
lgb_preds = np.expm1(gbm.predict(new_test.values))
submission['SalePrice'] = 0.75*xgb_preds + 0.25*lgb_preds
submission.to_csv('submission' + comment + '.csv', index=False)
 print(submission.head() )
y_train_set, x_train_set = parse_train_df(raw_train_df)
 x_test = parse_test_df(raw_test_df)
 x_train, x_val, y_train, y_val = train_test_split(x_train_set, y_train_set, test_size=0.20, random_state=42 )<count_values>
FileLinks('.' )
<choose_model_class><EOS> 
new_train.to_feather('new_train')
 new_test.to_feather('new_test' )