Kaggle competition: House Prices - Advanced Regression Techniques
use_feat_cols = []
train_feat = train[[ID_COL]].copy()
test_feat = test[[ID_COL]].copy()<categorify>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train.shape, test.shape
ord_enc = ce.OrdinalEncoder(cols=CAT_COLS)
 train_cat_feat = ord_enc.fit_transform(train[CAT_COLS])
 test_cat_feat = ord_enc.transform(test[CAT_COLS] )<concatenate>
train_ID = train['Id']
test_ID = test['Id']
train.drop(['Id'], axis=1, inplace=True)
 test.drop(['Id'], axis=1, inplace=True)
 train.shape, test.shape
train_feat = pd.concat([
train_feat, train_cat_feat], axis=1)
 test_feat = pd.concat([
test_feat, test_cat_feat], axis=1)
 use_feat_cols.extend(train_cat_feat.columns )<define_variables>
train["SalePrice"] = np.log1p(train["SalePrice"] )
train_cont_feat = train[CONT_COLS]
test_cont_feat = test[CONT_COLS]<concatenate>
def outlier_detect(feature, data):
outlier_index = []
for each in feature:
Q1 = np.percentile(data[each], 25)
 Q3 = np.percentile(data[each], 75)
 IQR = Q3 - Q1
min_quartile = Q1 - 1.5*IQR
max_quartile = Q3 + 1.5*IQR
outlier_list = data[(data[each] < min_quartile)|(data[each] > max_quartile)].index
outlier_index.extend(outlier_list)
 outlier_index = Counter(outlier_index)
 outlier_data = list(i for i, n in outlier_index.items() if n > 3)
 return outlier_data
train_feat = pd.concat([
train_feat, train_cont_feat], axis=1)
 test_feat = pd.concat([
test_feat, test_cont_feat], axis=1)
 use_feat_cols.extend(CONT_COLS )<define_variables>
outlier_data = outlier_detect(['OverallQual','GrLivArea','SalePrice'], train)
 train.loc[outlier_data]
def run_train_and_inference(
 X, X_test, y, use_model, model_params, train_params, seed_list, n_splits, cat_cols=None 
):
oof_pred_arr = np.zeros(len(X))
test_pred_arr = np.zeros(len(X_test))
feature_importances = pd.DataFrame()
score_list = []
for seed in seed_list:
if use_model == "cat":
model_params['random_state'] = seed
else:
model_params["seed"] = seed
kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 tmp_oof_pred = np.zeros(len(X))
tmp_test_pred = np.zeros(len(X_test))
for fold,(trn_idx, val_idx)in enumerate(kf.split(X, y)) :
print("*" * 100)
 print(f"Seed: {seed} - Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=use_model)
 with timer(prefix="Model training"):
model.train(
 params=model_params, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=train_params, cat_cols=cat_cols 
)
 with timer(prefix="Get Feature Importance"):
fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
fi_tmp["seed"] = seed
feature_importances = feature_importances.append(fi_tmp)
 with timer(prefix="Predict Valid"):
val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([seed, fold, score])
 tmp_oof_pred[val_idx] = val_pred
tmp_test_pred += model.predict(X_test)
 oof_score = mean_squared_error(y, tmp_oof_pred, squared=False)
 print(f"oof score: {oof_score: 5f}")
 score_list.append([seed, "oof", oof_score])
 oof_pred_arr += tmp_oof_pred
test_pred_arr += tmp_test_pred / n_splits
oof_pred_arr /= len(seed_list)
 test_pred_arr /= len(seed_list)
 oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["avg", "oof", oof_score])
 score_df = pd.DataFrame(
 score_list, columns=["seed", "fold", "rmse score"])
 return oof_pred_arr, test_pred_arr, score_df, feature_importances<prepare_x_and_y>
train.drop(train[(train['OverallQual']<5)&(train['SalePrice']>200000)].index, inplace=True)
 train.drop(train[(train['GrLivArea']>4500)&(train['SalePrice']<300000)].index, inplace=True)
 train.reset_index(drop=True, inplace=True )
X = train_feat[use_feat_cols]
X_test = test_feat[use_feat_cols]
y = train[TGT_COL].values
print(f"train_feat: {X.shape}, test_feat: {X_test.shape}" )<prepare_x_and_y>
train_labels = train['SalePrice'].reset_index(drop=True)
 train_features = train.drop(['SalePrice'], axis=1)
 test_features = test
all_features = pd.concat([train_features, test_features] ).reset_index(drop=True)
 all_features.shape
X_cat = X.copy()
X_cat[CAT_COLS] = train[CAT_COLS]
X_test_cat = X_test.copy()
X_test_cat = test[CAT_COLS]<init_hyperparams>
total = all_features.isnull().sum().sort_values(ascending=False)
 percent =(all_features.isnull().sum() /all_features.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(25 )
MODEL_PARAMS = {
"lgb": {
"objective": "root_mean_squared_error",
"boosting": "gbdt",
"max_depth": 8,
"learning_rate": 0.005,
"colsample_bytree": 0.2,
"subsample": 0.8,
"subsample_freq": 6,
"reg_alpha": 20,
"min_data_in_leaf": 200,
"n_jobs": 2,
"seed": RANDOM_SEED_LIST[0],
},
"xgb": {
"objective": "reg:squarederror",
"max_depth": 8,
"learning_rate": 0.003,
"colsample_bytree": 0.2,
"subsample": 0.8,
"reg_alpha" : 6,
"min_child_weight": 200,
"n_jobs": 2,
"seed": RANDOM_SEED_LIST[0],
'tree_method': "gpu_hist",
"gpu_id": 0,
},
"cat": {
'loss_function': 'RMSE',
"max_depth": 4,
'learning_rate': 0.03,
"bootstrap_type": 'Poisson',
"subsample": 0.8,
"border_count": 512,
"l2_leaf_reg": 200,
'random_state': RANDOM_SEED_LIST[0],
"thread_count": 2,
"task_type": "GPU",
"devices" : "0",
'num_boost_round': 50000,
}
}
TRAIN_PARAMS = {
"lgb": {
"num_boost_round": 50000,
"early_stopping_rounds": 200,
"verbose_eval": 200,
},
"xgb": {
"num_boost_round": 50000,
"early_stopping_rounds": 200,
"verbose_eval": 200,
},
"cat": {
'early_stopping_rounds': 200,
'verbose_eval': 200,
}
}<split>
all_features['MSSubClass'] = all_features['MSSubClass'].apply(str)
 all_features['YrSold'] = all_features['YrSold'].astype(str)
 all_features['MoSold'] = all_features['MoSold'].astype(str )
oof_pred_lgb, test_pred_lgb, score_lgb, feat_imps_lgb = run_train_and_inference(
 X, X_test, y, "lgb", MODEL_PARAMS["lgb"], TRAIN_PARAMS["lgb"], RANDOM_SEED_LIST, N_SPLITS )<filter>
def handle_missing(features):
features['Functional'] = features['Functional'].fillna('Typ')
 features['Electrical'] = features['Electrical'].fillna("SBrkr")
 features['KitchenQual'] = features['KitchenQual'].fillna("TA")
 features['Exterior1st'] = features['Exterior1st'].fillna(features['Exterior1st'].mode() [0])
 features['Exterior2nd'] = features['Exterior2nd'].fillna(features['Exterior2nd'].mode() [0])
 features['SaleType'] = features['SaleType'].fillna(features['SaleType'].mode() [0])
 features['MSZoning'] = features.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
features["PoolQC"] = features["PoolQC"].fillna("None")
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
features[col] = features[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
features[col] = features[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
features[col] = features[col].fillna('None')
 features['LotFrontage'] = features.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
objects = []
for i in features.columns:
if features[i].dtype == object:
objects.append(i)
 features.update(features[objects].fillna('None'))
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in features.columns:
if features[i].dtype in numeric_dtypes:
numeric.append(i)
 features.update(features[numeric].fillna(0))
return features
all_features = handle_missing(all_features )
score_lgb.loc[score_lgb.fold == "oof"]<split>
total = all_features.isnull().sum().sort_values(ascending=False)
 percent =(all_features.isnull().sum() /all_features.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(25 )
oof_pred_xgb, test_pred_xgb, score_xgb, feat_imps_xgb = run_train_and_inference(
 X, X_test, y, "xgb", MODEL_PARAMS["xgb"], TRAIN_PARAMS["xgb"], RANDOM_SEED_LIST, N_SPLITS )<filter>
numeric_dtypes = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric = []
for i in all_features.columns:
if all_features[i].dtype in numeric_dtypes:
numeric.append(i )
score_xgb.loc[score_xgb.fold == "oof"]<split>
for i in skew_index:
all_features[i] = boxcox1p(all_features[i], boxcox_normmax(all_features[i] + 1))
oof_pred_cat, test_pred_cat, score_cat, feat_imps_cat = run_train_and_inference(
 X, X_test, y, "cat", MODEL_PARAMS["cat"], TRAIN_PARAMS["cat"],
RANDOM_SEED_LIST, N_SPLITS, )<filter>
all_features['BsmtFinType1_Unf'] = 1*(all_features['BsmtFinType1'] == 'Unf')
 all_features['HasWoodDeck'] =(all_features['WoodDeckSF'] == 0)* 1
all_features['HasOpenPorch'] =(all_features['OpenPorchSF'] == 0)* 1
all_features['HasEnclosedPorch'] =(all_features['EnclosedPorch'] == 0)* 1
all_features['Has3SsnPorch'] =(all_features['3SsnPorch'] == 0)* 1
all_features['HasScreenPorch'] =(all_features['ScreenPorch'] == 0)* 1
all_features['YearsSinceRemodel'] = all_features['YrSold'].astype(int)- all_features['YearRemodAdd'].astype(int)
 all_features['Total_Home_Quality'] = all_features['OverallQual'] + all_features['OverallCond']
all_features = all_features.drop(['Utilities', 'Street', 'PoolQC',], axis=1)
 all_features['TotalSF'] = all_features['TotalBsmtSF'] + all_features['1stFlrSF'] + all_features['2ndFlrSF']
all_features['YrBltAndRemod'] = all_features['YearBuilt'] + all_features['YearRemodAdd']
all_features['Total_sqr_footage'] =(all_features['BsmtFinSF1'] + all_features['BsmtFinSF2'] +
all_features['1stFlrSF'] + all_features['2ndFlrSF'])
 all_features['Total_Bathrooms'] =(all_features['FullBath'] +(0.5 * all_features['HalfBath'])+
all_features['BsmtFullBath'] +(0.5 * all_features['BsmtHalfBath']))
all_features['Total_porch_sf'] =(all_features['OpenPorchSF'] + all_features['3SsnPorch'] +
all_features['EnclosedPorch'] + all_features['ScreenPorch'] +
all_features['WoodDeckSF'])
 all_features['TotalBsmtSF'] = all_features['TotalBsmtSF'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['2ndFlrSF'] = all_features['2ndFlrSF'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['GarageArea'] = all_features['GarageArea'].apply(lambda x: np.exp(6)if x <= 0.0 else x)
 all_features['GarageCars'] = all_features['GarageCars'].apply(lambda x: 0 if x <= 0.0 else x)
 all_features['LotFrontage'] = all_features['LotFrontage'].apply(lambda x: np.exp(4.2)if x <= 0.0 else x)
 all_features['MasVnrArea'] = all_features['MasVnrArea'].apply(lambda x: np.exp(4)if x <= 0.0 else x)
 all_features['BsmtFinSF1'] = all_features['BsmtFinSF1'].apply(lambda x: np.exp(6.5)if x <= 0.0 else x)
 all_features['haspool'] = all_features['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['has2ndfloor'] = all_features['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasgarage'] = all_features['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasbsmt'] = all_features['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_features['hasfireplace'] = all_features['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
score_cat.loc[score_cat.fold == "oof"]<define_variables>
def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YearRemodAdd','TotalSF']
all_features = logs(all_features, log_features )
model_names = ["lgb", "xgb", "cat"]<create_dataframe>
def squares(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(res[l]*res[l] ).values)
 res.columns.values[m] = l + '_sq'
m += 1
return res
squared_features = ['YearRemodAdd', 'LotFrontage_log',
'TotalBsmtSF_log', '1stFlrSF_log', '2ndFlrSF_log', 'GrLivArea_log',
'GarageCars_log', 'GarageArea_log']
all_features = squares(all_features, squared_features )
pd.DataFrame(
 np.corrcoef([
oof_pred_lgb,
oof_pred_xgb,
oof_pred_cat
]),
columns=model_names, index=model_names )<create_dataframe>
all_features = pd.get_dummies(all_features ).reset_index(drop=True)
 all_features.shape
pd.DataFrame(
 np.corrcoef([
test_pred_lgb,
test_pred_xgb,
test_pred_cat
]),
columns=model_names, index=model_names )<compute_test_metric>
all_features = all_features.loc[:,~all_features.columns.duplicated() ]
oof_pred_avg =(oof_pred_lgb + oof_pred_xgb + oof_pred_cat)/ 3
oof_score_avg = mean_squared_error(y, oof_pred_avg, squared=False)
 print(f"oof score avg: {oof_score_avg:.5f}")
 test_pred_avg =(test_pred_lgb + test_pred_xgb + test_pred_cat)/ 3<compute_test_metric>
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor, BaggingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.linear_model import Ridge, RidgeCV
from sklearn.linear_model import ElasticNet, ElasticNetCV
from sklearn.svm import SVR
from mlxtend.regressor import StackingCVRegressor
import lightgbm as lgb
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold, cross_val_score
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import scale
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.decomposition import PCA
weights = [0.5, 0.4, 0.1]
oof_pred_wavg = weights[0] * oof_pred_lgb + weights[1] * oof_pred_xgb + weights[2] * oof_pred_cat
oof_score_wavg = mean_squared_error(y, oof_pred_wavg, squared=False)
 print(f"oof score weighted avg: {oof_score_wavg:.6f}")
 test_pred_wavg = weights[0] * test_pred_lgb + weights[1] * test_pred_xgb + weights[2] * test_pred_cat<save_to_csv>
kf = KFold(n_splits=12, random_state=42, shuffle=True )
sub = smpl_sub.copy()
sub[TGT_COL] = test_pred_wavg
sub.to_csv("submission.csv", index=False)
 sub.head()<set_options>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, train_labels, scoring="neg_mean_squared_error", cv=kf))
return(rmse )
%matplotlib inline
sns.set_style("whitegrid")
 !pip install pycaret
warnings.filterwarnings("ignore" )<load_from_csv>
lightgbm = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=7000,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 xgboost = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42)
 ridge_alphas = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas, cv=kf))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=6000,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 rf = RandomForestRegressor(n_estimators=1200,
max_depth=15,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
oob_score=True,
random_state=42)
 stack_gen = StackingCVRegressor(regressors=(xgboost, lightgbm, svr, ridge, gbr, rf),
meta_regressor=xgboost,
use_features_in_secondary=True)
 dtreg = DecisionTreeRegressor(random_state = 100)
 rfr = RandomForestRegressor(n_estimators = 100, random_state = 0 )
train = pd.read_csv("/kaggle/input/tabular-playground-series-feb-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-feb-2021/test.csv" )<categorify>
scores = {}
score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['lgb'] =(score.mean() , score.std() )
for col in categorical_features:
le = LabelEncoder()
train[col] = le.fit_transform(train[col])
 test[col] = le.fit_transform(test[col] )<feature_engineering>
score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['xgb'] =(score.mean() , score.std() )
continous_features = train.iloc[:, 11:25].columns
for i in continous_features:
xt = boxcox1p(train[i], [0])
 train["boxcox"+ i] = xt<concatenate>
score = cv_rmse(dtreg)
 print("dtreg: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['dtreg'] =(score.mean() , score.std() )
for i in continous_features:
xt = boxcox1p(test[i], [0])
 test["boxcox"+ i] = xt<concatenate>
score = cv_rmse(rfr)
 print("rfr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rfr'] =(score.mean() , score.std() )
train_transform = pd.concat([train.iloc[:, 1:11], train.iloc[:, 26:]], axis=1)
 train_transform.columns<split>
score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std() )
X = train_transform
y = train["target"]
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )<train_on_grid>
score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std() )
def K_fold_CV(X, y, model, params):
rmse_error = []
skf = KFold(n_splits=5, shuffle=True, random_state=42)
 for fold,(tr_idx, ts_idx)in enumerate(skf.split(X, y)) :
print(f"Fold: {fold}")
 x_tr, y_tr = X.iloc[tr_idx], y.iloc[tr_idx]
x_ts, y_ts = X.iloc[ts_idx], y.iloc[ts_idx]
reg = model(**params)
 reg.fit(x_tr, y_tr,
eval_set=[(x_ts, y_ts)],
early_stopping_rounds=100,
verbose=False)
 error = mean_squared_error(y_ts, reg.predict(x_ts))
rmse_error.append(error)
 print(f"RMSE Error: {error}")
 print("-"*25)
 return reg, np.mean(rmse_error )<init_hyperparams>
score = cv_rmse(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std() )
param_lgb = {
"objective":"regression",
"metric":"rmse",
"feature_pre_filter":False,
"learning_rate":0.05,
"num_leaves":256,
"min_child_samples":20,
"sub_feature":0.4,
"sub_row":1,
"subsample_freq":0,
"lambda_l1":4.6,
"lambda_l2":1.9
}<compute_train_metric>
score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std() )
%%time
model_lgb, lgb_error = K_fold_CV(X, y, LGBMRegressor, params=param_lgb )<init_hyperparams>
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(train_labels))
param_cbr = {
'loss_function': 'RMSE',
"max_depth": 4,
'learning_rate': 0.03,
"bootstrap_type": 'Poisson',
"subsample": 0.8,
"border_count": 512,
"l2_leaf_reg": 200,
'random_state': 42,
"thread_count": 2,
'num_boost_round': 50000,
"task_type": "GPU",
"devices" : "0",
}<compute_train_metric>
print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, train_labels )
%%time
cbr, cbr_error = K_fold_CV(X, y, CatBoostRegressor, params=param_cbr )<init_hyperparams>
print('xgboost')
 xgb_model_full_data = xgboost.fit(X, train_labels )
param_xgb = {'n_estimators': 1811,
'reg_alpha': 0.02641656905930228,
'reg_lambda': 9.219592384947825,
'colsample_bytree': 0.3,
'subsample': 1.0,
'learning_rate': 0.017,
'max_depth': 5,
'num_leaves': 154,
'min_child_samples': 47,
'min_data_per_groups': 96,
'tree_method': "gpu_hist",
"gpu_id": 0}<compute_train_metric>
print('Svr')
 svr_model_full_data = svr.fit(X, train_labels )
model_xgb, xgb_error = K_fold_CV(X, y, XGBRegressor, params=param_xgb )<load_from_csv>
print('Ridge')
 ridge_model_full_data = ridge.fit(X, train_labels )
submission = pd.read_csv(".. /input/tabular-playground-series-feb-2021/sample_submission.csv" )<predict_on_test>
print('RandomForest')
 rf_model_full_data = rf.fit(X, train_labels )
pred = cbr.predict(test[X.columns])
 submission["target"] = pred<save_to_csv>
print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, train_labels )
submission.to_csv("submission.csv", index=False )<save_to_csv>
def blended_predictions(X):
return(( 0.1 * ridge_model_full_data.predict(X)) + \ 
(0.2 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.05 * rf_model_full_data.predict(X)) + \ 
(0.35 * stack_gen_model.predict(np.array(X))))
submission.to_csv("submission.csv", index=False )<set_options>
blended_score = rmsle(train_labels, blended_predictions(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
SEED = 0
seed_everything(SEED)
 %matplotlib inline
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Device:', tpu.master())
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 AUTOTUNE = tf.data.experimental.AUTOTUNE
tf.__version__ 
<define_variables>
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.shape
GCS_PATH = KaggleDatasets().get_gcs_path("gan-getting-started")
 GCS_PATH_1367 = KaggleDatasets().get_gcs_path("monet-tfrecords-256x256")
 MONET_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/monet_tfrec/*.tfrec'))
PHOTO_FILENAMES = tf.io.gfile.glob(str(GCS_PATH + '/photo_tfrec/*.tfrec'))
MONET_FILENAMES_1367 = tf.io.gfile.glob(str(GCS_PATH_1367 + '/mon*.tfrec'))
m=MONET_FILENAMES_1367
p=PHOTO_FILENAMES
PRETRAIN_FILENAMES=[m[0],p[0],m[1],p[1],m[2],p[2],m[3],p[3],m[0],p[4],m[1],p[5],m[2],p[6],m[3],p[7],m[0],p[8],m[1],p[9],m[2],p[10],m[3],p[11],m[0],p[12],m[1],p[13],m[2],p[14],m[3],p[15],m[0],p[16],m[1],p[17],m[2],p[18],m[4],p[19]]
PRETRAIN_LABELS=[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*274+[0]*352+[1]*271+[0]*350 
<categorify>
submission.iloc[:,1] = np.floor(np.expm1(blended_predictions(X_test)) )
IMAGE_SIZE = [256, 256]
def decode_image(image):
image = tf.image.decode_jpeg(image, channels=3)
 image =(tf.cast(image, tf.float32)/ 127.5)- 1
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def read_tfrecord(example):
tfrecord_format = {
"image": tf.io.FixedLenFeature([], tf.string)
 }
example = tf.io.parse_single_example(example, tfrecord_format)
 image = decode_image(example['image'])
 return image<normalization>
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission_regression_New_Improvement.csv", index=False )
def data_augment_color(image):
image = tf.image.random_flip_left_right(image)
 image =(image + 1)/ 2
image = tf.image.random_saturation(image, 0.7, 1.2)
 image = tf.clip_by_value(image, 0, 1)
 image =(image - 0.5)* 2
return image
def pretrain_data_augment(image):
image = tf.image.resize(image, [286, 286])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 return image
def to3030(label):
return np.ones(( 30,30,1),dtype="float32")*label<normalization>
submission['SalePrice'] *= 1.001619
submission.to_csv("submission_regression2.csv", index=False )
def data_augment(image):
p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)
 if p_crop >.5:
image = tf.image.resize(image, [286, 286])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_crop >.9:
image = tf.image.resize(image, [300, 300])
 image = tf.image.random_crop(image, size=[256, 256, 3])
 if p_rotate >.9:
image = tf.image.rot90(image, k=3)
 elif p_rotate >.7:
image = tf.image.rot90(image, k=2)
 elif p_rotate >.5:
image = tf.image.rot90(image, k=1)
 if p_spatial >.6:
image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial >.9:
image = tf.image.transpose(image)
 return image<create_dataframe>
scaler = StandardScaler()
X_train, X_test, y_train, y_test = train_test_split(X, train_labels, test_size=0.3, random_state=42)
 X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )
BATCH_SIZE = 1
def load_dataset(filenames):
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTOTUNE)
 return dataset
photo_ds = load_dataset(PHOTO_FILENAMES)
 monet_ds = load_dataset(MONET_FILENAMES)
 monet_ds = monet_ds.repeat()
photo_ds = photo_ds.repeat()
gan_ds = tf.data.Dataset.zip(( monet_ds, photo_ds)).batch(BATCH_SIZE ).prefetch(AUTOTUNE)
 fast_photo_ds = load_dataset(PHOTO_FILENAMES ).batch(32*strategy.num_replicas_in_sync ).prefetch(32)
 monet_ds_fid = load_dataset(MONET_FILENAMES ).batch(32*strategy.num_replicas_in_sync ).prefetch(32)
 pretrain_ds = load_dataset(PRETRAIN_FILENAMES)
 pretrain_labels=tf.data.Dataset.from_tensor_slices(np.array(PRETRAIN_LABELS,dtype="float32"))
pretrain_labels=pretrain_labels.map(to3030,num_parallel_calls=AUTOTUNE)
 pretrain_trans=pretrain_ds.map(pretrain_data_augment, num_parallel_calls=AUTOTUNE)
 pretrain_ds=tf.data.Dataset.zip(( pretrain_ds, pretrain_trans,pretrain_labels))
val_pretrain_ds=pretrain_ds.skip(11894 ).batch(32, drop_remainder=True ).prefetch(2)
 pretrain_ds=pretrain_ds.take(11894 ).shuffle(400 ).batch(32, drop_remainder=True ).prefetch(2 )<choose_model_class>
pca = PCA(n_components=3)
 X_train = pca.fit_transform(X_train)
 X_test = pca.transform(X_test )