Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<count_unique_values>
warnings.filterwarnings('ignore')
 BASE_PATH = "/kaggle/input/house-prices-advanced-regression-techniques/"
exclude_unique = []
for c in train_df.columns:
num_uniques = len(train_df[c].unique())
 if train_df[c].isnull().sum() != 0:
num_uniques -= 1
if num_uniques == 1:
exclude_unique.append(c)
 print("We exclude: %s" % exclude_unique)
 print("
",len(exclude_unique))<define_variables>
df = pd.read_csv(f"{BASE_PATH}train.csv")
 X = df.select_dtypes("number" ).drop("SalePrice", axis=1)
 y = df.SalePrice
pipe = make_pipeline(SimpleImputer() , RobustScaler() , LinearRegression())
 print(f"The R2 score is: {cross_val_score(pipe, X, y ).mean() :.4f}" )
exclude_other = ['parcelid', 'logerror','propertyzoningdesc']
train_features = []
for c in train_df.columns:
if c not in exclude_missing \
and c not in exclude_other and c not in exclude_unique:
train_features.append(c)
 print("We use these for training: %s" % train_features)
 print("
",len(train_features))<count_unique_values>
num_cols = df.drop("SalePrice", axis=1 ).select_dtypes("number" ).columns
cat_cols = df.select_dtypes("object" ).columns
numeric_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer()), 
('scaler', RobustScaler())])
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent', fill_value='missing')) , 
('onehot', OneHotEncoder(handle_unknown='ignore')) ])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numeric_transformer, num_cols), 
('cat', categorical_transformer, cat_cols)])
 pipe = Pipeline(steps=[('preprocessor', preprocessor),('classifier', LinearRegression())])
 X = df.drop("SalePrice", axis=1)
 y = df.SalePrice
print(f"The R2 score is: {cross_val_score(pipe, X, y ).mean() :.4f}" )
cat_feature_inds = []
cat_unique_thresh = 1000
for i, c in enumerate(train_features):
num_uniques = len(train_df[c].unique())
 if num_uniques < cat_unique_thresh \
and not 'sqft' in c \
and not 'cnt' in c \
and not 'nbr' in c \
and not 'number' in c:
cat_feature_inds.append(i)
 print("Cat features are: %s" % [train_features[ind] for ind in cat_feature_inds] )<train_model>
classifiers = [
DummyRegressor() ,
LinearRegression(n_jobs=-1),
Ridge(alpha=0.003, max_iter=30),
Lasso(alpha=.0005),
ElasticNet(alpha=0.0005, l1_ratio=.9),
KernelRidge(alpha=0.6, kernel="polynomial", degree=2, coef0=2.5),
SGDRegressor() ,
SVR(kernel="linear"),
LinearSVR() ,
RandomForestRegressor(n_jobs=-1, n_estimators=350,
max_depth=12, random_state=1),
GradientBoostingRegressor(n_estimators=500, max_depth=2),
lgb.LGBMRegressor(n_jobs=-1, max_depth=2, n_estimators=1000,
learning_rate=0.05),
xgb.XGBRegressor(objective="reg:squarederror", n_jobs=-1,
max_depth=2, n_estimators=1500, learning_rate=0.075),
]
clf_names = [
"dummy",
"linear",
"ridge",
"lasso",
"elastic",
"kernlrdg",
"sgdreg",
"svr",
"linearsvr",
"randomforest",
"gbm",
"lgbm",
"xgboost"
]
print("Replacing NaN values by -999 !!")
 train_df.fillna(-999, inplace=True)
 test_df.fillna(-999, inplace=True )<split>
def clean_data(data, is_train_data=True):
data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
if is_train_data == True:
data = data[data.GrLivArea < 4000]
return data
X_train, X_test, y_train, y_test = train_test_split(train_df[train_features], train_df.logerror, test_size=0.2, random_state=99)
 print(X_train.shape, y_train.shape)
 print(X_test.shape, y_test.shape)
 all_pool = Pool(train_df[train_features], train_df.logerror, cat_feature_inds)
 train_pool = Pool(X_train, y_train, cat_feature_inds)
 test_pool = Pool(X_test, y_test, cat_feature_inds )<init_hyperparams>
def prepare_data(df, is_train_data=True):
numerical = df.select_dtypes("number" ).copy()
categorical = df.select_dtypes("object" ).copy()
if is_train_data == True :
SalePrice = numerical.SalePrice
y = np.log1p(SalePrice)
 numerical.drop(["Id", "SalePrice"], axis=1, inplace=True)
 else:
numerical.drop(["Id"], axis=1, inplace=True)
 y = None
X = pd.concat([numerical, categorical], axis=1)
 return X, y, numerical.columns, categorical.columns
catboost_parameters = {
'iterations': 400,
'learning_rate': 0.035,
'depth': 7,
'verbose': 20,
'task_type': 'GPU',
'loss_function': 'MAE',
'eval_metric': 'MAE',
'random_seed': 0,
}<train_model>
def get_pipeline(classifier, num_cols, cat_cols):
numeric_transformer = Pipeline(steps=[ 
('imputer', make_pipeline(SimpleImputer(strategy='mean'))), 
('scaler', StandardScaler())])
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent', fill_value='missing')) , 
('onehot', OneHotEncoder(handle_unknown='ignore')) ])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numeric_transformer, num_cols), 
('cat', categorical_transformer, cat_cols)])
 return Pipeline(steps=[('preprocessor', preprocessor),('classifier', classifier)] )
model = CatBoostRegressor(**catboost_parameters)
 model.fit(train_pool, eval_set=test_pool )<compute_test_metric>
def train_models(df):
X, y, num_cols, cat_cols = prepare_data(df)
 pipelines = []
for clf_name, classifier in zip(clf_names, classifiers):
clf = get_pipeline(classifier, num_cols, cat_cols)
 clf.fit(X, y)
 pipelines.append(clf)
 return pipelines
print_feature_importance(model, train_pool, X_train )<train_model>
def predict_from_models(df_test, pipelines):
X_test, _ , _, _ = prepare_data(df_test, is_train_data=False)
 predictions = []
for pipeline in pipelines:
preds = pipeline.predict(X_test)
 predictions.append(np.expm1(preds))
return predictions
num_ensembles = 5
models = [None] * num_ensembles
for i in range(num_ensembles):
print("
Training(ensemble): %d..." %(i))
catboost_parameters['random_seed'] = i
models[i] = CatBoostRegressor(**catboost_parameters)
 models[i].fit(train_pool, eval_set=test_pool)
 print('-- Feature Importance --')
 print_feature_importance(models[i], train_pool, X_train )<feature_engineering>
df = pd.read_csv(f"{BASE_PATH}train.csv")
 df_test = pd.read_csv(f"{BASE_PATH}test.csv")
 df = clean_data(df)
 df_test = clean_data(df_test, is_train_data=False )
submission = pd.DataFrame({
'ParcelId': test_df['parcelid'],
})
 test_dates = {
'201610': pd.Timestamp('2016-09-30'),
'201611': pd.Timestamp('2016-10-31'),
'201612': pd.Timestamp('2016-11-30'),
'201710': pd.Timestamp('2017-09-30'),
'201711': pd.Timestamp('2017-10-31'),
'201712': pd.Timestamp('2017-11-30')
 }
for label, test_date in test_dates.items() :
print("Predicting for: %s..." %(label))
test_df['transactiondate'] = test_date
test_df = add_date_features(test_df)
 y_pred = 0.0
for i in range(num_ensembles):
print("Ensemble:", i)
 y_pred += models[i].predict(test_df[train_features])
 y_pred /= num_ensembles
submission[label] = y_pred
submission_major = 2
print("Creating submission: submission_%03d.csv..." %(submission_major))
submission.to_csv(
 'submission_%03d.csv' %(submission_major),
float_format='%.4f',
index=False)
 print("Finished." )<define_variables>
my_scores = score_models(df)
 display(my_scores )
ACTUALLY_DO_SOMETHING = True
CORRECT_FOR_BUG = True<import_modules>
models = train_models(df)
 predictions = predict_from_models(df_test, models)
 prediction_final = pd.DataFrame(predictions[2:] ).mean().T.values
submission = pd.DataFrame({'Id': df_test.Id.values, 'SalePrice': prediction_final})
 submission.to_csv(f"submission.csv", index=False )