Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<concatenate>
train = pd.read_csv(".. /input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-jan-2021/test.csv")
 sub = pd.read_csv(".. /input/tabular-playground-series-jan-2021/sample_submission.csv")
 train
all_data = pd.concat([train_data, test_data],keys=['train','test'])
 train_data.shape, test_data.shape, all_data.shape<drop_column>
train = train.drop(["id"], axis=1)
 features = [c for c in train.columns if "cont" in c]
test = test.drop("id", axis=1)
 test
all_data = all_data.drop(columns=['Id'], axis=1 )<drop_column>
fe = dict(
 rankgauss = True,
stats = True,
gaussmix = False,
pca = True,
tsne = True,
umap = True,
drop_original = False,
)
all_data['Age']=all_data['YrSold']-all_data['YearBuilt']+1
all_data['AgeRemodAdd']=all_data['YrSold']-all_data['YearRemodAdd']+1
all_data['AgeGarage']=all_data['YrSold']-all_data['GarageYrBlt']+1
all_data.drop(columns = 'YearBuilt')
 all_data.drop(columns = 'YearRemodAdd')
 all_data.drop(columns = 'GarageYrBlt' )<feature_engineering>
all_data = pd.concat([train, test], axis=0, ignore_index=True)
 targets = all_data.target[:300000]
all_data = all_data.drop("target", axis=1)
 COLS = [c for c in all_data.columns if "cont" in c]
all_data
for i in ['Alley','PoolQC','Fence','MiscFeature','FireplaceQu']:
all_data[i]=all_data[i].apply(lambda x: 'None' if str(x)=='nan' else x )<feature_engineering>
if fe["stats"]:
for stats in tqdm.tqdm(["sum", "var", "mean", "median", "std", "kurt", "skew"]):
all_data["cont_" + stats] = getattr(all_data[COLS], stats )(axis = 1)
 all_data
for i in('GarageType','GarageFinish','GarageQual','GarageCond'):
all_data[i].fillna('None',inplace=True)
 all_data['GarageArea'] =(np.where(( all_data['GarageType']=='None'), 0, all_data['GarageArea']))
all_data['GarageCars'] =(np.where(( all_data['GarageType']=='None'), 0, all_data['GarageCars']))
all_data['AgeGarage'] =(np.where(( all_data['GarageType']=='None'), 0, all_data['AgeGarage']))<feature_engineering>
sys.path.append(".. /input/rank-gauss")
 if fe["rankgauss"]:
scaler = GaussRankScaler()
rankgauss_feat = scaler.fit_transform(all_data[COLS])
 rankgauss_df = pd.DataFrame(rankgauss_feat, columns=[f"rankgauss_{i}" for i in range(rankgauss_feat.shape[1])])
 all_data = pd.concat([all_data, rankgauss_df], axis=1)
 all_data
for i in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[i].fillna('None',inplace=True)
 for i in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[i] =(np.where(( all_data['BsmtQual']=='None'), 0, all_data[i]))<categorify>
if fe["gaussmix"]:
def get_gmm_class_feature(feat, n):
gmm = GaussianMixture(n_components=n, random_state=42)
 gmm.fit(all_data[feat].values.reshape(-1, 1))
all_data[f'{feat}_class'] = gmm.predict(all_data[feat].values.reshape(-1, 1))
get_gmm_class_feature('cont1', 4)
 get_gmm_class_feature('cont2', 10)
 get_gmm_class_feature('cont3', 6)
 get_gmm_class_feature('cont4', 4)
 get_gmm_class_feature('cont5', 3)
 get_gmm_class_feature('cont6', 2)
 get_gmm_class_feature('cont7', 3)
 get_gmm_class_feature('cont8', 4)
 get_gmm_class_feature('cont9', 4)
 get_gmm_class_feature('cont10', 8)
 get_gmm_class_feature('cont11', 5)
 get_gmm_class_feature('cont12', 4)
 get_gmm_class_feature('cont13', 6)
 get_gmm_class_feature('cont14', 6)
 CLASS_COLS = [c for c in all_data.columns if "_class" in c]
CLASS_COLS_IDX = []
for c in CLASS_COLS:
CLASS_COLS_IDX.append(all_data.columns.get_loc(c))
assert len(CLASS_COLS)> 0
all_data
x= { 20:'1-STORY 1946 & NEWER ALL STYLES',
30:'1-STORY 1945 & OLDER',
40:'1-STORY W/FINISHED ATTIC ALL AGES',
45:'1-1/2 STORY - UNFINISHED ALL AGES',
50:'1-1/2 STORY FINISHED ALL AGES',
60:'2-STORY 1946 & NEWER',
70:'2-STORY 1945 & OLDER',
75:'2-1/2 STORY',
80:'SPLIT',
85:'SPLIT FOYER',
90:'DUPLEX',
120:'1-STORY PUD',
150:'1-1/2 STORY PUD',
160:'2-STORY PUD',
180:'PUD - MULTILEVEL',
190:'2 FAMILY CONVERSION'}
all_data['MSSubClass']=all_data['MSSubClass'].map(x )<sort_values>
if fe["pca"]:
pca = PCA(n_components = 0.9, random_state = 42 ).fit(all_data[COLS])
 pca_feat = pca.transform(all_data[COLS])
 pca_df = pd.DataFrame(pca_feat, columns = [f"pca_cont{i}" for i in range(pca.n_components_)])
 all_data = pd.concat([all_data, pca_df], axis=1)
 PCA_COLS = [c for c in all_data.columns if "pca" in c]
assert len(PCA_COLS)> 0
all_data
total = all_data.isnull().sum().sort_values(ascending = False)
 percent = round(all_data.isnull().sum().sort_values(ascending = False)/ len(all_data)*100, 2)
 missingValues = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missingValues.loc[(missingValues['Total'] > 0)]<categorify>
if fe["tsne"]:
tsne_components = 2
perplexity = [5, 10, 15, 20, 25]
for per in perplexity:
tsne = TSNE(n_components = tsne_components, perplexity = per, n_neighbors = 3.5 * per)
 tsne_feat = tsne.fit_transform(all_data[COLS])
 tsne_df = pd.DataFrame(tsne_feat, columns=[f"tsne_{per}_{i}" for i in range(tsne_components)])
 all_data = pd.concat([all_data, tsne_df], axis = 1)
 TSNE_COLS = [c for c in all_data.columns if "tsne" in c]
all_data
def HandleMissingValues(df):
num_cols = df.select_dtypes(include = ['int64', 'float64'])
 cat_cols = df.select_dtypes(include = 'object')
 values = {}
for a in cat_cols:
values[a] = 'unknown'
for a in num_cols:
_index = df[df[a].isna() ].index
_value = np.random.normal(loc = df[a].mean() , scale = df[a].std() , size = df[a].isna().sum())
 df[a].fillna(pd.Series(_value, index=_index), inplace=True)
 df.fillna(value=values, inplace=True)
 HandleMissingValues(all_data)
 all_data.isnull().sum().sum()<prepare_x_and_y>
if fe["umap"]:
umap_components = 10
umap = UMAP(n_components = umap_components)
 umap_feat = umap.fit_transform(all_data[COLS])
 umap_df = pd.DataFrame(umap_feat, columns=[f"umap{i}" for i in range(umap_components)])
 all_data = pd.concat([all_data, umap_df], axis=1)
 UMAP_COLS = [c for c in all_data.columns if "umap" in c]
assert len(UMAP_COLS)> 0
all_data
X = all_data.copy()
y = X.pop("SalePrice")
 mi_scores = make_mi_scores(X, y)
 mi_scores<drop_column>
if fe["drop_original"]:
all_data = all_data.drop(COLS, axis=1 )
def drop_uninformative(df, mi_scores):
return df.loc[:, mi_scores > 0.0]<concatenate>
dtypes = {c: "int8" for c in train.columns if "_class" in c}
train = train.astype(dtypes)
 X_train, X_val, y_train, y_val = train_test_split(train, targets, test_size=0.1, random_state=42)
 train_pool = Pool(X_train, y_train, cat_features = [c for c in train.columns if "_class" in c])
 val_pool = Pool(X_val, y_val, cat_features = [c for c in train.columns if "_class" in c] )
mi_scores = mi_scores.append(pd.Series(0.01, index=["SalePrice"]))
all_data = drop_uninformative(all_data, mi_scores )<categorify>
ITERATIONS = 5000
MAX_EVALS = 150
def objective_func(params, train_pool, val_pool):
model = CatBoostRegressor(iterations = ITERATIONS,task_type="GPU", devices='0:1', grow_policy="Lossguide",
loss_function = "RMSE", custom_metric = "RMSE", eval_metric="RMSE", verbose = 1000, **params)
 model.fit(train_pool, eval_set = val_pool, early_stopping_rounds = 200, plot=False)
 loss = model.get_best_score()
return {"loss": loss["validation"]["RMSE"], "status": STATUS_OK}
space = {
"learning_rate": hp.loguniform("learning_rate", np.log(0.01), np.log(1)) ,
"max_depth": hp.quniform("max_depth", 6, 11, 1),
"l2_leaf_reg": hp.quniform("l2_leaf_reg", 1, 5, 0.9),
"bagging_temperature": hp.quniform("bagging_temperature", 0, 4, 0.9),
"min_data_in_leaf": hp.quniform("min_data_in_leaf", 1, 42, 1),
"max_leaves": hp.quniform("max_leaves", 2**4-1, 2**6-1, 1),
}
fn = partial(objective_func, train_pool=train_pool, val_pool=val_pool)
 best_params = fmin(fn = fn, space=space, algo=tpe.suggest, max_evals = MAX_EVALS)
 model = CatBoostRegressor(iterations = ITERATIONS,task_type="GPU", devices='0:1', grow_policy="Lossguide",
loss_function = "RMSE", custom_metric = "RMSE", eval_metric="RMSE", verbose = 200, **best_params)
 model.fit(train_pool, eval_set = val_pool, early_stopping_rounds = 200, plot=True )
all_data = pd.get_dummies(all_data)
 all_data.head()<prepare_x_and_y>
sub["target"] = model.predict(test)
 sub
<split><EOS> 
sub.to_csv("submission.csv", index=False )