Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<count_values>
plt.style.use('fivethirtyeight')
 y_ = Fore.YELLOW
r_ = Fore.RED
g_ = Fore.GREEN
b_ = Fore.BLUE
m_ = Fore.MAGENTA
c_ = Fore.CYAN
sr_ = Style.RESET_ALL
warnings.filterwarnings('ignore')
%%time
display=full_data["display_address"].value_counts()
manager_id=full_data["manager_id"].value_counts()
building_id=full_data["building_id"].value_counts()
street=full_data["street_address"].value_counts()
bedrooms=full_data["bedrooms"].value_counts()
bathrooms=full_data["bathrooms"].value_counts()
created_dayofyear=full_data["created_dayofyear"].value_counts()
created_weekofyear=full_data["created_weekofyear"].value_counts()
full_data["display_count"]=full_data["display_address"].apply(lambda x:display[x])
 full_data["manager_count"]=full_data["manager_id"].apply(lambda x:manager_id[x])
 full_data["building_count"]=full_data["building_id"].apply(lambda x:building_id[x])
 full_data["street_count"]=full_data["street_address"].apply(lambda x:street[x])
 full_data["bedrooms_count"]=full_data["bedrooms"].apply(lambda x:bedrooms[x])
 full_data["bathrooms_count"]=full_data["bathrooms"].apply(lambda x:bathrooms[x])
 full_data["created_dayofyear_count"]=full_data["created_dayofyear"].\
apply(lambda x:created_dayofyear[x])
 full_data["created_weekofyear_count"]=full_data["created_weekofyear"].\
apply(lambda x:created_weekofyear[x])
 count_vars = ['manager_count', 'building_count', 'street_count', 'bedrooms_count',
'bathrooms_count', 'created_dayofyear_count', 'created_weekofyear_count']<merge>
path = '.. /input/tabular-playground-series-jan-2021/'
train_data = pd.read_csv(path + 'train.csv')
 test_data = pd.read_csv(path + 'test.csv')
 sample = pd.read_csv(path + 'sample_submission.csv' )
num_cat_vars =[]
price_by_manager = full_data.groupby('manager_id')['price'].agg([np.min,np.max,np.median,np.mean] ).reset_index()
price_by_manager.columns = ['manager_id','min_price_by_manager',
'max_price_by_manager','median_price_by_manager','mean_price_by_manager']
full_data = pd.merge(full_data,price_by_manager, how='left',on='manager_id')
 price_by_building = full_data.groupby('building_id')['price'].agg([np.min,np.max,np.median,np.mean] ).reset_index()
price_by_building.columns = ['building_id','min_price_by_building',
'max_price_by_building','median_price_by_building','mean_price_by_building']
full_data = pd.merge(full_data,price_by_building, how='left',on='building_id')
 full_data['price_percentile_by_manager']=\
full_data[['price','min_price_by_manager','max_price_by_manager']]\ 
.apply(lambda x:(x[0]-x[1])/(x[2]-x[1])if(x[2]-x[1])!=0 else 0.5,
axis=1)
 full_data['price_percentile_by_building']=\
full_data[['price','min_price_by_building','max_price_by_building']]\ 
.apply(lambda x:(x[0]-x[1])/(x[2]-x[1])if(x[2]-x[1])!=0 else 0.5,
axis=1)
 num_cat_vars.append('price_percentile_by_manager')
 num_cat_vars.append('price_percentile_by_building')
 print(num_cat_vars )<data_type_conversions>
train_data['cont13_cont4_mul'] = train_data['cont13']*train_data['cont4']
train_data['cont13_cont11_mul'] = train_data['cont13']*train_data['cont11']
train_data['cont13_cont7_mul'] = train_data['cont13']*train_data['cont7']
train_data['cont13_cont2_mul'] = train_data['cont13']*train_data['cont2']
train_data['cont13_cont10_mul'] = train_data['cont13']*train_data['cont10']
test_data['cont13_cont4_mul'] = test_data['cont13']*test_data['cont4']
test_data['cont13_cont11_mul'] = test_data['cont13']*test_data['cont11']
test_data['cont13_cont7_mul'] = test_data['cont13']*test_data['cont7']
test_data['cont13_cont2_mul'] = test_data['cont13']*test_data['cont2']
test_data['cont13_cont10_mul'] = test_data['cont13']*test_data['cont10']
for comb in itertools.combinations(cat_vars, 2):
comb_var_name = comb[0] +'-'+ comb[1]
full_data [comb_var_name] = full_data [ comb[0]].astype(str)+'_' + full_data [ comb[1]].astype(str)
 cat_vars.append(comb_var_name)
 cat_vars<feature_engineering>
num_bins = int(1 + np.log2(len(train_data)))
 train_data.loc[:,'bins'] = pd.cut(train_data['target'].to_numpy() ,bins=num_bins,labels=False)
 features = [f'cont{x}' for x in range(1,15)]
features += [
'cont13_cont4_mul',
'cont13_cont11_mul',
'cont13_cont7_mul',
'cont13_cont2_mul',
'cont13_cont10_mul',
]
target_feature = 'target'
train_data = train_data.query('target >=5')
 bins = train_data['bins'].to_numpy()
target = train_data[target_feature].to_numpy()
train_data = train_data[features].to_numpy()
test_data = test_data[features].to_numpy()
scaler = PowerTransformer()
train_data = scaler.fit_transform(train_data)
 test_data = scaler.transform(test_data )
full_data["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
cntvec = CountVectorizer(stop_words='english', max_features=200)
 feature_sparse =cntvec.fit_transform(full_data["features"]\ 
.apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x])))
 feature_vars = ['feature_' + v for v in cntvec.vocabulary_]
cntvec = CountVectorizer(stop_words='english', max_features=100)
 desc_sparse = cntvec.fit_transform(full_data["description"])
 desc_vars = ['desc_' + v for v in cntvec.vocabulary_]
cntvec = CountVectorizer(stop_words='english', max_features=10)
 st_addr_sparse = cntvec.fit_transform(full_data["street_address"])
 st_addr_vars = ['desc_' + v for v in cntvec.vocabulary_]<categorify>
def rmse_score(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
LBL = preprocessing.LabelEncoder()
LE_vars=[]
LE_map=dict()
for cat_var in cat_vars:
print("Label Encoding %s" %(cat_var))
LE_var=cat_var+'_le'
full_data[LE_var]=LBL.fit_transform(full_data[cat_var])
 LE_vars.append(LE_var)
 LE_map[cat_var]=LBL.classes_
print("Label-encoded feaures: %s" %(LE_vars))<categorify>
seed = 2021
nfolds = 5
params={
'objective':'regression',
'metrics':'rmse',
'boosting':'gbdt',
'min_data_per_group': 5,
'num_leaves': 256,
'max_depth': -1,
'learning_rate': 0.005,
'subsample_for_bin': 200000,
'lambda_l1': 1.074622455507616e-05,
'lambda_l2': 2.0521330798729704e-06,
'n_jobs': -1,
'cat_smooth': 1.0,
'silent': True,
'importance_type': 'gain',
'feature_pre_filter': False,
'bagging_fraction': 0.8206341150202605,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 0.001,
'bagging_freq': 6,
'feature_fraction': 0.5,
'min_gain_to_split': 0.0,
'min_child_samples': 20}
OHE = preprocessing.OneHotEncoder(sparse=True)
 start=time.time()
OHE.fit(full_data[LE_vars])
 OHE_sparse=OHE.transform(full_data[LE_vars])
 print('One-hot-encoding finished in %f seconds' %(time.time() -start))
OHE_vars = [var[:-3] + '_' + str(level ).replace(' ','_')\
for var in cat_vars for level in LE_map[var] ]
print("OHE_sparse size :" ,OHE_sparse.shape)
 print("One-hot encoded catgorical feature samples : %s" %(OHE_vars[:100]))<prepare_x_and_y>
lgbm_preds = np.zeros(test_data.shape[0])
 kfold = StratifiedKFold(n_splits=nfolds,random_state=seed)
 lgbm_scores = list()
for train_idx, valid_idx in kfold.split(X=train_data,y=bins):
lgb_train = lgb.Dataset(train_data[train_idx],target[train_idx])
 lgb_valid = lgb.Dataset(train_data[valid_idx],target[valid_idx],reference=lgb_train)
 lgb_model = lgb.train(params,
lgb_train,
valid_sets=[lgb_train,lgb_valid],
num_boost_round=10000,
verbose_eval=200,
early_stopping_rounds=100, 
)
 lgbm_scores.append(rmse_score(target[valid_idx],lgb_model.predict(train_data[valid_idx])))
 lgbm_preds += lgb_model.predict(test_data)/nfolds
print("mean rmse score",np.mean(lgbm_scores))
full_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars +geo_num_vars+ count_vars + LE_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,"testing data size: ", test_x.shape )<categorify>
params = {'lambda': 0.0030282073258141168,
'alpha': 0.01563845128469084,
'colsample_bytree': 0.55,
'subsample': 0.7,
'learning_rate': 0.01,
'max_depth': 15,
'random_state': 2020,
'min_child_weight': 257,
}
class MeanEncoder:
def __init__(self, categorical_features, n_splits=5, target_type='classification', prior_weight_func=None):

 self.categorical_features = categorical_features
self.n_splits = n_splits
self.learned_stats = {}
if target_type == 'classification':
self.target_type = target_type
self.target_values = []
else:
self.target_type = 'regression'
self.target_values = None
if isinstance(prior_weight_func, dict):
self.prior_weight_func = eval('lambda x: 1 /(1 + np.exp(( x - k)/ f)) ', dict(prior_weight_func, np=np))
elif callable(prior_weight_func):
self.prior_weight_func = prior_weight_func
else:
self.prior_weight_func = lambda x: 1 /(1 + np.exp(( x - 2)/ 1))
@staticmethod
def mean_encode_subroutine(X_train, y_train, X_test, variable, target, prior_weight_func):
X_train = X_train[[variable]].copy()
X_test = X_test[[variable]].copy()
if target is not None:
nf_name = '{}_pred_{}'.format(variable, target)
 X_train['pred_temp'] =(y_train == target ).astype(int)
 else:
nf_name = '{}_pred'.format(variable)
 X_train['pred_temp'] = y_train
prior = X_train['pred_temp'].mean()
col_avg_y = X_train.groupby(by=variable, axis=0)['pred_temp'].agg({'mean': 'mean', 'beta': 'size'})
 col_avg_y['beta'] = prior_weight_func(col_avg_y['beta'])
 col_avg_y[nf_name] = col_avg_y['beta'] * prior +(1 - col_avg_y['beta'])* col_avg_y['mean']
col_avg_y.drop(['beta', 'mean'], axis=1, inplace=True)
 nf_train = X_train.join(col_avg_y, on=variable)[nf_name].values
nf_test = X_test.join(col_avg_y, on=variable ).fillna(prior, inplace=False)[nf_name].values
return nf_train, nf_test, prior, col_avg_y
def fit_transform(self, X, y):

 X_new = X.copy()
if self.target_type == 'classification':
skf = StratifiedKFold(self.n_splits)
 else:
skf = KFold(self.n_splits)
 if self.target_type == 'classification':
self.target_values = sorted(set(y))
self.learned_stats = {'{}_pred_{}'.format(variable, target): [] for variable, target in
product(self.categorical_features, self.target_values)}
for variable, target in product(self.categorical_features, self.target_values):
nf_name = '{}_pred_{}'.format(variable, target)
 X_new.loc[:, nf_name] = np.nan
for large_ind, small_ind in skf.split(y, y):
nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(
 X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, target, self.prior_weight_func)
 X_new.iloc[small_ind, -1] = nf_small
self.learned_stats[nf_name].append(( prior, col_avg_y))
else:
self.learned_stats = {'{}_pred'.format(variable): [] for variable in self.categorical_features}
for variable in self.categorical_features:
nf_name = '{}_pred'.format(variable)
 X_new.loc[:, nf_name] = np.nan
for large_ind, small_ind in skf.split(y, y):
nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(
 X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, None, self.prior_weight_func)
 X_new.iloc[small_ind, -1] = nf_small
self.learned_stats[nf_name].append(( prior, col_avg_y))
return X_new
def transform(self, X):

 X_new = X.copy()
if self.target_type == 'classification':
for variable, target in product(self.categorical_features, self.target_values):
nf_name = '{}_pred_{}'.format(variable, target)
 X_new[nf_name] = 0
for prior, col_avg_y in self.learned_stats[nf_name]:
X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable ).fillna(prior, inplace=False)[
nf_name]
X_new[nf_name] /= self.n_splits
else:
for variable in self.categorical_features:
nf_name = '{}_pred'.format(variable)
 X_new[nf_name] = 0
for prior, col_avg_y in self.learned_stats[nf_name]:
X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable ).fillna(prior, inplace=False)[
nf_name]
X_new[nf_name] /= self.n_splits
return X_new<categorify>
xgb_preds = np.zeros(test_data.shape[0])
 kfold = StratifiedKFold(n_splits=nfolds,random_state=seed)
 params['random_state'] = seed
xgb_scores = list()
for train_idx, valid_idx in kfold.split(X=train_data,y=bins):
xgb_train = xgb.DMatrix(train_data[train_idx],label=target[train_idx])
 xgb_valid = xgb.DMatrix(train_data[valid_idx],label=target[valid_idx])
 xgb_model = xgb.train(params,
xgb_train,
10000,
verbose_eval=200,
evals=[(xgb_train,'train'),(xgb_valid,'valid')],
early_stopping_rounds=100)
 xgb_scores.append(rmse_score(target[valid_idx],xgb_model.predict(xgb.DMatrix(train_data[valid_idx]))))
xgb_preds += xgb_model.predict(xgb.DMatrix(test_data)) /nfolds
print("mean rmse score",np.mean(xgb_scores))
mean_encoder = MeanEncoder(categorical_features=['manager_id','building_id'])
 mean_encoded_train = mean_encoder.fit_transform(train_data, train_data['target'])
 mean_encoded_test = mean_encoder.transform(test_data)
 mean_coded_vars = list(set(mean_encoded_train.columns)- set(train_data.columns))
mean_coded_vars.append('listing_id')
 full_data = pd.merge(full_data,
pd.concat([mean_encoded_train[mean_coded_vars], mean_encoded_test[mean_coded_vars]]),
how='left',
on='listing_id'
)<categorify>
params = {'l2_leaf_reg': 0.02247766515106271,
'max_bin': 364,
'subsample': 0.6708650091202213,
'learning_rate': 0.010290546311954876,
'max_depth': 10,
'verbose':200,
'random_state': seed,
'min_data_in_leaf': 300,
'loss_function': 'RMSE',
'n_estimators': 25000,
'rsm':0.5,
'early_stopping_rounds':100}
CB_encoder = PolynomialWrapper(CatBoostEncoder())
 train_cb = CB_encoder.fit_transform(full_data[:train_size][cat_vars], full_data[:train_size]['target'])
 test_cb = CB_encoder.transform(full_data[train_size:][cat_vars])
 CB_vars = [f'cb_{c}' for c in train_cb.columns]
train_cb.columns = CB_vars
test_cb.columns = CB_vars
full_data = pd.concat([full_data, pd.concat([train_cb, test_cb])], axis=1 )<prepare_x_and_y>
cat_preds = np.zeros(test_data.shape[0])
 kfold = StratifiedKFold(n_splits=nfolds,random_state=seed)
 params['random_state'] = seed
cat_scores = list()
for train_idx, valid_idx in kfold.split(X=train_data,y=bins):
cat_train = Pool(train_data[train_idx],target[train_idx])
 cat_valid = Pool(train_data[valid_idx],target[valid_idx])
 cat_model = CatBoost(params)
 cat_model.fit(cat_train,eval_set=cat_valid)
 cat_scores.append(rmse_score(target[valid_idx],cat_model.predict(train_data[valid_idx])))
 cat_preds += cat_model.predict(test_data)/nfolds
print('mean rmse score:',np.mean(cat_scores))
full_vars = num_vars + date_num_vars + additional_num_vars + interactive_num_vars+ geo_cat_vars +geo_num_vars+ count_vars + LE_vars + mean_coded_vars + CB_vars
train_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [:train_size]
train_y = full_data['target'][:train_size].values
test_x = sparse.hstack([full_data[full_vars],
feature_sparse,
desc_sparse,
st_addr_sparse] ).tocsr() [train_size:]
test_y = full_data['target'][train_size:].values
full_vars = full_vars + feature_vars + desc_vars + st_addr_vars
print("training data size: ", train_x.shape,"testing data size: ", test_x.shape )<feature_engineering>
stacking_preds = pd.read_csv('.. /input/tps-simple-stacking/submission.csv')
 stacking_preds = stacking_preds.target.to_numpy()
<prepare_x_and_y><EOS> 
sample.target =(0.6 * lgbm_preds.ravel() + 0.2 * xgb_preds.ravel() + 0.1 * cat_preds.ravel() + 0.1 * stacking_preds)
 sample.to_csv("submission.csv",index=False)
 sample.head()