Kaggle competition: Tabular Playground Series - Jan 2021
unique_test_shops = test["shop_id"].unique()
sales = sales[sales["shop_id"].isin(unique_test_shops)]
print(f"Number of Unique Shops in Test Data:{len(unique_test_shops)}
Number of Unique Shops in Sales Data:{len(sales['shop_id'].unique())}" )<drop_column>
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib_venn import venn2
import shap
from optuna.integration import _lightgbm_tuner as lgb_tuner
import optuna
from catboost import CatBoost
from catboost import Pool
from catboost import cv
import category_encoders as ce
from tqdm import tqdm
import lightgbm as lgb
import xgboost as xgb
import joblib
import os
import logging
import datetime
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn import preprocessing
from sklearn.preprocessing import LabelEncoder
from sklearn.decomposition import PCA
from scipy import stats
from sklearn.preprocessing import StandardScaler
from sklearn import metrics
from catboost import CatBoostRegressor
shops.drop("shop_name", axis=1, inplace=True )<feature_engineering>
df_train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 df_test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 submission = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv" )
shops["city"] = shops["clean"].apply(lambda x: x.split() [0] )<categorify>
y = df_train["target"]
X = df_train.drop(["target","id"], axis=1 )
le = LabelEncoder()
shops["city"] = le.fit_transform(shops["city"])
 shops.drop("clean", axis=1, inplace=True)
<feature_engineering>
fold_num = 10
EARLY_STOPPING_ROUNDS = 10
VERBOSE_EVAL = 10000
LGB_ROUND_NUM = 10000
objective = 'regression'
metric = 'rmse'
params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': objective,
'metric': metric,
'verbosity': -1,
"seed": 42,
}
@contextmanager
def timer(logger=None, format_str='{:.3f}[s]', prefix=None, suffix=None):
if prefix: format_str = str(prefix)+ format_str
if suffix: format_str = format_str + str(suffix)
 start = time()
yield
d = time() - start
out_str = format_str.format(d)
 if logger:
logger.info(out_str)
 else:
print(out_str)
 def fit_lgbm(X, y, cv, params: dict=None, verbose: int=50):
metric_func = mean_squared_error
if params is None:
params = {}
models = []
oof_pred = np.zeros_like(y, dtype=np.float)
 for i,(idx_train, idx_valid)in enumerate(cv):
x_train, y_train = X[idx_train], y[idx_train]
x_valid, y_valid = X[idx_valid], y[idx_valid]
lgb_train = lgb.Dataset(x_train, y_train)
 lgb_valid = lgb.Dataset(x_valid, y_valid)
 with timer(prefix='fit fold={} '.format(i + 1)) :
lgb_model = lgb_tuner.train(params, lgb_train,
num_boost_round=LGB_ROUND_NUM,
valid_names=["train", "valid"],
valid_sets=[lgb_train, lgb_valid],
early_stopping_rounds=EARLY_STOPPING_ROUNDS,
verbose_eval=VERBOSE_EVAL)
 pred_i = lgb_model.predict(x_valid)
 oof_pred[idx_valid] = pred_i
models.append(lgb_model)
 print(f'Fold {i} RMSE: {metric_func(y_valid, pred_i)**.5:.4f}')
 score = metric_func(y, oof_pred)**.5
print('FINISHED | Whole RMSE: {:.4f}'.format(score))
return oof_pred, models
items["item_name"] = items["item_name"].str.lower()
items["item_name_clean"] = items["item_name"].apply(cleans)
 items.drop("item_name", axis=1, inplace=True )<categorify>
fold = KFold(n_splits=5, shuffle=True, random_state=71)
 cv = list(fold.split(X, y))
oof, models = fit_lgbm(X.values, y, cv, params=params )
items["item_name_five"] = [x[:5] for x in items["item_name_clean"]]
items["item_name_five"] = le.fit_transform(items["item_name_five"])
 items.drop("item_name_clean", axis=1, inplace=True )<groupby>
def visualize_importance(models, feat_train_df):

 feature_importance_df = pd.DataFrame()
for i, model in enumerate(models):
_df = pd.DataFrame()
_df['feature_importance'] = model.feature_importance()
_df['column'] = feat_train_df.columns
_df['fold'] = i + 1
feature_importance_df = pd.concat([feature_importance_df, _df], axis=0, ignore_index=True)
 order = feature_importance_df.groupby('column')\ 
.sum() [['feature_importance']]\ 
.sort_values('feature_importance', ascending=False ).index[:50]
fig, ax = plt.subplots(figsize=(max(6, len(order)*.4), 7))
sns.boxenplot(data=feature_importance_df, x='column', y='feature_importance', order=order, ax=ax, palette='viridis')
 ax.tick_params(axis='x', rotation=90)
 ax.grid()
fig.tight_layout()
return fig, ax
items["first_sale_date"] = sales.groupby("item_id" ).agg({"date_block_num":"min"})["date_block_num"]
items<data_type_conversions>
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8)
 def opt(trial):
n_estimators = trial.suggest_int('n_estimators', 0, 1000)
 max_depth = trial.suggest_int('max_depth', 1, 20)
 learning_rate = trial.suggest_discrete_uniform('learning_rate', 0.01,0.1,0.01)
 min_child_weight = trial.suggest_int('min_child_weight', 1, 20)
 subsample = trial.suggest_discrete_uniform('subsample', 0.5, 0.9, 0.1)
 colsample_bytree = trial.suggest_discrete_uniform('colsample_bytree', 0.5, 0.9, 0.1)
 xgboost_tuna = xgb.XGBRegressor(
 random_state=42,
n_estimators = n_estimators,
max_depth = max_depth,
min_child_weight = min_child_weight,
subsample = subsample,
colsample_bytree = colsample_bytree, 
)
 xgboost_tuna.fit(X_train,y_train)
 tuna_pred_test = xgboost_tuna.predict(X_test)
 return mean_squared_error(y_test, tuna_pred_test )
items[items["first_sale_date"].isna() ]
items["first_sale_date"] = items["first_sale_date"].fillna(34 )<feature_engineering>
def fit_xgb(X, y, cv, params: dict=None, verbose: int=50):
metric_func = mean_squared_error
if params is None:
params = {}
models = []
oof_pred = np.zeros_like(y, dtype=np.float)
 for i,(idx_train, idx_valid)in enumerate(cv):
x_train, y_train = X[idx_train], y[idx_train]
x_valid, y_valid = X[idx_valid], y[idx_valid]
model_xgb = xgb.XGBRegressor(**params)
 with timer(prefix='fit fold={} '.format(i + 1)) :
model_xgb.fit(x_train, y_train, eval_set=[(x_valid, y_valid)],verbose=-1)
 pred_i = model_xgb.predict(x_valid)
 oof_pred[idx_valid] = pred_i
models.append(model_xgb)
 print(f'Fold {i} RMSE: {metric_func(y_valid, pred_i)**.5:.4f}')
 score = metric_func(y, oof_pred)**.5
print('FINISHED | Whole RMSE: {:.4f}'.format(score))
return oof_pred, models
categories["category"] = categories["category_name"].apply(lambda x: x.split() [0])
 categories<count_values>

 params_xgb = {'n_estimators': 208,
'max_depth': 4,
'learning_rate':0.08,
'min_child_weight': 13,
'subsample': 0.8,
'colsample_bytree': 0.8}
oof_xgb, models_xgb = fit_xgb(X.values, y, cv, params=params_xgb )
categories["category"].value_counts()<feature_engineering>
def opt_cb(trial):
params = {
'iterations' : trial.suggest_int('iterations', 50, 300),
'depth' : trial.suggest_int('depth', 4, 10),
'learning_rate' : trial.suggest_loguniform('learning_rate', 0.01, 0.3),
'random_strength' :trial.suggest_int('random_strength', 0, 100),
'bagging_temperature' :trial.suggest_loguniform('bagging_temperature', 0.01, 100.00),
'od_type': trial.suggest_categorical('od_type', ['IncToDec', 'Iter']),
'od_wait' :trial.suggest_int('od_wait', 10, 50)
 }
train_pool = Pool(X_train, y_train)
 test_pool = Pool(X_test, y_test)
 catboost_tuna = CatBoostRegressor(**params)
 catboost_tuna.fit(train_pool)
 tuna_pred_test = catboost_tuna.predict(test_pool)
 pred_labels = np.rint(tuna_pred_test)
 return mean_squared_error(y_test, pred_labels )
categories.loc[categories["category"] == "Game"] = "Games"<feature_engineering>
def fit_cb(X, y, cv, params: dict=None, verbose: int=50):
metric_func = mean_squared_error
if params is None:
params = {}
models = []
oof_pred = np.zeros_like(y, dtype=np.float)
 for i,(idx_train, idx_valid)in enumerate(cv):
x_train, y_train = X[idx_train], y[idx_train]
x_valid, y_valid = X[idx_valid], y[idx_valid]
train_pool = Pool(x_train, label = y_train)
 valid_pool = Pool(x_valid, label = y_valid)
 model_cb = CatBoost(params)
 with timer(prefix='fit fold={} '.format(i + 1)) :
model_cb.fit(train_pool,
eval_set = valid_pool,
use_best_model = True,
silent = True,
plot = False)
 print(model_cb.get_best_score())
 pred_i = model_cb.predict(x_valid)
 oof_pred[idx_valid] = pred_i
models.append(model_cb)
 print(f'Fold {i} RMSE: {metric_func(y_valid, pred_i)**.5:.4f}')
 score = metric_func(y, oof_pred)**.5
print('FINISHED | Whole RMSE: {:.4f}'.format(score))
return oof_pred, models
def make_misc(x):

 if len(categories[categories['category']==x])>= 5:
return x
else:
return 'Misc'
categories["cats"] = categories["category"].apply(make_misc)
 categories<drop_column>
params_cb = {
'loss_function': 'RMSE',
'max_depth': 3,
'learning_rate': 0.08,
'subsample': 0.8,
'num_boost_round': 1000,
'early_stopping_rounds': 100,
}
oof_cb, models_cb = fit_cb(X.values, y, cv, params=params_cb )
categories.drop(["category", "category_name"], axis=1, inplace=True )<drop_column>
df_test = df_test.drop("id",axis=1 )
categories["cats_le"] = le.fit_transform(categories["cats"])
 categories.drop("cats", inplace=True, axis=1 )<feature_engineering>
pred_lgb = np.array([model.predict(df_test.values)for model in models])
 pred_lgb = np.mean(pred_lgb, axis=0)
 pred_lgb = np.where(pred_lgb < 0, 0, pred_lgb)
 pred_xgb = np.array([model.predict(df_test.values)for model in models_xgb])
 pred_xgb = np.mean(pred_xgb, axis=0)
 pred_xgb = np.where(pred_xgb < 0, 0, pred_xgb)
 pred_cb = np.array([model.predict(df_test.values)for model in models_cb])
 pred_cb = np.mean(pred_cb, axis=0)
 pred_cb = np.where(pred_cb < 0, 0, pred_cb)
 tmp_sub = pd.DataFrame({"lgb":pred_lgb,
"xgb":pred_xgb,
"cb":pred_cb})
 tmp_sub["pred"] = tmp_sub.mean(axis="columns" )
sales = sales[sales["item_price"] > 0]
sales = sales[sales["item_price"] < 50000]
sales = sales[sales["item_cnt_day"] > 0]
sales = sales[sales["item_cnt_day"] < 1000]
sales["item_price"] = sales["item_price"].apply(lambda x: round(x,2))
sales<merge>
submission["target"] = tmp_sub["pred"].copy()
group = sales.groupby(index_feats ).agg({"item_cnt_day": "sum"})
 group = group.reset_index()
group = group.rename(columns={"item_cnt_day": "item_cnt_month"})
 train = pd.merge(train, group, on=index_feats, how="left")
 train<set_options>
submission.to_csv("submission.csv", index=False )
del group
gc.collect()<merge>
submission.to_csv("submission.csv", index=False )
<set_options><EOS> 
oof_df = pd.DataFrame({"lgb":oof,
"xgb":oof_xgb,
"cb":oof_cb})
 oof_df["pred"] = oof_df.mean(axis="columns" )