Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<merge>
%matplotlib inline
agg_prev = group(prev, 'PREV_', {**PREVIOUS_AGG, **categorical_agg})
 agg_prev = agg_prev.merge(active_agg_df, how='left', on='SK_ID_CURR')
 del active_agg_df; gc.collect()<merge>
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))
agg_prev = group_and_merge(approved, agg_prev, 'APPROVED_', PREVIOUS_APPROVED_AGG)
 refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
agg_prev = group_and_merge(refused, agg_prev, 'REFUSED_', PREVIOUS_REFUSED_AGG)
 del approved, refused; gc.collect()<merge>
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = {}):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(X_train, label=y_train, group_id=[0] * len(X_train))
self.val_data = Pool(X_val, label=y_val, group_id=[0] * len(X_val))
self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError
for loan_type in ['Consumer loans', 'Cash loans']:
type_df = prev[prev['NAME_CONTRACT_TYPE_{}'.format(loan_type)] == 1]
prefix = 'PREV_' + loan_type.split(" ")[0] + '_'
agg_prev = group_and_merge(type_df, agg_prev, prefix, PREVIOUS_LOAN_TYPE_AGG)
 del type_df; gc.collect()<feature_engineering>
ID_COL = "id"
FEAT_COLS = [f"cont{i}" for i in range(1, 15)]
TGT_COL = "target"
N_SPLITS =10
RANDOM_SEED = 2021
USE_MODEL = "lgb"
MODEL_PARAMS = {
"objective": "root_mean_squared_error",
"boosting": "gbdt",
"learning_rate": 0.01,
"seed": RANDOM_SEED,
'max_depth': -1,
'colsample_bytree':.85,
"subsample":.85,
"n_jobs": 2,
}
TRAIN_PARAMS = {
"num_boost_round": 20000,
"early_stopping_rounds": 300,
"verbose_eval": 100,
}
pay['LATE_PAYMENT'] = pay['DAYS_ENTRY_PAYMENT'] - pay['DAYS_INSTALMENT']
pay['LATE_PAYMENT'] = pay['LATE_PAYMENT'].apply(lambda x: 1 if x > 0 else 0)
 dpd_id = pay[pay['LATE_PAYMENT'] > 0]['SK_ID_PREV'].unique()<merge>
X = train[FEAT_COLS]
X_test = test[FEAT_COLS]
y = train[TGT_COL].values
kf = KFold(n_splits=N_SPLITS, shuffle=True, random_state=RANDOM_SEED)
 trn_val_indexs = list(kf.split(X, y))
agg_dpd = group_and_merge(prev[prev['SK_ID_PREV'].isin(dpd_id)], agg_prev,
'PREV_LATE_', PREVIOUS_LATE_PAYMENTS_AGG)
 del agg_dpd, dpd_id; gc.collect()<merge>
oof_pred_arr = np.zeros(len(X))
test_preds_arr = np.zeros(( N_SPLITS, len(X_test)))
 feature_importances = pd.DataFrame()
score_list = []
df = pd.merge(df, agg_prev, on='SK_ID_CURR', how='left' )<split>
for fold,(trn_idx, val_idx)in enumerate(trn_val_indexs):
print("*" * 100)
 print(f"Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=USE_MODEL)
 with timer(prefix="Model training"):
model.train(
 params=MODEL_PARAMS, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=TRAIN_PARAMS)
 fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
feature_importances = feature_importances.append(fi_tmp)
 val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([fold, score])
 oof_pred_arr[val_idx] = val_pred
test_pred = model.predict(X_test)
 test_preds_arr[fold] = test_pred
train = df[df['TARGET'].notnull() ]
test = df[df['TARGET'].isnull() ]
del df
del agg_prev
gc.collect()<drop_column>
oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["oof", oof_score])
 pd.DataFrame(
 score_list, columns=["fold", "rmse score"] )
<categorify><EOS> 
sub = smpl_sub.copy()
sub[TGT_COL] = test_preds_arr.mean(axis=0)
 sub.to_csv("submission.csv", index=False)
 sub.head()