Kaggle competition: Tabular Playground Series - Jan 2021
shop_mean_features = []
all_data, shop_mean_features = add_mean_feats(all_data, shop_mean_features, ["date_block_num", "shop_id", "category_id"] )<drop_column>
warnings.filterwarnings('ignore')
 RANDOM_SEED = 123
cat_mean_features = []
all_data, cat_mean_features = add_mean_feats(all_data, cat_mean_features, ["date_block_num", "category_id"] )<feature_engineering>
train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 sample = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv" )
all_data, cat_mean_features = add_mean_feats(all_data, cat_mean_features, ["date_block_num", "cats_le"] )<merge>
train['magic1'] = train['cont10']/train['cont11']
train['magic2'] = train['cont11']/train['cont10']
train['magic3'] = train['cont1']/train['cont7']
train['magic4'] = train['cont7']/train['cont1']
train['magic5'] = train['cont4']/train['cont6']
test['magic1'] = test['cont10']/test['cont11']
test['magic2'] = test['cont11']/test['cont10']
test['magic3'] = test['cont1']/test['cont7']
test['magic4'] = test['cont7']/test['cont1']
test['magic5'] = test['cont4']/test['cont6']
def add_lags(df, lag_features, index_features, lag_feature, lags=[1,2,3], clip=False):

 df_temp = df[index_features + [lag_feature]].copy()
for i in lags:
feat_name = lag_feature + "_lag" + str(i)
 df_temp.columns = index_features + [feat_name]
df_temp["date_block_num"] += i
df = pd.merge(df, df_temp.drop_duplicates() , on=index_features, how="left")
 df[feat_name] = df[feat_name].fillna(0)
 if clip:
lag_feats_to_clip.append(feat_name)
 df = downcast1(df)
 del df_temp
gc.collect()
return df, lag_feats_to_clip<define_variables>
train = train.drop('id', axis=1)
 test = test.drop('id', axis=1)
 X = train.drop('target', axis=1)
 y = train.target
lag_feats_to_clip = []
index_features = ["date_block_num", "shop_id", "item_id"]
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, index_features, "item_cnt_month", clip=True)
 all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, index_features, "item_cnt", clip=True)
 all_data.sample(10 )<prepare_x_and_y>
cat = CatBoostRegressor(iterations=1000 )
X_test_temp = all_data[all_data["date_block_num"]==34]
X_test_temp[item_mean_features].sum()<categorify>
model = [cat]
for mod in model:
score = cross_val_score(mod, X, y, cv=3, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("CAT RMSE Mean Score: ", np.mean(score))
for item in item_mean_features:
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, index_features, item, clip=True )<categorify>
model = [cat]
for mod in model:
score = cross_val_score(mod, X, y, cv=10, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("CAT RMSE Mean Score: ", np.mean(score))
for shop in shop_mean_features:
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, ["date_block_num", "shop_id", "category_id"],
shop, clip=True)
<drop_column>
lgbm = lightgbm.LGBMRegressor(random_state=RANDOM_SEED, n_jobs=-1, metric= 'rmse' )
for cat in cat_mean_features:
all_data, lag_feats_to_clip = add_lags(all_data, lag_feats_to_clip, ["date_block_num", "category_id"], cat, lags=[1,2,3], clip=True)
<drop_column>
model = [lgbm]
for mod in model:
score = cross_val_score(mod, X, y, cv=3, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("LGBM RMSE Mean Score: ", np.mean(score))
all_data = all_data.drop(item_mean_features, axis=1)
 all_data = all_data.drop(shop_mean_features, axis=1)
 all_data = all_data.drop(cat_mean_features, axis=1)
 all_data = all_data.drop(all_data[all_data["date_block_num"]<3].index )<set_options>
model = [lgbm]
for mod in model:
score = cross_val_score(mod, X, y, cv=10, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("LGBM RMSE Mean Score: ", np.mean(score))
del X_test_temp
gc.collect()<feature_engineering>
xgbr = XGBRegressor(random_state=RANDOM_SEED )
all_data["item_cnt_month_3lag_mean"] = all_data[["item_cnt_month_lag1", "item_cnt_month_lag2", "item_cnt_month_lag3"]].mean(axis=1 )<feature_engineering>
model = [xgbr]
for mod in model:
score = cross_val_score(mod, X, y, cv=3, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("XGB RMSE Mean Score: ", np.mean(score))
all_data[lag_feats_to_clip + ["item_cnt_month_3lag_mean",
"item_cnt_month"]] = all_data[lag_feats_to_clip + ["item_cnt_month_3lag_mean",
"item_cnt_month"]].clip(0,20 )<feature_engineering>
dtrain = lightgbm.Dataset(data=X, label=y)
 def hyp_lgbm(num_leaves, feature_fraction, bagging_fraction, max_depth, min_split_gain, min_child_weight, learning_rate):
params = {'application':'regression','num_iterations': 5000,
'early_stopping_round':100, 'metric':'rmse'}
params["num_leaves"] = int(round(num_leaves))
params['feature_fraction'] = max(min(feature_fraction, 1), 0)
 params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
 params['max_depth'] = int(round(max_depth))
params['min_split_gain'] = min_split_gain
params['min_child_weight'] = min_child_weight
params['learning_rate'] = learning_rate
cv_result = lightgbm.cv(params, dtrain, nfold=3,
seed=RANDOM_SEED, stratified=False,
verbose_eval =None, metrics=['rmse'])
 return -np.min(cv_result['rmse-mean'] )
all_data["lag_grad_1"] = all_data["item_cnt_month_lag1"] / all_data["item_cnt_month_lag2"]
all_data["lag_grad_1"] = all_data["lag_grad_1"].replace([np.inf, -np.inf], np.nan ).fillna(0)
 all_data["lag_grad_2"] = all_data["item_cnt_month_lag2"] / all_data["item_cnt_month_lag3"]
all_data["lag_grad_2"] = all_data["lag_grad_2"].replace([np.inf, -np.inf], np.nan ).fillna(0 )<feature_engineering>
pds = {
'num_leaves':(5, 50),
'feature_fraction':(0.2, 1),
'bagging_fraction':(0.2, 1),
'max_depth':(2, 20),
'min_split_gain':(0.001, 0.1),
'min_child_weight':(10, 50),
'learning_rate':(0.01, 0.5),
}
all_data["new_items"] = all_data["first_sale_date"] == all_data["date_block_num"]<drop_column>
def cat_hyp(depth, bagging_temperature, l2_leaf_reg, learning_rate):
params = {"iterations": 100,
"loss_function": "RMSE",
"verbose": False}
params["depth"] = int(round(depth))
params["bagging_temperature"] = bagging_temperature
params["learning_rate"] = learning_rate
params["l2_leaf_reg"] = l2_leaf_reg
cat_feat = []
cv_dataset = cgb.Pool(data=X,
label=y,
cat_features=cat_feat)
 scores = cgb.cv(cv_dataset,
params,
fold_count=3)
 return -np.min(scores['test-RMSE-mean'] )
all_data["time_since_first_sale"] = all_data["date_block_num"] - all_data["first_sale_date"]
all_data.drop("first_sale_date", inplace=True, axis=1 )<feature_engineering>
pds = {'depth':(4, 10),
'bagging_temperature':(0.1,10),
'l2_leaf_reg':(0.1, 10),
'learning_rate':(0.1, 0.2)
 }
all_data["month"] = all_data["date_block_num"] % 12<drop_column>
dtrain = xgb.DMatrix(X, y, feature_names=X.columns.values)
 def hyp_xgb(max_depth, subsample, colsample_bytree,min_child_weight, gamma, learning_rate):
params = {
'objective': 'reg:squarederror',
'eval_metric':'rmse',
'nthread':-1
}
params['max_depth'] = int(round(max_depth))
params['subsample'] = max(min(subsample, 1), 0)
 params['colsample_bytree'] = max(min(colsample_bytree, 1), 0)
 params['min_child_weight'] = int(min_child_weight)
 params['gamma'] = max(gamma, 0)
 params['learning_rate'] = learning_rate
scores = xgb.cv(params, dtrain, num_boost_round=500,verbose_eval=False,
early_stopping_rounds=10, nfold=3)
 return -scores['test-rmse-mean'].iloc[-1]
all_data.drop(["item_cnt"], axis=1, inplace=True )<data_type_conversions>
pds ={
'min_child_weight':(3, 20),
'gamma':(0, 5),
'subsample':(0.7, 1),
'colsample_bytree':(0.1, 1),
'max_depth':(3, 10),
'learning_rate':(0.01, 0.5)
 }
all_data["category_id"] = all_data["category_id"].astype("int8" )<prepare_x_and_y>
from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import LinearRegression
X_train = all_data[all_data["date_block_num"]<33]
y_train = X_train["item_cnt_month"]
X_train = X_train.drop("item_cnt_month", axis=1)
 X_val = all_data[all_data["date_block_num"] == 33]
y_val = X_val["item_cnt_month"]
X_val = X_val.drop("item_cnt_month", axis=1)
 X_test = all_data[all_data["date_block_num"]==34]
X_test = X_test.drop("item_cnt_month", axis=1)
 del all_data
gc.collect() 
<save_to_csv>
param_lgbm = {
'bagging_fraction': 0.973905385549851,
'feature_fraction': 0.2945585590881137,
'learning_rate': 0.03750332268701348,
'max_depth': int(7.66),
'min_child_weight': int(41.36),
'min_split_gain': 0.04033836353603582,
'num_leaves': int(46.42),
'application':'regression',
'num_iterations': 5000,
'metric': 'rmse'
}
param_cat = {
'bagging_temperature': 0.31768713094131684,
'depth': int(8.03),
'l2_leaf_reg': 1.3525686450404295,
'learning_rate': 0.2,
'iterations': 100,
'loss_function': 'RMSE',
'verbose': False
}
param_xgb = {
'colsample_bytree': 0.8119098377889549,
'gamma': 2.244423418642122,
'learning_rate': 0.015800631696721114,
'max_depth': int(9.846),
'min_child_weight': int(15.664),
'subsample': 0.82345,
'objective': 'reg:squarederror',
'eval_metric':'rmse',
'num_boost_roun' : 500
}
def preds(model, test, name):

 prediction = model.predict(test)
 df_sub = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sample_submission.csv")
 df_sub["item_cnt_month"] = prediction.clip(0,20)
 df_sub.to_csv(f"{name}.csv", index=False)
 print("Complete." )<create_dataframe>
from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn import svm
import lightgbm
params = {'metric': 'rmse',
'num_leaves': 255,
'learning_rate': 0.005,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'force_col_wise' : True,
'random_state': 10,
'num_rounds':1500,
'early_stopping':150}
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_val = lgb.Dataset(X_val, y_val)
 model = lgb.train(params=params, train_set=lgb_train, valid_sets=(lgb_train, lgb_val), verbose_eval=50 )<predict_on_test>
estimators = [ 
('lgbm', lightgbm.LGBMRegressor(**param_lgbm, random_state=RANDOM_SEED, n_jobs=-1)) , 
('xgbr', XGBRegressor(**param_xgb, random_state=RANDOM_SEED, nthread=-1)) , 
('cat', CatBoostRegressor(**param_cat)) , 
('mlp', MLPRegressor())
 ]
preds(model, X_test, "lgb_model" )<set_options>
reg = StackingRegressor(
 estimators=estimators,
final_estimator=LinearRegression() ,
n_jobs=-1,
cv=5 
)
 reg.fit(X, y)
 y_pred = reg.predict(test )
<train_model><EOS> 
sample['target'] = y_pred
sample.to_csv("submission.csv", index=False )