Kaggle competition: Predict Future Sales
voted = []
for x in range(0, len(ids)) :
a = np.array(predictions)[:,x] 
(values, counts)= np.unique(a,return_counts=True)
 if max(c)>=(top_n/2):
pred = values[counts==max(counts)][0]
else:
pred = int(a[np.argmax(scores)])
 voted.append(pred)
<save_to_csv>
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )
Y = pd.DataFrame()
Y["id"] = ids
Y["type"] = dummies.columns[voted]
Y.to_csv("submission_voter.csv",index=False )<import_modules>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]
pd.set_option('display.max_columns', None)
 warnings.filterwarnings(action="ignore")
 pd.options.display.max_seq_items = 8000
pd.options.display.max_rows = 8000<load_from_csv>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
testing_set_scores = pd.DataFrame(data=[
['predection_output_stack_gen.csv(the final stacked model)', 0.55600],
['prediction_abr_lasso.csv', 0.54510],
['prediction_abr_tree.csv', 0.54816],
['prediction_gbr.csv', 0.54475],
['prediction_gbr_2.csv', 0.54645],
['prediction_lasso.csv', 0.54711],
['prediction_lgm.csv', 0.54998],
['prediction_lightgbm_2.csv', 0.54869],
['prediction_lightgbm_goss.csv', 0.55056],
['prediction_rf.csv', 0.55127],
['prediction_rf_2.csv', 0.55164],
['prediction_ridge.csv', 0.54308]
],columns=['model','score'] ).sort_index(axis=0, ascending=False)
 plt.barh(testing_set_scores['model'], testing_set_scores['score'])
 plt.xlim(0.5405,0.5576)
 for score, model in zip(testing_set_scores['score'], testing_set_scores['model']):
plt.text(x=score, y=model, s=score, ha='left', va='center', fontsize=9 )<load_from_csv>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train = pd.read_csv('/kaggle/input/train.csv', index_col=0)
 test = pd.read_csv('/kaggle/input/test.csv', index_col=0)
 print("train.shape:", train.shape, "; test.shape:",test.shape, "; columns only in train.csv:", set(train.columns)- set(test.columns))
train.head(n=2 )<count_missing_values>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )
def percent_missing(df):
data = pd.DataFrame(df)
 df_cols = list(pd.DataFrame(data))
dict_x = {}
for i in range(0, len(df_cols)) :
dict_x.update({df_cols[i]: round(data[df_cols[i]].isnull().mean() *100,2)})
 return dict_x
train_features_missing = percent_missing(train_features)
 print('Top 5 percent train_features missing', sorted(train_features_missing.items() , key=lambda x: x[1], reverse=True)[:5])
 test_features_missing = percent_missing(test_features)
 print('Top 5 percent train_features missing', sorted(test_features_missing.items() , key=lambda x: x[1], reverse=True)[:5] )<define_variables>
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )
category_feature_names = [fea for fea in train_features.columns if fea != 'y' and train_features[fea].dtype == 'object']
binary_feature_names = [fea for fea in train_features.columns if fea != 'y' and train_features[fea].dtype != 'object']<categorify>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
def encoding_features_with_expend(train_features, test_features, n_comp=12):
print('input: train shape={}; test shape={}'.format(train_features.shape, test_features.shape))
category_feature_names = [fea for fea in train_features.columns if train_features[fea].dtype == 'object']
oh_encoder = OneHotEncoder(handle_unknown='ignore')
 oh_encoder.fit(list(train_features[category_feature_names].values)+ list(test_features[category_feature_names].values))
train_cat_fea_oh_ndarray = oh_encoder.transform(train_features[category_feature_names] ).toarray()
test_cat_fea_oh_ndarray = oh_encoder.transform(test_features[category_feature_names] ).toarray()
train_cat_fea_oh = pd.DataFrame(data=train_cat_fea_oh_ndarray)
 test_cat_fea_oh = pd.DataFrame(data=test_cat_fea_oh_ndarray)
 print("train_cat_fea_oh.shape:", train_cat_fea_oh.shape)
 train_cat_fea_oh.index = train_features.index
test_cat_fea_oh.index = test_features.index
train_non_cat_fea = train_features.drop(category_feature_names, axis=1)
 test_non_cat_fea = test_features.drop(category_feature_names, axis=1)
 train_fea_oh = pd.concat([train_cat_fea_oh, train_non_cat_fea], axis=1)
 test_fea_oh = pd.concat([test_cat_fea_oh, test_non_cat_fea], axis=1)
 print('after one-hot: train shape={}; test shape={}'.format(train_fea_oh.shape, test_fea_oh.shape))
return train_fea_oh, test_fea_oh
train_features_oh, test_features_oh = encoding_features_with_expend(train_features, test_features )<define_variables>
train['revenue'] = train['item_price'] * train['item_cnt_day']
train_features, test_features = train_features_oh, test_features_oh<prepare_x_and_y>
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts
BASE_MODEL_KF = 7
STACK_MODEL_KF = 7
is_verify_code = False
X = train_features
y = train_labels
X_test = test_features
if is_verify_code:
X = X.head(n=BASE_MODEL_KF * STACK_MODEL_KF)
 y = y.head(n=BASE_MODEL_KF * STACK_MODEL_KF)
 X_test = X_test.head(n=BASE_MODEL_KF * STACK_MODEL_KF)
 X.shape, y.shape, X_test.shape<compute_train_metric>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
kf = KFold(n_splits=BASE_MODEL_KF, random_state=37, shuffle=True)
 def cv_rmse(model, X=X, y=y):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def cv_r2(model, X=X, y=y, print_params=True):
r2 = cross_val_score(model, X, y, scoring="r2", cv=kf)
 print("---"*20)
 if print_params:
print("cv_r2: mean=", r2.mean() , "; std=", r2.std())
 print("--- model params: -------
")
 print(model)
 else:
print("cv_r2: ", r2.mean() , ";", r2.std())
 print("---"*20)
 return(r2)
 def plot_grid_cv_results(grid_with_train_score_returned):
grid = grid_with_train_score_returned
cv_results = grid.cv_results_
rst = pd.DataFrame(cv_results)[['mean_test_score', 'mean_train_score', 'std_test_score', 'std_train_score']]
rst[['mean_test_score', 'mean_train_score']].plot.bar(figsize=(50,5), grid=True)
 rst[['std_test_score', 'std_train_score']].plot.bar(figsize=(50,5), grid=True)
 print('---'*20)
 print(rst)
 print('---'*20)
 for i in range(0, len(cv_results['params'])) :
print(i, "\t:", cv_results['params'][i])
 print('---'*20)
 print('best_index:', grid.best_index_)
 print('best_score:', grid.best_score_)
 print('best_param:', grid.best_params_)
 return rst<define_search_space>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
%%time
ridge = Pipeline([('ridge', Ridge(alpha=45, normalize=False, random_state=42)) ])
 normalize_alpha_list = list(range(25,35,2)) + list(range(35,45,1)) + list(range(45,55,2))
def grid_search_ridge(model=ridge):
params = [
{'ridge__alpha':normalize_alpha_list, 'ridge__normalize':[False]}
]
grid = GridSearchCV(model, params, cv=kf, scoring="r2", return_train_score=True)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
ridge = grid_search_ridge(ridge)
 cv_r2(ridge, X=X, y=y, print_params=False )<choose_model_class>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
%%time
rf = RandomForestRegressor(
 max_depth = 4,
n_estimators = 1000,
min_impurity_decrease = 0.175,
criterion = "mse",
min_samples_split = 2,
min_samples_leaf = 1,
min_weight_fraction_leaf = 0.02,
max_features = 0.88,
oob_score = True,
n_jobs = -1,
warm_start = False,
random_state = 42)
 def grid_search_random_forest(model = rf):
params = {"max_depth": [4],
"max_features": [0.88],
"min_weight_fraction_leaf":[0.02],
"min_impurity_decrease":[0.175, 0.2, 0.25],
"n_estimators":[1000]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
rf = grid_search_random_forest() 
<choose_model_class>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
%%time
rf_2 = RandomForestRegressor(
 n_estimators = 900,
max_features = None,
max_depth = 5,
min_samples_split = 5,
min_samples_leaf = 5,
oob_score = True,
random_state = 42)
 def grid_search_random_forest_2(model = rf_2):
params = {"n_estimators":[900],
"max_depth":[4,5,6],
"min_samples_split":[5],
"min_samples_leaf":[5]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
rf_2 = grid_search_random_forest_2() 
<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
gbr = GradientBoostingRegressor(
 n_estimators = 1000,
learning_rate = 0.006,
max_features = 1.0,
min_impurity_decrease = 0.5,
subsample = 1,
max_depth = 3,
max_leaf_nodes = None,
min_samples_leaf = 1,
min_samples_split = 2,
loss = 'huber',
random_state = 42)
 def grid_search_gradient_boosting_regressor(model = gbr):
params = [{
"max_depth":[None], "max_leaf_nodes":[6]
}, {
"max_depth":[3], "max_leaf_nodes":[None]
}]
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
gbr = grid_search_gradient_boosting_regressor() 
<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
gbr_2 = GradientBoostingRegressor(
 n_estimators = 500,
learning_rate = 0.01,
max_depth = 4,
max_features = 0.6,
min_samples_leaf = 4,
min_samples_split = 40,
loss = 'huber',
random_state = 42)
 def grid_search_gradient_boosting_regressor_2(model = gbr_2):
params = {
"n_estimators":[500],
"learning_rate":[0.01],
"max_features":[0.8, 0.85],
"min_samples_split":[110, 120, 130],
"min_samples_leaf":[4],
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
gbr_2 = grid_search_gradient_boosting_regressor_2() 
<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
lightgbm = LGBMRegressor(
 boosting_type = 'gbdt',
objective = 'regression',
num_leaves = 31,
max_depth = 3,
learning_rate = 0.002,
n_estimators = 1000,
min_split_gain = 0.1,
max_bin = 40,
reg_alpha = 0,
reg_lambda = 0,
bagging_freq = 4,
bagging_fraction = 1.0,
feature_fraction = 1.0,
min_sum_hessian_in_leaf = 0.001,
n_jobs=-1, verbose = -1, random_state = 42)
 def grid_search_light_gbm(model = lightgbm):
params = {
"n_estimators": [1000],
"learning_rate": [0.002, 0.00225, 0.0025, 0.00275, 0.003],
"max_bin": [40],
"max_depth" : [3]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
lightgbm = grid_search_light_gbm() 
<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
lightgbm_2 = LGBMRegressor(
 boosting_type = 'gbdt',
objective = 'regression',
num_leaves = 60,
max_depth = -1,
learning_rate = 0.01,
n_estimators = 350,
max_bin = 90,
bagging_fraction = 0.35,
reg_alpha = 0,
reg_lambda = 0,
bagging_freq = 4,
bagging_seed = 8,
feature_fraction = 0.8,
feature_fraction_seed = 8,
verbose = -1, random_state = 42)
 def grid_search_light_gbm_2(model = lightgbm_2):
params = [{
"num_leaves": [None], "max_depth":[3],
"feature_fraction": [0.8, 0.84, 0.88, 0.92]
}, {
"num_leaves": [6], "max_depth":[None],
"feature_fraction": [0.8, 0.84, 0.88, 0.92]
}]
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
lightgbm_2 = grid_search_light_gbm_2() 
<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
abr_tree = AdaBoostRegressor(
 base_estimator = DecisionTreeRegressor(
 max_depth = 4,
min_samples_split = 2,
min_samples_leaf = 1,
min_weight_fraction_leaf = 0.0,
min_impurity_decrease = 0.4,
random_state = 53),
n_estimators = 1000,
learning_rate = 0.001,
loss = "exponential",
random_state = 42)
 def grid_search_ada_boost_tree_regressor(model = abr_tree):
params = {
"base_estimator__max_depth": [4],
"base_estimator__min_impurity_decrease":[0.55,0.6],
"base_estimator__min_weight_fraction_leaf":[0.0],
"learning_rate" : [0.001],
"loss" : ['exponential']
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
abr_tree = grid_search_ada_boost_tree_regressor() 
<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
lightgbm_goss = LGBMRegressor(
 boosting_type = 'goss',
objective = 'regression',
num_leaves = 31,
max_depth = 3,
learning_rate = 0.005,
n_estimators = 1000,
min_split_gain = 0.1,
max_bin = 40,
reg_alpha = 0,
reg_lambda = 0,
bagging_freq = 4,
bagging_fraction = 1.0,
feature_fraction = 1.0,
min_sum_hessian_in_leaf = 0.001,
n_jobs=-1, verbose = -1, random_state = 42 
)
 def grid_search_light_gbm_goss(model = lightgbm_goss):
params = {
"boosting_type":["goss"],
"n_estimators": [1000],
"learning_rate": [0.002],
"max_bin": [80],
"max_depth" : [3],
"num_leaves" : [6,7]
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
lightgbm_goss = grid_search_light_gbm_goss()<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
alpha_list = [0.02, 0.025, 0.03, 0.05, 0.1, 0.5, 1, 1.5, 3, 6, 10, 13, 16, 20, 23, 26, 30, 33, 36, 40, 43, 46, 50]
abr_lasso = AdaBoostRegressor(
 base_estimator = LassoCV(alphas=alpha_list, normalize=False, random_state=42),
n_estimators = 24,
learning_rate = 0.0005,
loss = "exponential",
random_state = 42)
 def grid_search_abr_lasso(model = abr_lasso):
params = {
"n_estimators" : [18,12],
"learning_rate" : [0.0005, 0.001],
}
grid = GridSearchCV(model, params, scoring="r2", return_train_score=True, cv=kf)
 grid.fit(X, y)
 plot_grid_cv_results(grid)
 return grid.best_estimator_
abr_lasso = grid_search_abr_lasso(abr_lasso)
<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
alphas_list_1 = [0.001, 0.003, 0.005, 0.007, 0.009]
alphas_list_2 = [0.1, 0.3, 0.5, 0.7, 0.9]
alphas_list_3 = [1, 3, 5, 7, 9]
alphas_list_4 = [10, 13, 15, 17, 19, 21, 23, 25, 27, 29]
alphas_list_5 = [31, 33, 35, 37, 39, 41, 43, 45, 47, 49]
alphas_list_6 = [51, 53, 55, 57, 59, 61, 63, 65, 67, 70]
alphas_list_7 = [75, 80, 85, 90, 95, 100, 200, 400, 800, 1600]
kf_stack_gen = KFold(n_splits=STACK_MODEL_KF, random_state=37, shuffle=True)
 stack_gen = StackingCVRegressor(
 regressors =(gbr, rf, abr_tree, lightgbm, lasso, lightgbm_goss, rf_2, gbr_2, ridge, lightgbm_2, abr_lasso),
meta_regressor = make_pipeline(RobustScaler() , RidgeCV(scoring='r2', alphas=alphas_list_4)) ,
cv = kf_stack_gen,
n_jobs = 8,
use_features_in_secondary=False)
 def grid_search_stack_gen(meta_model_list, use_2nd_fea_list=[False], model=stack_gen):
params = [{'meta_regressor': meta_model_list, 'use_features_in_secondary': use_2nd_fea_list}]
grid_search = GridSearchCV(model, params, cv=3, scoring='r2', return_train_score=True)
 grid_search.fit(X, y)
 print("grid_search.best_params_: ", grid_search.best_params_)
 print("grid_search.best_estimator_: ", grid_search.best_estimator_)
 print("means test scores:")
 cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(mean_score, params)
 print("means training scores:")
 for mean_score, params in zip(cvres["mean_train_score"], cvres["params"]):
print(mean_score, params)
 print("details: -------------------------------------")
 print(cvres)
 return grid_search.best_estimator_ 
<save_to_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
print('fit stack_gen with full training data')
 stack_gen_model = stack_gen.fit(X,y)
 submission = pd.read_csv("/kaggle/input/sample_submission.csv")
 print("submission.shape:", submission.shape, "; submission.columns: ", submission.columns)
 print("X_test.shape:", X_test.shape)
 submission.iloc[:,1] = stack_gen_model.predict(X_test)
 submission.to_csv("predection_output_stack_gen.csv", index=False )<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
%%time
base_models_fitted = {}
def pred_testset_with_base_model() :
base_model_list = [('lasso', lasso),('ridge', ridge),('rf', rf),('gbr',gbr), 
('lgm', lightgbm),('abr_tree', abr_tree),('abr_lasso', abr_lasso), 
('lightgbm_goss', lightgbm_goss),('rf_2', rf_2),('gbr_2', gbr_2), 
('lightgbm_2', lightgbm_2)]
for name, model in base_model_list:
file_name = "prediction_" + name + ".csv"
trained_model = model.fit(X,y)
 submission.iloc[:,1] = trained_model.predict(X_test)
 submission.to_csv(file_name, index=False)
 base_models_fitted[name] = trained_model
print(file_name)
 pred_testset_with_base_model()<load_from_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", df_train.shape)
 print("Test shape : ", df_test.shape)
 df_train.head()<count_unique_values>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
df_train.nunique()<concatenate>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
alldata = pd.concat([df_train, df_test], sort=False )<import_modules>
matrix['month'] = matrix['date_block_num'] % 12
from sklearn.preprocessing import LabelEncoder<categorify>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
le = LabelEncoder()<feature_engineering>
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
cata_cols = ['X0', 'X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'X8']
alldata[cata_cols] = alldata[cata_cols].apply(le.fit_transform )<split>
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
df_train = alldata[:len(df_train)]
df_test = alldata[len(df_train):]<drop_column>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
y = df_train['y']
df_train.drop(['y'], axis=1, inplace=True)
 df_test.drop(['y'], axis=1, inplace=True )<load_from_csv>
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ", df_train.shape)
 print("Test shape : ", df_test.shape)
 alldata = pd.concat([df_train, df_test], sort=False)
 le = LabelEncoder()
cata_cols = ['X0', 'X1', 'X2', 'X3', 'X4', 'X5', 'X6', 'X8']
alldata[cata_cols] = alldata[cata_cols].apply(le.fit_transform)
 df_train = alldata.iloc[:len(df_train), :]
df_test = alldata.iloc[len(df_train):, :]
y = df_train['y']
df_train.drop(['y'], axis=1, inplace=True)
 df_test.drop(['y'], axis=1, inplace=True )<import_modules>
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts
from catboost import CatBoostRegressor<choose_model_class>
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;
model = CatBoostRegressor(random_seed=42, depth = 4 )<train_model>
data = pd.read_pickle('data.pkl' )
model.fit(df_train, y, cat_features=[1,2,3,4,5,6,7,8] )<predict_on_test>
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]
preds = model.predict(df_test )<save_to_csv>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
submmision = pd.read_csv(".. /input/sample_submission.csv")
 submmision["y"] = preds
submmision.to_csv("benz_catboost.csv", index=False )<define_variables>
del data
gc.collect() ;
N_FOLDS = 5
MAX_EVALS = 5<load_from_csv>
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts
<create_dataframe><EOS> 
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))