Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<train_model>
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 100)
 %matplotlib inline
sys.version_info
ada_tree_real.fit(X_train, y_train)
 ada_tree_discrete.fit(X_train, y_train )<compute_test_metric>
items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shops = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 cats = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID' )
ada_report(ada_models=(ada_tree_real, ada_tree_discrete), X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test )<categorify>
print('train size, item in train, shop in train', train.shape[0], train.item_id.nunique() , train.shop_id.nunique())
 print('train size, item in train, shop in train', test.shape[0], test.item_id.nunique() ,test.shop_id.nunique())
 print('new items:', len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test))
cachedir = mkdtemp()
ada_pipe = Pipeline([('reduce_dim', PCA(random_state=166446054)) , 
('classify', AdaBoostClassifier(random_state=16446054)) ], memory=cachedir )<choose_model_class>
train.isnull().sum()
ada_param_grid = [
{
'reduce_dim__n_components': [10, 70, 140, 259],
'classify__algorithm': ['SAMME.R', 'SAMME'],
'classify__learning_rate': [1],
'classify__n_estimators': [300],
'classify__base_estimator': [DecisionTreeClassifier(max_depth=2)]
}
]<train_on_grid>
train.isnull().sum()
ada_grid = GridSearchCV(ada_pipe, cv=5,
n_jobs=4, param_grid=ada_param_grid,
scoring='f1_macro', refit=True,)
 _ = ada_grid.fit(X.iloc[:, 3:], y )<create_dataframe>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
ada_param_results = pd.DataFrame(ada_grid.cv_results_ )<import_modules>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold<import_modules>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )
from sklearn.model_selection import cross_val_score<import_modules>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
from sklearn import preprocessing<import_modules>
train['revenue'] = train['item_price'] * train['item_cnt_day']
from sklearn.model_selection import RandomizedSearchCV<choose_model_class>
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float32))
lgb_clf = lgb.LGBMClassifier(learning_rate=0.01, n_estimators=15000,
objective='multiclass', matric='logloss', boosting_type='dart',
num_leaves=700, max_depth=10,
class_weight='balanced', silent=True, n_jobs=-1,
colsample_bytree = 0.93, min_child_samples = 95, subsample = 0.96 )<find_best_model_class>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
kfold = 10
kf = StratifiedKFold(n_splits=kfold, shuffle=True)
 predicts_result = []
test_result = []
for train_index, test_index in kf.split(X.iloc[:, 3:], X.Target):
print("
X_train, X_val = X.iloc[:, 3:].iloc[train_index], X.iloc[:, 3:].iloc[test_index]
y_train, y_val = X.Target.iloc[train_index], X.Target.iloc[test_index]
lgb_clf.fit(X_train, y_train, eval_set=[(X_val, y_val)], early_stopping_rounds=1000, verbose=10000)
 test_result.append(f1_score(y_pred=lgb_clf.predict(X_val), y_true=y_val, average="macro"))
predicts_result.append(lgb_clf.predict(X_predict.iloc[:, 3:]))<compute_test_metric>
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
for result in test_result:
print(result)
 print("Mean test f1: ", np.mean(test_result))<predict_on_test>
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
lgb_train_predict = lgb_clf.predict(X.iloc[:, 3:])
 lgb_test_f1 = f1_score(y_pred=lgb_train_predict, y_true=X.Target, average="macro")
 print("LGBM train f1: ", lgb_test_f1 )<load_from_csv>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
submission_history = pd.read_excel(".. /input/competitionjournal/costa_rican_competition_result.xlsx")
 submission_history.set_index("sequence", inplace=True)
 plot_submission(submission_history )<set_options>
matrix = lag_feature(matrix, [1,2,3,6,12], 'item_cnt_month' )
%matplotlib inline
warnings.filterwarnings("ignore" )<categorify>
def add_group_stats(matrix_, groupby_feats, target, enc_feat, last_periods):
if not 'date_block_num' in groupby_feats:
print('date_block_num must in groupby_feats')
 return matrix_
group = matrix_.groupby(groupby_feats)[target].sum().reset_index()
max_lags = np.max(last_periods)
 for i in range(1,max_lags+1):
shifted = group[groupby_feats+[target]].copy(deep=True)
 shifted['date_block_num'] += i
shifted.rename({target:target+'_lag_'+str(i)},axis=1,inplace=True)
 group = group.merge(shifted, on=groupby_feats, how='left')
 group.fillna(0,inplace=True)
 for period in last_periods:
lag_feats = [target+'_lag_'+str(lag)for lag in np.arange(1,period+1)]
mean = group[lag_feats].sum(axis=1)/float(period)
 mean2 =(group[lag_feats]**2 ).sum(axis=1)/float(period)
 group[enc_feat+'_avg_sale_last_'+str(period)] = mean
group[enc_feat+'_std_sale_last_'+str(period)] =(mean2 - mean**2 ).apply(np.sqrt)
 group[enc_feat+'_std_sale_last_'+str(period)].replace(np.inf,0,inplace=True)
 group[enc_feat+'_avg_sale_last_'+str(period)] /= group[enc_feat+'_avg_sale_last_'+str(period)].mean()
group[enc_feat+'_std_sale_last_'+str(period)] /= group[enc_feat+'_std_sale_last_'+str(period)].mean()
cols = groupby_feats + [f_ for f_ in group.columns.values if f_.find('_sale_last_')>=0]
matrix = matrix_.merge(group[cols], on=groupby_feats, how='left')
 return matrix
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train, display_results=True):
ranked_list = []
zero_features = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
if display_results:
print("Feature ranking:")
 for f in range(X_train.shape[1]):
if display_results:
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 if importances[indices[f]] == 0.0:
zero_features.append(X_train.columns[indices[f]])
 return ranked_list, zero_features<feature_engineering>
ts = time.time()
matrix = add_group_stats(matrix, ['date_block_num', 'item_id'], 'item_cnt_month', 'item', [6,12])
 matrix = add_group_stats(matrix, ['date_block_num', 'shop_id'], 'item_cnt_month', 'shop', [6,12])
 matrix = add_group_stats(matrix, ['date_block_num', 'item_category_id'], 'item_cnt_month', 'category', [12])
 matrix = add_group_stats(matrix, ['date_block_num', 'city_code'], 'item_cnt_month', 'city', [12])
 matrix = add_group_stats(matrix, ['date_block_num', 'type_code'], 'item_cnt_month', 'type', [12])
 matrix = add_group_stats(matrix, ['date_block_num', 'subtype_code'], 'item_cnt_month', 'subtype', [12])
 time.time() - ts
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df<categorify>
def target_encoding(matrix_, groupby_feats, target, enc_feat, lags):
print('target encoding for',groupby_feats)
 group = matrix_.groupby(groupby_feats ).agg({target:'mean'})
 group.columns = [enc_feat]
group.reset_index(inplace=True)
 matrix = matrix_.merge(group, on=groupby_feats, how='left')
 matrix[enc_feat] = matrix[enc_feat].astype(np.float16)
 matrix = lag_feature(matrix, lags, enc_feat)
 matrix.drop(enc_feat, axis=1, inplace=True)
 return matrix
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<load_from_csv>
ts = time.time()
matrix = target_encoding(matrix, ['date_block_num'], 'item_cnt_month', 'date_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'item_id'], 'item_cnt_month', 'date_item_avg_item_cnt', [1,2,3,6,12])
 matrix = target_encoding(matrix, ['date_block_num', 'shop_id'], 'item_cnt_month', 'date_shop_avg_item_cnt', [1,2,3,6,12])
 matrix = target_encoding(matrix, ['date_block_num', 'item_category_id'], 'item_cnt_month', 'date_cat_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'shop_id', 'item_category_id'], 'item_cnt_month', 'date_shop_cat_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'city_code'], 'item_cnt_month', 'date_city_avg_item_cnt', [1])
 matrix = target_encoding(matrix, ['date_block_num', 'item_id', 'city_code'], 'item_cnt_month', 'date_item_city_avg_item_cnt', [1])
 time.time() - ts
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id<categorify>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<feature_engineering>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefe'] == "no", "edjefe"] = 0
train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "edjefa"] = train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "escolari"]
train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "edjefe"] = train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "edjefa"] = test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "edjefe"] = test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "escolari"]
train.loc[train['edjefa'] == "yes", "edjefa"] = 4
train.loc[train['edjefe'] == "yes", "edjefe"] = 4
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
train['edjefe'] = train['edjefe'].astype("int")
 train['edjefa'] = train['edjefa'].astype("int")
 test['edjefe'] = test['edjefe'].astype("int")
 test['edjefa'] = test['edjefa'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0<categorify>
matrix['month'] = matrix['date_block_num'] % 12
matrix['year'] =(matrix['date_block_num'] / 12 ).astype(np.int8 )
train['roof_waste_material'] = np.nan
test['roof_waste_material'] = np.nan
train['electricity_other'] = np.nan
test['electricity_other'] = np.nan
def fill_roof_exception(x):
if(x['techozinc'] == 0)and(x['techoentrepiso'] == 0)and(x['techocane'] == 0)and(x['techootro'] == 0):
return 1
else:
return 0
def fill_no_electricity(x):
if(x['public'] == 0)and(x['planpri'] == 0)and(x['noelec'] == 0)and(x['coopele'] == 0):
return 1
else:
return 0
train['roof_waste_material'] = train.apply(lambda x : fill_roof_exception(x),axis=1)
 test['roof_waste_material'] = test.apply(lambda x : fill_roof_exception(x),axis=1)
 train['electricity_other'] = train.apply(lambda x : fill_no_electricity(x),axis=1)
 test['electricity_other'] = test.apply(lambda x : fill_no_electricity(x),axis=1 )<concatenate>
ts = time.time()
last_sale = pd.DataFrame()
for month in range(1,35):
last_month = matrix.loc[(matrix['date_block_num']<month)&(matrix['item_cnt_month']>0)].groupby(['item_id','shop_id'])['date_block_num'].max()
df = pd.DataFrame({'date_block_num':np.ones([last_month.shape[0],])*month,
'item_id': last_month.index.get_level_values(0 ).values,
'shop_id': last_month.index.get_level_values(1 ).values,
'item_shop_last_sale': last_month.values})
 last_sale = last_sale.append(df)
 last_sale['date_block_num'] = last_sale['date_block_num'].astype(np.int8)
 matrix = matrix.merge(last_sale, on=['date_block_num','item_id','shop_id'], how='left')
 time.time() - ts
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<categorify>
ts = time.time()
last_sale = pd.DataFrame()
for month in range(1,35):
last_month = matrix.loc[(matrix['date_block_num']<month)&(matrix['item_cnt_month']>0)].groupby('item_id')['date_block_num'].max()
df = pd.DataFrame({'date_block_num':np.ones([last_month.shape[0],])*month,
'item_id': last_month.index.values,
'item_last_sale': last_month.values})
 last_sale = last_sale.append(df)
 last_sale['date_block_num'] = last_sale['date_block_num'].astype(np.int8)
 matrix = matrix.merge(last_sale, on=['date_block_num','item_id'], how='left')
 time.time() - ts
train, test = train_test_apply_func(train, test, convert_OHE2LE )<categorify>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )<categorify>
matrix.to_pickle('data.pkl')
 del matrix
del group
del items
del shops
del cats
del train
gc.collect() ;
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['adult'] = df['hogar_adul'] - df['hogar_mayor']
df['dependency_count'] = df['hogar_nin'] + df['hogar_mayor']
df['dependency'] = df['dependency_count'] / df['adult']
df['dependency'] = df['dependency'].replace({np.inf: 0})
 df['child_percent'] = df['hogar_nin']/df['hogar_total']
df['elder_percent'] = df['hogar_mayor']/df['hogar_total']
df['adult_percent'] = df['hogar_adul']/df['hogar_total']
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['r4h1_percent_in_male'] = df['r4h1'] / df['r4h3']
df['r4m1_percent_in_female'] = df['r4m1'] / df['r4m3']
df['r4h1_percent_in_total'] = df['r4h1'] / df['hhsize']
df['r4m1_percent_in_total'] = df['r4m1'] / df['hhsize']
df['r4t1_percent_in_total'] = df['r4t1'] / df['hhsize']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['rent_per_adult'] = df['v2a1']/df['hogar_adul']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df['tablet_per_person_household'] = df['v18q1']/df['hhsize']
df['phone_per_person_household'] = df['qmobilephone']/df['hhsize']
df['rez_esc_escolari'] = df['rez_esc']/df['escolari']
df['rez_esc_r4t1'] = df['rez_esc']/df['r4t1']
df['rez_esc_r4t2'] = df['rez_esc']/df['r4t2']
df['rez_esc_r4t3'] = df['rez_esc']/df['r4t3']
df['rez_esc_age'] = df['rez_esc']/df['age']
df['escolari_age'] = df['escolari']/df['age']
df['escolari_age_mean'] = df.groupby("idhogar")["escolari_age"].transform("mean")
 df['age_sum'] = df.groupby("idhogar")["age"].transform("sum")
 df['age_std'] = df.groupby("idhogar")["age"].transform("std")
 df['escolari_age_std'] = df.groupby("idhogar")["escolari_age"].transform("std")
 df['escolari_age_max'] = df.groupby("idhogar")["escolari_age"].transform("max")
 df['escolari_age_min'] = df.groupby("idhogar")["escolari_age"].transform("min")
 df['escolari_std'] = df.groupby("idhogar")["escolari"].transform("min")
 df['escolari_age'] = df['escolari_age'].fillna(0)
 df['escolari_age_std'] = df['escolari_age_std'].fillna(-1)
 df['age_std'] = df['age_std'].fillna(-1)
 df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
extract_features(train)
 extract_features(test)
<drop_column>
data = pd.read_pickle('./data.pkl')
 data.head()
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<split>
data = data[[
'date_block_num',
'shop_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code','subtype_code',
'item_cnt_month_lag_1','item_cnt_month_lag_2','item_cnt_month_lag_3','item_cnt_month_lag_6','item_cnt_month_lag_12',
'item_avg_sale_last_6', 'item_std_sale_last_6',
'item_avg_sale_last_12', 'item_std_sale_last_12',
'shop_avg_sale_last_6', 'shop_std_sale_last_6',
'shop_avg_sale_last_12', 'shop_std_sale_last_12',
'category_avg_sale_last_12', 'category_std_sale_last_12',
'city_avg_sale_last_12', 'city_std_sale_last_12',
'type_avg_sale_last_12', 'type_std_sale_last_12',
'subtype_avg_sale_last_12', 'subtype_std_sale_last_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1','date_item_avg_item_cnt_lag_2','date_item_avg_item_cnt_lag_3','date_item_avg_item_cnt_lag_6','date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1','date_shop_avg_item_cnt_lag_2','date_shop_avg_item_cnt_lag_3','date_shop_avg_item_cnt_lag_6','date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month','year',
'item_shop_last_sale','item_last_sale',
'item_shop_first_sale','item_first_sale',
]]
cat_feats = ['shop_id','city_code','item_category_id','type_code','subtype_code']
def split_data(train, y, sample_weight=None, households=None, test_percentage=0.20, seed=None):
train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
if sample_weight is not None:
y_train_weights = sample_weight[~cv_idx]
return X_train, y_train, X_test, y_test, y_train_weights
return X_train, y_train, X_test, y_test<split>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del data
gc.collect() ;
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=None)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.15), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar<split>
sys.version_info
y_train_weights = class_weight.compute_sample_weight('balanced', y_train, indices=None )<define_variables>
ts = time.time()
model = LGBMRegressor(
 max_depth = 8,
n_estimators = 500,
colsample_bytree=0.7,
min_child_weight = 300,
reg_alpha = 0.1,
reg_lambda = 1,
random_state = 42, 
)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=10,
early_stopping_rounds = 40,
categorical_feature = cat_feats)
 time.time() - ts
extra_drop_features = ['agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil6_COUNT',
'agg18_estadocivil7_COUNT',
'agg18_parentesco10_COUNT',
'agg18_parentesco11_COUNT',
'agg18_parentesco12_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco2_COUNT',
'agg18_parentesco3_COUNT',
'agg18_parentesco4_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco6_COUNT',
'agg18_parentesco7_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco9_COUNT',
'electricity_other',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'geo_energcocinar_LE_1',
'geo_energcocinar_LE_2',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_hogar_mayor',
'geo_manual_elec_LE_2',
'geo_pared_LE_0',
'geo_pared_LE_3',
'geo_pared_LE_4',
'geo_pared_LE_5',
'geo_pared_LE_6',
'num_over_18',
'parentesco_LE',
'rez_esc_r4t2',
'rez_esc_r4t3']<define_variables>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 X_train_level2 = pd.DataFrame({
"ID": np.arange(Y_pred.shape[0]),
"item_cnt_month": Y_pred
})
 X_train_level2.to_csv('lgb_valid.csv', index=False)
 submission = pd.DataFrame({
"ID": np.arange(Y_test.shape[0]),
"item_cnt_month": Y_test
})
 submission.to_csv('lgb_submission.csv', index=False )
xgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']<init_hyperparams>
np.random.seed(233333 )
opt_parameters_1 = {'max_depth':35, 'learning_rate':0.125, 'silent':1, "n_estimators": 325, 'objective':'multi:softmax', 'min_child_weight': 2, 'num_class': 4, 'gamma': 2.0, 'colsample_bylevel': 0.9, 'subsample': 0.84, 'colsample_bytree': 0.88, 'reg_lambda': 0.4 }
opt_parameters_2 = {'max_depth':35, 'learning_rate':0.14, 'silent':1, "n_estimators": 275, 'objective':'multi:softmax', 'min_child_weight': 2, 'num_class': 4, 'gamma': 2.25, 'colsample_bylevel': 1, 'subsample': 0.95, 'colsample_bytree': 0.85, 'reg_lambda': 0.35 }
opt_parameters_3 = {'max_depth':35, 'learning_rate':0.15, 'silent':1, "n_estimators": 300, 'objective':'multi:softmax', 'min_child_weight': 2, 'num_class': 4, 'gamma': 2.75, 'colsample_bylevel': 0.95, 'subsample': 0.94, 'colsample_bytree': 0.9, 'reg_lambda': 0.375 }
opt_parameters_4 = {'max_depth':35, 'learning_rate':0.13, 'silent':1, "n_estimators": 350, 'objective':'multi:softmax', 'min_child_weight': 2, 'num_class': 4, 'gamma': 2.5, 'colsample_bylevel': 1, 'subsample': 0.95, 'colsample_bytree': 0.88, 'reg_lambda': 0.325 }
xgb_param_list = [opt_parameters_1, opt_parameters_2, opt_parameters_3, opt_parameters_4]<train_model>
data = pd.read_pickle('data.pkl')
 data = data[[
'date_block_num',
'item_cnt_month',
'item_cnt_month_lag_1','item_cnt_month_lag_2','item_cnt_month_lag_3','item_cnt_month_lag_6','item_cnt_month_lag_12',
'item_avg_sale_last_6', 'item_std_sale_last_6',
'item_avg_sale_last_12', 'item_std_sale_last_12',
'shop_avg_sale_last_6', 'shop_std_sale_last_6',
'shop_avg_sale_last_12', 'shop_std_sale_last_12',
'category_avg_sale_last_12', 'category_std_sale_last_12',
'city_avg_sale_last_12', 'city_std_sale_last_12',
'type_avg_sale_last_12', 'type_std_sale_last_12',
'subtype_avg_sale_last_12', 'subtype_std_sale_last_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1','date_item_avg_item_cnt_lag_2','date_item_avg_item_cnt_lag_3','date_item_avg_item_cnt_lag_6','date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1','date_shop_avg_item_cnt_lag_2','date_shop_avg_item_cnt_lag_3','date_shop_avg_item_cnt_lag_6','date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month','year',
'item_shop_last_sale','item_last_sale',
'item_shop_first_sale','item_first_sale',
]]
def evaluate_macroF1_xgb(predictions, truth):
pred_labels = predictions.argmax(axis=1)
 truth = truth.get_label()
f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', 1-f1)
 fit_params={"early_stopping_rounds":150,
"eval_metric" : evaluate_macroF1_xgb,
"eval_set" : [(X_test,y_test)],
'verbose': False,
}
fit_params['verbose'] = 50<split>
def Sales_prediction_model(input_shape):
in_layer = Input(input_shape)
 x = Dense(16,kernel_initializer='RandomUniform', kernel_regularizer=l2(0.02), activation = "relu" )(in_layer)
 x = Dense(8, kernel_initializer='RandomUniform', kernel_regularizer=l2(0.02), activation = "relu" )(x)
 x = Dense(1, kernel_initializer='RandomUniform', kernel_regularizer=l2(0.02), activation = "relu" )(x)
 model = Model(inputs = in_layer, outputs = x, name='Sales_prediction_model')
 return model
X_train.fillna(0,inplace=True)
 X_valid.fillna(0,inplace=True)
 X_test.fillna(0,inplace=True)
 input_shape = [X_train.shape[1]]
model = Sales_prediction_model(input_shape)
 model.compile(optimizer = Adam(lr=0.0005), loss = ["mse"], metrics=['mse'])
 model.fit(X_train, Y_train, validation_data =(X_valid, Y_valid), batch_size = 10000, epochs=5)
np.random.seed(100)
 def _parallel_fit_estimator(estimator1, X, y, drop_cols=[], use_cv=True, sample_weight=None, threshold=True, **fit_params):
estimator = clone(estimator1)
 if use_cv:
if sample_weight is not None:
X_train, y_train, X_cv, y_cv, y_train_weight = split_data(X, y, sample_weight, households=train_households)
 else:
X_train, y_train, X_cv, y_cv = split_data(X, y, None, households=train_households)
 else:
X_train = X
y_train = y
X_cv = X
y_cv = y
if isinstance(estimator1, xgb.XGBClassifier):
fit_params["eval_set"] = [(X_cv,y_cv)]
else:
fit_params["eval_set"] = [(X_train, y_train),(X_cv,y_cv)]
if sample_weight is not None:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
_ = estimator.fit(X_train, y_train, sample_weight=y_train_weight, **fit_params)
 else:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
_ = estimator.fit(X_train, y_train, **fit_params)
 if not isinstance(estimator1, ExtraTreesClassifier)and not isinstance(estimator1, RandomForestClassifier)and not isinstance(estimator1, xgb.XGBClassifier):
best_cv_round = np.argmax(estimator.evals_result_['valid']['macroF1'])
 best_cv = np.max(estimator.evals_result_['valid']['macroF1'])
 best_train = estimator.evals_result_['train']['macroF1'][best_cv_round]
else:
best_train = f1_score(y_train, estimator.predict(X_train), average="macro")
 best_cv = f1_score(y_cv, estimator.predict(X_cv), average="macro")
 print("Train F1:", best_train)
 print("Test F1:", best_cv)
 if threshold:
if(( best_cv > 0.37)and(best_train > 0.75)) or(( best_cv > 0.44)and(best_train > 0.65)) :
return estimator
else:
print("Unacceptable!!! Trying again...")
 return _parallel_fit_estimator(estimator1, X, y, sample_weight=sample_weight, **fit_params)
 else:
return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, drop_cols=[], split_data=True, sample_weight=None, threshold=True, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y, drop_cols=drop_cols, use_cv=split_data,
sample_weight=sample_weight, threshold=threshold, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
Y_pred = model.predict(X_valid ).clip(0, 20)[:,0]
Y_test = model.predict(X_test ).clip(0, 20)[:,0]
X_train_level2 = pd.DataFrame({
"ID": np.arange(Y_pred.shape[0]),
"item_cnt_month": Y_pred
})
 X_train_level2.to_csv('nn_valid.csv', index=False)
 submission = pd.DataFrame({
"ID": np.arange(Y_test.shape[0]),
"item_cnt_month": Y_test
})
 submission.to_csv('nn_submission.csv', index=False )
clfs = []
for i in range(5):
clf = xgb.XGBClassifier(random_state=217+i, n_jobs=4, **xgb_param_list[i % len(xgb_param_list)])
 clfs.append(( 'xgb{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del(clfs)
 _ = vc.fit(X_train.drop(xgb_drop_cols, axis=1), y_train, sample_weight=y_train_weights, drop_cols=[], threshold=False, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
sys.version_info
global_score = f1_score(y_test, clf_final.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 print('Validation score of a single XGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on XGBMs with soft voting: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on XGBMs with hard voting: {:.4f}'.format(global_score_hard))<compute_test_metric>
ts = time.time()
model = XGBRegressor(
 max_depth=7,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
gamma = 0.005,
eta=0.1,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=10,
early_stopping_rounds = 40, 
)
 time.time() - ts
_ = feature_importance(vc.estimators_[0], X_train.drop(xgb_drop_cols, axis=1), display_results=True )<define_variables>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 X_train_level2 = pd.DataFrame({
"ID": np.arange(Y_pred.shape[0]),
"item_cnt_month": Y_pred
})
 X_train_level2.to_csv('xgb_valid.csv', index=False)
 submission = pd.DataFrame({
"ID": np.arange(Y_test.shape[0]),
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )
extra_drop_features = ['agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil6_COUNT',
'agg18_estadocivil7_COUNT',
'agg18_parentesco10_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco11_COUNT',
'agg18_parentesco11_MEAN',
'agg18_parentesco12_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco1_COUNT',
'agg18_parentesco2_COUNT',
'agg18_parentesco3_COUNT',
'agg18_parentesco4_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco5_COUNT',
'agg18_parentesco6_COUNT',
'agg18_parentesco6_MEAN',
'agg18_parentesco7_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco8_MEAN',
'agg18_parentesco9_COUNT',]
lgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']<init_hyperparams>
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import Ridge, LinearRegression
import gc
lgb_parameters_1 = {'max_depth': -1, 'learning_rate': 0.125, 'colsample_bytree': 0.88, 'min_child_samples': 90, 'num_leaves': 16, 'subsample': 0.94, 'reg_lambda': 0.425, "n_estimators": 5000}
lgb_parameters_2 = {'max_depth': 50, 'learning_rate': 0.11, 'colsample_bytree': 0.95, 'min_child_samples': 90, 'num_leaves': 25, 'subsample': 0.94, 'reg_lambda': 0.34, "n_estimators": 3000}
lgb_parameters_3 = {'max_depth': 35, 'learning_rate': 0.12, 'colsample_bytree': 0.88, 'min_child_samples': 56, 'num_leaves': 20, 'subsample': 0.95, 'reg_lambda': 0.375, "n_estimators": 4000}
lgb_parameters_4 = {'max_depth': 20, 'learning_rate': 0.13, 'colsample_bytree': 0.89, 'min_child_samples': 90, 'num_leaves': 14, 'subsample': 0.96, 'reg_lambda': 0.39, "n_estimators": 2500}
lgb_parameters_5 = {'max_depth': 25, 'learning_rate': 0.14, 'colsample_bytree': 0.93, 'min_child_samples': 70, 'num_leaves': 17, 'subsample': 0.95, 'reg_lambda': 0.35, "n_estimators": 4500 }
lgb_param_list = [lgb_parameters_1, lgb_parameters_2, lgb_parameters_3, lgb_parameters_4, lgb_parameters_5]<compute_train_metric>
data = pd.read_pickle('data.pkl')
 Y_train_level2 = data[data.date_block_num == 33]['item_cnt_month']
del data
gc.collect()
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 90, 'num_leaves': 16, 'subsample': 0.94, 'reg_lambda': 0.5, }
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 90, 'num_leaves': 25, 'subsample': 0.94, 'reg_lambda': 0.5, }
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":150,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<choose_model_class>
X_train_level2 = pd.DataFrame()
df = pd.read_csv('./lgb_valid.csv')
 X_train_level2['lgb'] = df['item_cnt_month']
df = pd.read_csv('./xgb_valid.csv')
 X_train_level2['xgb'] = df['item_cnt_month']
df = pd.read_csv('./nn_valid.csv')
 X_train_level2['nn'] = df['item_cnt_month']
X_test_level2 = pd.DataFrame()
df = pd.read_csv('./lgb_submission.csv')
 X_test_level2['lgb'] = df['item_cnt_month']
df = pd.read_csv('./xgb_submission.csv')
 X_test_level2['xgb'] = df['item_cnt_month']
df = pd.read_csv('./nn_submission.csv')
 X_test_level2['nn'] = df['item_cnt_month']
<compute_train_metric><EOS> 
best_alpha = 1;
best_rmse = 100;
for alpha in np.arange(0,1,0.02):
Y_pred_level2 = alpha*X_train_level2['lgb'] +(1-alpha)*X_train_level2['xgb']
rmse = np.sqrt(mean_squared_error(Y_train_level2, Y_pred_level2))
if(rmse<best_rmse):
best_rmse = rmse
best_alpha = alpha
Y_test_level2 = best_alpha*X_test_level2['lgb'] +(1-best_alpha)*X_test_level2['xgb']
print('best alpha:', best_alpha)
 print('weighted average of lgb and xgb validation rmse: ',best_rmse)
 submission = pd.DataFrame({
"ID": np.arange(Y_test_level2.shape[0]),
"item_cnt_month": Y_test_level2
})
 submission.to_csv('./blended_submission1.csv', index=False)
 model = LinearRegression()
model.fit(X_train_level2, Y_train_level2)
 Y_pred_level2 = model.predict(X_train_level2)
 Y_test_level2 = model.predict(X_test_level2)
 rmse = np.sqrt(mean_squared_error(Y_train_level2, Y_pred_level2))
print('Linear regression validation rmse: ',rmse)
 submission = pd.DataFrame({
"ID": np.arange(Y_test_level2.shape[0]),
"item_cnt_month": Y_test_level2
})
 submission.to_csv('./blended_submission2.csv', index=False )