Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<split>
RANDOM_SEED = 42
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=1977)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.15), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar<define_variables>
PATH = '.. /input/competitive-data-science-predict-future-sales'
items = pd.read_csv(PATH + '/items.csv')
 shops = pd.read_csv(PATH + '/shops.csv')
 cats = pd.read_csv(PATH + '/item_categories.csv')
 train = pd.read_csv(PATH + '/sales_train.csv')
 test = pd.read_csv(PATH + '/test.csv' ).set_index('ID' )
extra_drop_features = [
'geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'rez_esc',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']<drop_column>
summary_stats_table(train )
lgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )<init_hyperparams>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1000]
train = train[train.item_price > 0].reset_index(drop=True)
 train.loc[train.item_cnt_day < 0, 'item_cnt_day'] = 0
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 11, 'shop_id'] = 10
test.loc[test.shop_id == 11, 'shop_id'] = 10
train.loc[train.shop_id == 40, 'shop_id'] = 39
test.loc[test.shop_id == 40, 'shop_id'] = 39
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 95, 'num_leaves': 25, 'subsample': 0.94, 'reg_lambda': 0.7, }
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":700,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops['category'] = shops['shop_name'].str.split(' ' ).map(lambda x:x[1] ).astype(str)
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
category = ['ТЦ', 'ТРК', 'ТРЦ', 'ТК']
shops.category = shops.category.apply(lambda x: x if(x in category)else 'etc')
 shops.groupby(['category'] ).sum()
shops['shop_city'] = shops.city
shops['shop_category'] = shops.category
shops['shop_city'] = LabelEncoder().fit_transform(shops['shop_city'])
 shops['shop_category'] = LabelEncoder().fit_transform(shops['shop_category'])
 shops = shops[['shop_id','shop_city', 'shop_category']]
shops.head()
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
X_train, y_train, X_test, y_test = split_data(X, y, households=train_households, seed=1972)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X_train, y_train, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
cats['type_code'] = cats.item_category_name.apply(lambda x: x.split(' ')[0] ).astype(str)
 cats.loc[(cats.type_code == 'Игровые')|(cats.type_code == 'Аксессуары'), 'type_code'] = 'Игры'
cats.loc[cats.type_code == 'PC', 'type_code'] = 'Музыка'
category = ['Игры', 'Карты', 'Кино', 'Книги','Музыка', 'Подарки', 'Программы', 'Служебные', 'Чистые']
cats['type_code'] = cats.type_code.apply(lambda x: x if(x in category)else 'etc')
 cats['type_code'] = LabelEncoder().fit_transform(cats['type_code'])
 cats['split'] = cats.item_category_name.apply(lambda x: x.split('-'))
cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
clfs = []
for i in range(35):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
items['name_1'], items['name_2'] = items['item_name'].str.split('[', 1 ).str
items['name_1'], items['name_3'] = items['item_name'].str.split('(', 1 ).str
items['name_2'] = items['name_2'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items['name_3'] = items['name_3'].str.replace('[^A-Za-z0-9А-Яа-я]+', ' ' ).str.lower()
items = items.fillna('0')
 result_1 = Counter(' '.join(items['name_2'].values.tolist() ).split(' ')).items()
result_1 = sorted(result_1, key=itemgetter(1))
result_1 = pd.DataFrame(result_1, columns=['feature', 'count'])
 result_1 = result_1[(result_1['feature'].str.len() > 1)&(result_1['count'] > 200)]
result_2 = Counter(' '.join(items['name_3'].values.tolist() ).split(" ")).items()
result_2 = sorted(result_2, key=itemgetter(1))
result_2 = pd.DataFrame(result_2, columns=['feature', 'count'])
 result_2 = result_2[(result_2['feature'].str.len() > 1)&(result_2['count'] > 200)]
result = pd.concat([result_1, result_2])
 result = result.drop_duplicates(subset=['feature'] ).reset_index(drop=True)
 print('Most common aditional features:', result)
 items['type'] = items.name_2.apply(lambda x: x[0:8] if x.split(' ')[0] == 'xbox' else x.split(' ')[0])
 items.loc[(items.type == 'x360')|(items.type == 'xbox360'), 'type'] = 'xbox 360'
items.loc[items.type == '', 'type'] = 'mac'
items.type = items.type.apply(lambda x: x.replace(' ',''))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == 'рс'), 'type'] = 'pc'
items.loc[(items.type == 'рs3'), 'type'] = 'ps3'
group_sum = items.groupby('type' ).sum()
drop_list = group_sum.loc[group_sum.item_category_id < 200].index
print('drop list:', drop_list)
 items.name_2 = items.type.apply(lambda x: 'etc' if x in drop_list else x)
 items = items.drop(['type'], axis=1)
 print(items.groupby('name_2' ).count() [['item_id']])
 items['name_2'] = LabelEncoder().fit_transform(items['name_2'] ).astype(np.int8)
 items['name_3'] = LabelEncoder().fit_transform(items['name_3'] ).astype(np.int16)
 items.drop(['item_name', 'name_1'], axis=1, inplace=True )
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_train_metric>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 print('Use time:', time.time() - ts)
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
train['revenue'] = train['item_price'] * train['item_cnt_day']
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
print('Use time:', time.time() - ts)
ranked_features = feature_importance(clf_final, X_train )<create_dataframe>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16)
 ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 print('Use time:', time.time() - ts)
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids<predict_on_test>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['shop_city'] = matrix['shop_city'].astype(np.int8)
 matrix['shop_category'] = matrix['shop_category'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 print('Use time:', time.time() - ts)
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1<save_to_csv>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'-lag'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False )<load_from_csv>
dict_simple = {'date_block_num': 'date', 'item_id': 'item', 'shop_id': 'shop',
'item_category_id': 'itemcate', 'item_price':'price',
'item_cnt_month': 'cnt', }
def sum_names(name_list):
names = ''
for x in name_list:
names += x+'+'
return names
def group_agg(matrix, groupby_feats, transform_feat, aggtype='mean'):
group = matrix.groupby(groupby_feats ).agg({transform_feat: [aggtype]})
 groupby_feats_simple = [dict_simple[x] if x in dict_simple.keys() else x
for x in groupby_feats]
transform_feat_simple = dict_simple[transform_feat] \
if transform_feat in dict_simple.keys() else transform_feat
group_name = f'{sum_names(groupby_feats_simple)[:-1]}-{aggtype.upper() }-{transform_feat_simple}'
group.columns = [ group_name ]
group.reset_index(inplace=True)
 return group, group_name
def add_groupmean_lag(matrix, groupby_feats, transform_feat, lags):
group, group_name = group_agg(matrix, groupby_feats, transform_feat)
 matrix = pd.merge(matrix, group, on=groupby_feats, how='left')
 matrix[group_name] = matrix[group_name].astype(np.float16)
 if lags != []:
matrix = lag_feature(matrix, lags, group_name)
 matrix.drop([group_name], axis=1, inplace=True)
 return matrix
df = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<define_variables>
ts = time.time()
transform_feat = 'item_cnt_month'
groupby_feats = ['date_block_num']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'item_id']
lags = [1,2,3]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id']
lags = [1,2,3]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'item_category_id']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id', 'item_category_id']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id', 'item_id']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_id', 'subtype_code']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'shop_city']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 groupby_feats = ['date_block_num', 'item_id', 'shop_city']
lags = [1]
matrix = add_groupmean_lag(matrix, groupby_feats, transform_feat, lags)
 print('Use time:', time.time() - ts)
categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
categorical_columns<data_type_conversions>
ts = time.time()
fetures_to_drop = []
transform_feat = 'item_price'
groupby_feats = ['item_id']
group, mean_price_col = group_agg(train, groupby_feats,
transform_feat, aggtype='mean')
 matrix = pd.merge(matrix, group, on=groupby_feats, how='left')
 matrix[mean_price_col] = matrix[mean_price_col].astype(np.float16)
 transform_feat = 'item_price'
groupby_feats = ['date_block_num','item_id']
group, mean_monthlyprice_col = group_agg(train, groupby_feats,
transform_feat, aggtype='mean')
 matrix = pd.merge(matrix, group, on=groupby_feats, how='left')
 matrix[mean_monthlyprice_col] = matrix[mean_monthlyprice_col].astype(np.float16)
 lags = [1,2,3]
matrix = lag_feature(matrix, lags, mean_monthlyprice_col)
 for i in lags:
matrix['delta_price-lag'+str(i)] = \ 
(matrix[f'{mean_monthlyprice_col}-lag'+str(i)] - matrix[mean_price_col])\
/ matrix[mean_price_col]
matrix['delta_price-lag']=0
bool_loc = np.ones(len(matrix)) ==1
for i in lags:
matrix.loc[bool_loc, 'delta_price-lag'] = matrix.loc[bool_loc,'delta_price-lag'+str(i)]
bool_loc &= matrix['delta_price-lag'+str(i)]==0
matrix['delta_price-lag'] = matrix['delta_price-lag'].astype(np.float16)
 matrix['delta_price-lag'].fillna(0, inplace=True)
 fetures_to_drop.append(mean_price_col)
 fetures_to_drop.append(mean_monthlyprice_col)
 for i in lags:
fetures_to_drop += [f'{mean_monthlyprice_col}-lag'+str(i)]
fetures_to_drop += ['delta_price-lag'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 print('Use time:', time.time() - ts)
def replace_yes_no(df):
df[['dependency', 'edjefe', 'edjefa']] = df[['dependency','edjefe', 'edjefa']].replace({'yes':1, 'no':1} ).astype(float)
 return df
df = replace_yes_no(df)
 df_test = replace_yes_no(df_test )<categorify>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 print('Use time:', time.time() - ts)
def tablets(df):
df['v18q1'][df['v18q1'].isnull() ] = 0
df = df.drop(columns = 'v18q')
 return df
df = tablets(df)
 df_test = tablets(df_test )<merge>
total_block_num = 35
date_block_num = np.arange(total_block_num)
 date_block = [pd.Timestamp(2013, 1, 1)+pd.DateOffset(months=x)for x in date_block_num]
df_date = pd.DataFrame(date_block_num, columns=['date_block_num'])
 df_date['date_block'] = date_block
df_date['year'] = df_date['date_block'].dt.year
df_date['month'] = df_date['date_block'].dt.month
for i in range(len(df_date)) :
day_to_count = 0
calendar_matrix = calendar.monthcalendar(df_date['year'].iloc[i],df_date['month'].iloc[i])
 for j in range(7):
num_days = sum(1 for x in calendar_matrix if x[j] != 0)
 df_date.loc[i, f'week{j}'] = num_days
df_date = df_date[['date_block_num', 'year','month','week0','week1',
'week2','week3','week4','week5','week6']]
df_date['days'] = df_date[['week0','week1','week2','week3','week4','week5','week6']].sum(axis=1)
 df_date['year'] = df_date['year']-2012
df_date = df_date.astype(np.int8)
 matrix = pd.merge(matrix, df_date, on=['date_block_num'], how='left')
def escolari(df):
escolari_mean = df.groupby(['idhogar'], as_index = False)['escolari'].mean().rename(columns = {'mean': 'escolari_mean'})
 escolari_mean.columns = ['idhogar', 'escolari_mean']
escolari_max = df.groupby(['idhogar'], as_index = False)['escolari'].max().rename(columns = {'max': 'escolari_max'})
 escolari_max.columns = ['idhogar', 'escolari_max']
df = df.merge(escolari_mean, how = 'left', on = 'idhogar')
 df = df.merge(escolari_max, how = 'left', on = 'idhogar')
 return df
df = escolari(df)
 df_test = escolari(df_test )<categorify>
matrix['item_shop_first_sale'] = \
matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = \
matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
def water_provision(df):
df['water_prov'] = 0
df.loc[df['abastaguadentro'] == 1, 'water_prov'] = 2
df.loc[df['abastaguafuera'] == 1, 'water_prov'] = 1
df.loc[df['abastaguano'] == 1, 'water_prov'] = 0
df = df.drop(columns = ['abastaguadentro','abastaguafuera', 'abastaguano'])
 return df
df = water_provision(df)
 df_test = water_provision(df_test )<feature_engineering>
matrix = matrix[matrix.date_block_num > 3]
def walls_roof_floor(df):
df['walls'] = 0
df.loc[df['epared1'] == 1, 'walls'] = 1
df.loc[df['epared2'] == 1, 'walls'] = 2
df.loc[df['epared3'] == 1, 'walls'] = 3
df['roof'] = 0
df.loc[df['etecho1'] == 1, 'roof'] = 1
df.loc[df['etecho2'] == 1, 'roof'] = 2
df.loc[df['etecho3'] == 1, 'roof'] = 3
df['floor'] = 0
df.loc[df['eviv1'] == 1, 'floor'] = 1
df.loc[df['eviv2'] == 1, 'floor'] = 2
df.loc[df['eviv3'] == 1, 'floor'] = 3
df = df.drop(columns = ['epared1','epared2', 'epared3', 'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3'])
 return df
df = walls_roof_floor(df)
 df_test = walls_roof_floor(df_test )<feature_engineering>
def fill_na(df):
for col in df.columns:
if('-lag' in col)&(df[col].isnull().any()):
print(col)
 if('cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix )
def education_level(df):
df['education'] = 0
df.loc[df['instlevel1'] == 1, 'education'] = 1
df.loc[df['instlevel2'] == 1, 'education'] = 2
df.loc[df['instlevel3'] == 1, 'education'] = 3
df.loc[df['instlevel4'] == 1, 'education'] = 4
df.loc[df['instlevel5'] == 1, 'education'] = 5
df.loc[df['instlevel6'] == 1, 'education'] = 6
df.loc[df['instlevel7'] == 1, 'education'] = 7
df.loc[df['instlevel8'] == 1, 'education'] = 8
df.loc[df['instlevel9'] == 1, 'education'] = 9
df = df.drop(columns = ['instlevel1','instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9'])
 return df
df = education_level(df)
 df_test = education_level(df_test )<feature_engineering>
matrix.isna().sum()
def tipovivi(df):
df['tipovivi'] = 0
df.loc[df['tipovivi1'] == 1, 'tipovivi'] = 1
df.loc[df['tipovivi2'] == 1, 'tipovivi'] = 2
df.loc[df['tipovivi3'] == 1, 'tipovivi'] = 3
df.loc[df['tipovivi4'] == 1, 'tipovivi'] = 4
df.loc[df['tipovivi5'] == 1, 'tipovivi'] = 5
df = df.drop(columns = ['tipovivi1','tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5'])
 return df
df = tipovivi(df)
 df_test = tipovivi(df_test )<categorify>
del group
del items
del shops
del cats
del train
gc.collect() ;
matrix.to_pickle('.. /working/data.pkl')
 del matrix
gc.collect() ;
def rubbish(df):
df['rubbish'] = 0
df.loc[df['elimbasu1'] == 1, 'rubbish'] = 1
df.loc[df['elimbasu2'] == 1, 'rubbish'] = 2
df.loc[df['elimbasu3'] == 1, 'rubbish'] = 3
df.loc[df['elimbasu4'] == 1, 'rubbish'] = 4
df.loc[df['elimbasu5'] == 1, 'rubbish'] = 5
df.loc[df['elimbasu6'] == 1, 'rubbish'] = 0
df = df.drop(columns = ['elimbasu1','elimbasu2', 'elimbasu3', 'elimbasu4', 'elimbasu5', 'elimbasu6'])
 return df
df = rubbish(df)
 df_test = rubbish(df_test )<categorify>
data = pd.read_pickle('.. /working/data.pkl')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 print(len(data.columns))
data.columns
def energy(df):
df['energy'] = 0
df.loc[df['energcocinar1'] == 1, 'energy'] = 1
df.loc[df['energcocinar2'] == 1, 'energy'] = 2
df.loc[df['energcocinar3'] == 1, 'energy'] = 3
df.loc[df['energcocinar4'] == 1, 'energy'] = 4
df = df.drop(columns = ['energcocinar1','energcocinar2', 'energcocinar3', 'energcocinar4'])
 return df
df = energy(df)
 df_test = energy(df_test )<feature_engineering>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del data
gc.collect() ;
def toilet(df):
df['toilet'] = 0
df.loc[df['sanitario1'] == 1, 'toilet'] = 1
df.loc[df['sanitario5'] == 1, 'toilet'] = 2
df.loc[df['sanitario6'] == 1, 'toilet'] = 3
df.loc[df['sanitario3'] == 1, 'toilet'] = 4
df.loc[df['sanitario2'] == 1, 'toilet'] = 5
df = df.drop(columns = ['sanitario1','sanitario2', 'sanitario3', 'sanitario5', 'sanitario6'])
 return df
df = toilet(df)
 df_test = toilet(df_test )<feature_engineering>
lgb_train = lgb.Dataset(X_train, Y_train)
 lgb_eval = lgb.Dataset(X_valid, Y_valid, reference=lgb_train)
 del X_train
gc.collect() ;
def new_variables(df):
df['rent_by_hhsize'] = df['v2a1'] / df['hhsize']
df['rent_by_people'] = df['v2a1'] / df['r4t3']
df['rent_by_rooms'] = df['v2a1'] / df['rooms']
df['rent_by_living'] = df['v2a1'] / df['tamviv']
df['rent_by_minor'] = df['v2a1'] / df['hogar_nin']
df['rent_by_adult'] = df['v2a1'] / df['hogar_adul']
df['children_by_adults'] = df['hogar_nin'] / df['hogar_adul']
df['house_quali'] = df['walls'] + df['roof'] + df['floor']
df['tablets_by_adults'] = df['v18q1'] / df['hogar_adul']
df['ratio_nin'] = df['hogar_nin'] / df['hogar_adul']
return df
df = new_variables(df)
 df_test = new_variables(df_test )<prepare_x_and_y>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
params = {'num_leaves': 2000, 'max_depth': 19, 'max_bin': 107, 'n_estimators': 3747,
'bagging_freq': 1, 'bagging_fraction': 0.7135681370918421,
'feature_fraction': 0.49446461478601994, 'min_data_in_leaf': 88,
'learning_rate': 0.015980721586917768, 'num_threads': 3,
'min_sum_hessian_in_leaf': 6,
'random_state' : RANDOM_SEED,
'verbosity' : 1,
'bagging_seed' : RANDOM_SEED,
'boost_from_average' : 'true',
'boost' : 'gbdt',
'metric' : 'rmse',}
model = lgb.train(params,
lgb_train,
num_boost_round=20,
valid_sets=[lgb_train,lgb_eval],
early_stopping_rounds=20,
verbose_eval=1, 
)
 y_pred = model.predict(X_valid)
 rmsle(Y_valid, y_pred )
<prepare_output><EOS> 
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('lgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('lgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('lgb_test.pickle', 'wb'))