Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<predict_on_test>
le = LabelEncoder()
pd.set_option('display.max_rows', 400)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 40)
 warnings.filterwarnings("ignore" )
vc.voting = 'soft'
y_subm_xgb = y_subm.copy(deep=True)
 y_subm_xgb['Target'] = vc.predict(test.drop(xgb_drop_cols, axis=1)) + 1
vc2.voting = 'soft'
y_subm_rf = y_subm.copy(deep=True)
 y_subm_rf['Target'] = vc2.predict(test.drop(et_drop_cols, axis=1)) + 1
vc3.voting = 'soft'
y_subm_lgb = y_subm.copy(deep=True)
 y_subm_lgb['Target'] = vc3.predict(test.drop(lgb_drop_cols, axis=1)) + 1
y_subm_ens = y_subm.copy(deep=True)
 y_subm_ens['Target'] = combine_voters(test)+ 1<define_variables>
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 test.head()
now = datetime.now()
sub_file_xgb = 'submission_soft_XGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_lgb = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_lgb_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_rf = 'submission_soft_RF_{:.4f}_{}.csv'.format(global_rf_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_ens = 'submission_ens_{:.4f}_{}.csv'.format(global_combo_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm_xgb.to_csv(sub_file_xgb, index=False)
 y_subm_lgb.to_csv(sub_file_lgb, index=False)
 y_subm_rf.to_csv(sub_file_rf, index=False)
 y_subm_ens.to_csv(sub_file_ens, index=False )<set_options>
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 pd.DataFrame(categories.category_name.values.reshape(-1, 4))
%matplotlib inline<load_from_csv>
categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
categories['group_id'] = le.fit_transform(categories.group_name.values)
 categories.sample(5 )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print('train data shape :',train_df.shape)
 print('test data shape :',test_df.shape )<count_values>
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
items['item_name'] = items['item_name'].str.replace('.', '')
 for i in [r'[^\w\d\s\.]', r'\bthe\b', r'\bin\b', r'\bis\b',
r'\bfor\b', r'\bof\b', r'\bon\b', r'\band\b',
r'\bto\b', r'\bwith\b' , r'\byo\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(r'\b.\b', ' ')
 items['item_name_no_space'] = items['item_name'].str.replace(' ', '')
 items['item_name_first4'] = [x[:4] for x in items['item_name_no_space']]
items['item_name_first6'] = [x[:6] for x in items['item_name_no_space']]
items['item_name_first11'] = [x[:11] for x in items['item_name_no_space']]
del items['item_name_no_space']
items.item_name_first4 = le.fit_transform(items.item_name_first4.values)
 items.item_name_first6 = le.fit_transform(items.item_name_first6.values)
 items.item_name_first11 = le.fit_transform(items.item_name_first11.values)
 items = items.join(categories.set_index('category_id'), on='category_id')
 items.sample(10 )
y=train_df['Target']
y.value_counts()<count_missing_values>
dupes = items[(items.duplicated(subset=['item_name','category_id'],keep=False)) ]
dupes['in_test'] = dupes.item_id.isin(test.item_id.unique())
 dupes = dupes.groupby('item_name' ).agg({'item_id':['first','last'],'in_test':['first','last']})
 dupes = dupes[(dupes[('in_test', 'first')]==False)|(dupes[('in_test', 'last')]==False)]
temp = dupes[dupes[('in_test', 'first')]==True]
keep_first = dict(zip(temp[('item_id', 'last')], temp[('item_id', 'first')]))
temp = dupes[dupes[('in_test', 'first')]==False]
keep_second = dict(zip(temp[('item_id', 'first')], temp[('item_id', 'last')]))
item_map = {**keep_first, **keep_second}
train_null = train_df.isnull().sum()
train_null[train_null > 0]<count_missing_values>
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 sales =(sales 
.query('0 < item_price < 50000 and 0 < item_cnt_day < 1001')
.replace({
'shop_id':{0:57, 1:58, 11:10},
'item_id':item_map
})
)
 sales = sales[sales['shop_id'].isin(test.shop_id.unique())]
sales['date'] = pd.to_datetime(sales.date,format='%d.%m.%Y')
 sales['weekday'] = sales.date.dt.dayofweek
sales['first_sale_day'] = sales.date.dt.dayofyear
sales['first_sale_day'] += 365 *(sales.date.dt.year-2013)
 sales['first_sale_day'] = sales.groupby('item_id')['first_sale_day'].transform('min' ).astype('int16')
 sales['revenue'] = sales['item_cnt_day']*sales['item_price']
test_null = test_df.isnull().sum()
test_null[test_null > 0]<count_unique_values>
temp = sales.groupby(['shop_id','weekday'] ).agg({'item_cnt_day':'sum'} ).reset_index()
temp = pd.merge(temp, sales.groupby(['shop_id'] ).agg({'item_cnt_day':'sum'} ).reset_index() , on='shop_id', how='left')
 temp.columns = ['shop_id','weekday', 'shop_day_sales', 'shop_total_sales']
temp['day_quality'] = temp['shop_day_sales']/temp['shop_total_sales']
temp = temp[['shop_id','weekday','day_quality']]
dates = pd.DataFrame(data={'date':pd.date_range(start='2013-01-01',end='2015-11-30')})
 dates['weekday'] = dates.date.dt.dayofweek
dates['month'] = dates.date.dt.month
dates['year'] = dates.date.dt.year - 2013
dates['date_block_num'] = dates['year']*12 + dates['month'] - 1
dates['first_day_of_month'] = dates.date.dt.dayofyear
dates['first_day_of_month'] += 365 * dates['year']
dates = dates.join(temp.set_index('weekday'), on='weekday')
 dates = dates.groupby(['date_block_num','shop_id','month','year'] ).agg({'day_quality':'sum','first_day_of_month':'min'} ).reset_index()
dates.query('shop_id == 28' ).head(15 )
train_grphh = train_df.groupby('idhogar')['Target'].apply(lambda x: x.nunique() == 1)
 err_train = train_grphh[train_grphh !=True]
print('Number of households with incorrect poverty level :',len(err_train))<feature_engineering>
sales =(sales 
.groupby(['date_block_num', 'shop_id', 'item_id'])
.agg({
'item_cnt_day':'sum',
'revenue':'sum',
'first_sale_day':'first'
})
.reset_index() 
.rename(columns={'item_cnt_day':'item_cnt'})
)
 sales.sample(5 )
for household in err_train.index:
target = int(train_df[(train_df['idhogar']==household)&(train_df['parentesco1']==1.0)]['Target'])
 train_df.loc[train_df['idhogar']==household,'Target'] = target<train_model>
test['date_block_num'] = 34
del test['ID']
train_df,test_df = train_df.align(test_df,join='inner',axis=1)
 print('Training Features shape: ', train_df.shape)
 print('Testing Features shape: ', test_df.shape )<concatenate>
df = pd.concat([df,test] ).fillna(0)
 df = df.reset_index()
del df['index']
data = pd.concat([train_df,test_df],axis=0)
 data.head()<filter>
df = pd.merge(df, sales, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 df = pd.merge(df, dates, on=['date_block_num','shop_id'], how='left')
 df = pd.merge(df, items.drop(columns=['item_name','group_name','category_name']), on='item_id', how='left' )
<count_missing_values>
shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops_cats = pd.DataFrame(
 np.array(list(product(*[df['shop_id'].unique() , df['category_id'].unique() ]))),
columns =['shop_id', 'category_id'] 
)
 temp = df.groupby(['category_id', 'shop_id'] ).agg({'item_cnt':'sum'} ).reset_index()
temp2 = temp.groupby('shop_id' ).agg({'item_cnt':'sum'} ).rename(columns={'item_cnt':'shop_total'})
 temp = temp.join(temp2, on='shop_id')
 temp['category_proportion'] = temp['item_cnt']/temp['shop_total']
temp = temp[['shop_id', 'category_id', 'category_proportion']]
shops_cats = pd.merge(shops_cats, temp, on=['shop_id','category_id'], how='left')
 shops_cats = shops_cats.fillna(0)
 shops_cats = shops_cats.pivot(index='shop_id', columns=['category_id'])
 kmeans = KMeans(n_clusters=7, random_state=0 ).fit(shops_cats)
 shops_cats['shop_cluster'] = kmeans.labels_.astype('int8')
 shops = shops.join(shops_cats['shop_cluster'], on='shop_id' )
<feature_engineering>
shops.dropna(inplace=True)
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_type'] = 'regular'
shops.loc[shops['shop_name'].str.contains(r'tc'), 'shop_type'] = 'tc'
shops.loc[shops['shop_name'].str.contains(r'mall|center|mega'), 'shop_type'] = 'mall'
shops.loc[shops['shop_id'].isin([9,20]), 'shop_type'] = 'special'
shops.loc[shops['shop_id'].isin([12,55]), 'shop_type'] = 'online'
shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
shops.shop_city = le.fit_transform(shops.shop_city.values)
 shops.shop_type = le.fit_transform(shops.shop_type.values)
 shops.head()
<count_unique_values>
df = pd.merge(df, shops.drop(columns='shop_name'), on='shop_id', how='left')
 df.head()
cat_cols = data.nunique() ==2
cat_cols = list(cat_cols[cat_cols].keys() )<feature_engineering>
df['first_sale_day'] = df.groupby('item_id')['first_sale_day'].transform('max' ).astype('int16')
 df.loc[df['first_sale_day']==0, 'first_sale_day'] = 1035
df['prev_days_on_sale'] = [max(idx)for idx in zip(df['first_day_of_month']-df['first_sale_day'],[0]*len(df)) ]
del df['first_day_of_month']
cols = ['edjefe', 'edjefa','dependency']
data[cols] = data[cols].replace({'no': 0, 'yes':1} ).astype(float)
 data['hogar_mid'] = data['hogar_adul'] - data['hogar_mayor']
data['bedroom%'] = data['bedrooms']/data['rooms']
data['person/rooms'] = data['rooms']/data['hhsize']
data['male_ratio'] = data['r4h3']/data['r4t3']
data['female_ratio'] = data['r4m3']/data['r4t3']
data['female_per_room'] = data['r4m3']/data['rooms']
data['female_per_bedroom'] = data['r4m3']/data['bedrooms']
data['hogarmid_per_bedroom'] = data['hogar_mid']/data['bedrooms']
data['escolari_age'] = data['escolari']/data['age']
data['hogar_nin_per_room'] = data['rooms']/(data['hogar_nin']+1)
 data['male_per_bedroom'] = data['r4h3']/data['bedrooms']
data['male_per_room'] = data['r4h3']/data['rooms']
data['dependencyXmale']= data['dependency']*data['r4h3']
data['dependencyXhogar_mid'] = data['dependency']*data['hogar_mid']
data['dependencyXhogar_adul'] = data['dependency']*data['hogar_adul']
data['dependency_prod_sum']=data['dependencyXmale'] + data['dependencyXhogar_mid'] + data['dependencyXhogar_adul']
data['overcrowding_room_and_bedroom'] =(data['hacdor'] + data['hacapo'])/2
data['no_appliances'] = data['refrig'] + data['computer'] + data['television'] 
<merge>
del sales, categories, shops, shops_cats, temp, temp2, test, dupes, item_map,
df.head()
df_group = pd.DataFrame()
other_list = ['escolari', 'age', 'escolari_age']
for item in other_list:
for function in ['mean','std','min','max','sum']:
group_data = data[item].groupby(data['idhogar'] ).agg(function)
 new_col = item + '_' + function
df_group[new_col] = group_data
df_group = df_group.reset_index()
data = pd.merge(data, df_group, on='idhogar' )<normalization>
df['item_cnt_unclipped'] = df['item_cnt']
df['item_cnt'] = df['item_cnt'].clip(0, 20 )
data.fillna(-1,inplace=True)
<import_modules>
def downcast(df):
float_cols = [c for c in df if df[c].dtype in ["float64"]]
int_cols = [c for c in df if df[c].dtype in ['int64']]
df[float_cols] = df[float_cols].astype('float32')
 df[int_cols] = df[int_cols].astype('int16')
 return df
df = downcast(df )
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )<compute_test_metric>
df['item_age'] =(df['date_block_num'] - df.groupby('item_id')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first4_age'] =(df['date_block_num'] - df.groupby('item_name_first4')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first6_age'] =(df['date_block_num'] - df.groupby('item_name_first6')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first11_age'] =(df['date_block_num'] - df.groupby('item_name_first11')['date_block_num'].transform('min')).astype('int8')
 df['category_age'] =(df['date_block_num'] - df.groupby('category_id')['date_block_num'].transform('min')).astype('int8')
 df['group_age'] =(df['date_block_num'] - df.groupby('group_id')['date_block_num'].transform('min')).astype('int8')
 df['shop_age'] =(df['date_block_num'] - df.groupby('shop_id')['date_block_num'].transform('min')).astype('int8' )
def macro_f1_score(labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True<compute_train_metric>
temp = df.query('item_cnt > 0' ).groupby(['item_id','shop_id'] ).agg({'date_block_num':'min'} ).reset_index()
temp.columns = ['item_id', 'shop_id', 'item_shop_first_sale']
df = pd.merge(df, temp, on=['item_id','shop_id'], how='left')
 df['item_shop_first_sale'] = df['item_shop_first_sale'].fillna(50)
 df['item_age_if_shop_sale'] =(df['date_block_num'] > df['item_shop_first_sale'])* df['item_age']
df['item_age_without_shop_sale'] =(df['date_block_num'] <= df['item_shop_first_sale'])* df['item_age']
del df['item_shop_first_sale']
X = train_df
logR = LogisticRegression(class_weight='balanced',C=0.0005)
 cv_score = cross_val_score(logR, X, y, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}')
<save_to_csv>
def agg_cnt_col(df, merging_cols, new_col,aggregation):
temp = df.groupby(merging_cols ).agg(aggregation ).reset_index()
temp.columns = merging_cols + [new_col]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_cnt_col(df, ['date_block_num','item_id'],'item_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'item_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','group_id','shop_id'],'group_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','group_id'],'group_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_id'],'shop_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_city'],'city_cnt',{'item_cnt':'mean'} )
logR.fit(X,y)
 preds_log = logR.predict(test_df)
 sub_log = pd.DataFrame({'Id':test_df.index, 'Target':preds_log})
 sub_log.to_csv('sub_log1.csv', index=False)
<set_options>
def new_item_sales(df, merging_cols, new_col):
temp =(
 df 
.query('item_age==0')
.groupby(merging_cols)['item_cnt'] 
.mean() 
.reset_index() 
.rename(columns={'item_cnt': new_col})
)
 df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = new_item_sales(df, ['date_block_num','category_id','shop_id'], 'new_items_in_cat')
 df = new_item_sales(df, ['date_block_num','category_id'], 'new_items_in_cat_all_shops' )
%matplotlib inline<load_from_csv>
def agg_price_col(df, merging_cols, new_col):
temp = df.groupby(merging_cols ).agg({'revenue':'sum','item_cnt_unclipped':'sum'} ).reset_index()
temp[new_col] = temp['revenue']/temp['item_cnt_unclipped']
temp = temp[merging_cols + [new_col]]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_price_col(df,['date_block_num','item_id'],'item_price')
 df = agg_price_col(df,['date_block_num','category_id'],'category_price')
 df = agg_price_col(df,['date_block_num'],'block_price' )
train= pd.read_csv(".. /input/train.csv")
 test= pd.read_csv(".. /input/test.csv")
 train.head()<count_values>
df = downcast(df )
train['Target'].value_counts()<drop_column>
def lag_feature(df, lag, col, merge_cols):
temp = df[merge_cols + [col]]
temp = temp.groupby(merge_cols ).agg({f'{col}':'first'} ).reset_index()
temp.columns = merge_cols + [f'{col}_lag{lag}']
temp['date_block_num'] += lag
df = pd.merge(df, temp, on=merge_cols, how='left')
 df[f'{col}_lag{lag}'] = df[f'{col}_lag{lag}'].fillna(0 ).astype('float32')
 return df
train2=train.drop('Target',axis=1 )<concatenate>
lag12_cols = {
'item_cnt':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops':['date_block_num', 'item_id'],
'category_cnt':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops':['date_block_num', 'category_id'],
'group_cnt':['date_block_num', 'shop_id', 'group_id'],
'group_cnt_all_shops':['date_block_num', 'group_id'],
'shop_cnt':['date_block_num', 'shop_id'],
'city_cnt':['date_block_num', 'shop_city'],
'new_items_in_cat':['date_block_num', 'shop_id', 'category_id'],
'new_items_in_cat_all_shops':['date_block_num', 'category_id']
}
for col,merge_cols in lag12_cols.items() :
df[f'{col}_lag1to12'] = 0
for i in range(1,13):
df = lag_feature(df, i, col, merge_cols)
 df[f'{col}_lag1to12'] += df[f'{col}_lag{i}']
if i > 2:
del df[f'{col}_lag{i}']
if col == 'item_cnt':
del df[f'{col}_lag1']
del df[f'{col}_lag2']
else:
del df[col]
data = train2.append(test,sort=True )<count_values>
lag2_cols = {
'item_cnt_unclipped':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops_median':['date_block_num', 'item_id'],
'category_cnt_median':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops_median':['date_block_num', 'category_id']
}
for col in lag2_cols:
df = lag_feature(df, 1, col, merge_cols)
 df = lag_feature(df, 2, col, merge_cols)
 if col!='item_cnt_unclipped':
del df[col]
data['dependency'].value_counts()<data_type_conversions>
df['item_cnt_diff'] = df['item_cnt_unclipped_lag1']/df['item_cnt_lag1to12']
df['item_cnt_all_shops_diff'] = df['item_cnt_all_shops_lag1']/df['item_cnt_all_shops_lag1to12']
df['category_cnt_diff'] = df['category_cnt_lag1']/df['category_cnt_lag1to12']
df['category_cnt_all_shops_diff'] = df['category_cnt_all_shops_lag1']/df['category_cnt_all_shops_lag1to12']
mapping = {"yes": 1, "no": 0}
data['dependency'] = data['dependency'].replace(mapping ).astype(np.float64)
 data['edjefa'] = data['edjefa'].replace(mapping ).astype(np.float64)
 data['edjefe'] = data['edjefe'].replace(mapping ).astype(np.float64)
 data[['dependency', 'edjefa', 'edjefe']].describe()<feature_engineering>
df = lag_feature(df, 1, 'category_price',['date_block_num', 'category_id'])
 df = lag_feature(df, 1, 'block_price',['date_block_num'])
 del df['category_price'], df['block_price']
data.loc[data['rez_esc'] == 99.0 , 'rez_esc'] = 5<feature_engineering>
df.loc[(df['item_age']>0)&(df['item_cnt_lag1to12'].isna()), 'item_cnt_lag1to12'] = 0
df.loc[(df['category_age']>0)&(df['category_cnt_lag1to12'].isna()), 'category_cnt_lag1to12'] = 0
df.loc[(df['group_age']>0)&(df['group_cnt_lag1to12'].isna()), 'group_cnt_lag1to12'] = 0
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 )<feature_engineering>
df['item_cnt_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],df['shop_age'],[12]*len(df)) ]
df['item_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],[12]*len(df)) ]
df['category_cnt_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['category_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
df['group_cnt_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],df['shop_age'],[12]*len(df)) ]
df['group_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],[12]*len(df)) ]
df['city_cnt_lag1to12'] /= [min(idx)for idx in zip(df['date_block_num'],[12]*len(df)) ]
df['shop_cnt_lag1to12'] /= [min(idx)for idx in zip(df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
data['v18q1'] = data['v18q1'].fillna(0)
 data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()<feature_engineering>
df = downcast(df )
elec = []
for i, row in data.iterrows() :
if row['noelec'] == 1:
elec.append(0)
 elif row['coopele'] == 1:
elec.append(1)
 elif row['public'] == 1:
elec.append(2)
 elif row['planpri'] == 1:
elec.append(3)
 else:
elec.append(np.nan)
 data['elec'] = elec
data['elec-missing'] = data['elec'].isnull()<drop_column>
def past_information(df, merging_cols, new_col, aggregation):
temp = []
for i in range(1,35):
block = df.query(f'date_block_num < {i}' ).groupby(merging_cols ).agg(aggregation ).reset_index()
block.columns = merging_cols + [new_col]
block['date_block_num'] = i
block = block[block[new_col]>0]
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=['date_block_num']+merging_cols, how='left')
 return df
df = past_information(df, ['item_id'],'last_item_price',{'item_price':'last'})
 df = past_information(df, ['shop_id','item_id'],'item_cnt_sum_alltime',{'item_cnt':'sum'})
 df = past_information(df, ['item_id'],'item_cnt_sum_alltime_allshops',{'item_cnt':'sum'})
 del df['revenue'], df['item_cnt_unclipped'], df['item_price']
data = data.drop(columns = ['noelec', 'coopele', 'public', 'planpri'])
<drop_column>
df['relative_price_item_block_lag1'] = df['last_item_price']/df['block_price_lag1']
data['walls'] = np.argmax(np.array(data[['epared1', 'epared2', 'epared3']]),
axis = 1)
 data = data.drop(columns = ['epared1', 'epared2', 'epared3'] )<drop_column>
df['item_cnt_per_day_alltime'] =(df['item_cnt_sum_alltime']/df['prev_days_on_sale'] ).fillna(0)
 df['item_cnt_per_day_alltime_allshops'] =(df['item_cnt_sum_alltime_allshops']/df['prev_days_on_sale'] ).fillna(0 )
data['roof'] = np.argmax(np.array(data[['etecho1', 'etecho2', 'etecho3']]),
axis = 1)
 data = data.drop(columns = ['etecho1', 'etecho2', 'etecho3'] )<drop_column>
gc.collect()
df = downcast(df )
data['floor'] = np.argmax(np.array(data[['eviv1', 'eviv2', 'eviv3']]),
axis = 1)
 data = data.drop(columns = ['eviv1', 'eviv2', 'eviv3'] )<drop_column>
def matching_name_cat_age(df,n,all_shops):
temp_cols = [f'same_name{n}catage_cnt','date_block_num', f'item_name_first{n}','item_age','category_id']
if all_shops:
temp_cols[0] += '_all_shops'
else:
temp_cols += ['shop_id']
temp = []
for i in range(1,35):
block =(
 df 
.query(f'date_block_num < {i}')
.groupby(temp_cols[2:])
.agg({'item_cnt':'mean'})
.reset_index() 
.rename(columns={'item_cnt':temp_cols[0]})
)
 block = block[block[temp_cols[0]]>0]
block['date_block_num'] = i
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=temp_cols[1:], how='left')
 return df
for n in [4,6,11]:
for all_shops in [True,False]:
df = matching_name_cat_age(df,n,all_shops )
data['flush'] = np.argmax(np.array(data[["sanitario1",'sanitario5', 'sanitario2', 'sanitario3',"sanitario6"]]),
axis = 1)
 data = data.drop(columns = ["sanitario1",'sanitario5', 'sanitario2', 'sanitario3',"sanitario6"] )<drop_column>
df = downcast(df)
 int8_cols = [
'item_cnt','month','group_id','shop_type',
'shop_city','shop_id','date_block_num','category_id',
'item_age',
]
int16_cols = [
'item_id','item_name_first4',
'item_name_first6','item_name_first11'
]
for col in int8_cols:
df[col] = df[col].astype('int8')
 for col in int16_cols:
df[col] = df[col].astype('int16' )
data = data[[x for x in data if not x.startswith('SQB')]]
data = data.drop(columns = ['agesq'] )<drop_column>
def nearby_item_data(df,col):
if col in ['item_cnt_unclipped_lag1','item_cnt_lag1to12']:
cols = ['date_block_num', 'shop_id', 'item_id']
temp = df[cols + [col]]
else:
cols = ['date_block_num', 'item_id']
temp = df.groupby(cols ).agg({col:'first'} ).reset_index() [cols + [col]]
temp.columns = cols + [f'below_{col}']
temp['item_id'] += 1
df = pd.merge(df, temp, on=cols, how='left')
 temp.columns = cols + [f'above_{col}']
temp['item_id'] -= 2
df = pd.merge(df, temp, on=cols, how='left')
 return df
item_cols = ['item_cnt_unclipped_lag1','item_cnt_lag1to12',
'item_cnt_all_shops_lag1','item_cnt_all_shops_lag1to12']
for col in item_cols:
df = nearby_item_data(df,col)
 del temp
data['waterprovision'] = np.argmax(np.array(data[['abastaguano', 'abastaguafuera', 'abastaguadentro']]),
axis = 1)
 data = data.drop(columns = ['abastaguano', 'abastaguafuera', 'abastaguadentro'] )<drop_column>
results = Counter()
items['item_name'].str.split().apply(results.update)
 words = []
cnts = []
for key, value in results.items() :
words.append(key)
 cnts.append(value)
 counts = pd.DataFrame({'word':words,'count':cnts})
 common_words = counts.query('count>200' ).word.to_list()
for word in common_words:
items[f'{word}_in_name'] = items['item_name'].str.contains(word ).astype('int8')
 drop_cols = [
'item_id','category_id','item_name','item_name_first4',
'item_name_first6','item_name_first11',
'category_name','group_name','group_id'
]
items = items.drop(columns=drop_cols )
data['inst'] = np.argmax(np.array(data[[c for c in data if c.startswith('instl')]]), axis = 1)
 data = data.drop(columns = [c for c in data if c.startswith('instlevel')])
<drop_column>
df = df.join(items, on='item_id' )
data['waterprovision'] = np.argmax(np.array(data[['energcocinar1','energcocinar4', 'energcocinar3', 'energcocinar2']]),
axis = 1)
 data = data.drop(columns = ['energcocinar1','energcocinar4', 'energcocinar3', 'energcocinar2'] )<feature_engineering>
def binary_encode(df, letters, cols):
encoder = ce.BinaryEncoder(cols=[f'item_name_first{letters}'], return_df=True)
 temp = encoder.fit_transform(df[f'item_name_first{letters}'])
 df = pd.concat([df,temp], axis=1)
 del df[f'item_name_first{letters}_0']
name_cols = [f'item_name_first{letters}_{x}' for x in range(1,cols)]
df[name_cols] = df[name_cols].astype('int8')
 return df
df = binary_encode(df, 11, 15)
 del df['item_name_first4'], df['item_name_first6']
data.loc[pd.isnull(data['meaneduc']), 'meaneduc'] = data.loc[pd.isnull(data['meaneduc']), 'escolari']<split>
df.to_pickle('df_complete.pkl' )
train2=data.iloc[0:9557,:]
test2=data.iloc[9557:33413,:]<drop_column>
%reset -f
test2.drop(['Id','idhogar'],axis=1,inplace=True )<drop_column>
pd.set_option('display.max_rows', 160)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 30)
 warnings.filterwarnings("ignore" )
X=train2.drop(['Id','idhogar'],axis=1 )<prepare_x_and_y>
df = pd.read_pickle('.. /input/files-top-scoring-notebook-output-exploration/df_complete.pkl')
 X_train = df[~df.date_block_num.isin([0,1,33,34])]
y_train = X_train['item_cnt']
del X_train['item_cnt']
X_val = df[df['date_block_num']==33]
y_val = X_val['item_cnt']
del X_val['item_cnt']
X_test = df[df['date_block_num']==34].drop(columns='item_cnt')
 X_test = X_test.reset_index()
del X_test['index']
del df
y=train['Target']<import_modules>
def build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features):
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_val = lgb.Dataset(X_val, y_val)
 model = lgb.train(params=params, train_set=lgb_train, valid_sets=(lgb_train, lgb_val), verbose_eval=50,
categorical_feature=cat_features)
 return model
import xgboost as xgb<choose_model_class>
params = {
'objective': 'rmse',
'metric': 'rmse',
'num_leaves': 1023,
'min_data_in_leaf':10,
'feature_fraction':0.7,
'learning_rate': 0.01,
'num_rounds': 1000,
'early_stopping_rounds': 30,
'seed': 1
}
cat_features = ['category_id','month','shop_id','shop_city']
lgb_model = build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features)
 lgb_model.save_model('initial_lgb_model.txt' )
xg=xgb.XGBClassifier(n_estimators=200 )<train_model>
submission = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv')
 submission['item_cnt_month'] = lgb_model.predict(X_test ).clip(0,20)
 submission[['ID', 'item_cnt_month']].to_csv('initial_lgb_submission.csv', index=False )
xg.fit(X,y )<predict_on_test>
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
for i in [r'[^\w\d\s]', r'\bthe\b', r'\bin\b', r'\bfor\b', r'\bof\b', r'\bd\b', r'\bis\b', r'\bon\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(' ', '')
 items = items.join(categories.set_index('category_id'), on='category_id')
 shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
preds = xg.predict(test2 )<compute_test_metric>
X_train['lgb_pred'] = lgb_model.predict(X_train ).clip(0,20)
 X_train['target'] = y_train
X_train['sq_err'] =(X_train['lgb_pred']-X_train['target'])**2
X_val['lgb_pred'] = lgb_model.predict(X_val ).clip(0,20)
 X_val['target'] = y_val
X_val['sq_err'] =(X_val['lgb_pred']-X_val['target'])**2
X_test['lgb_pred'] = lgb_model.predict(X_test ).clip(0,20 )
def macro_f1_score(
 labels, predictions):
predictions = predictions.reshape(len(np.unique(labels)) , -1 ).argmax(axis = 0)
 metric_value = f1_score(labels, predictions, average = 'macro')
 return 'macro_f1', metric_value, True<import_modules>
data = X_train.groupby('date_block_num' ).agg({'lgb_pred':'mean','target':'mean','sq_err':'mean'} ).reset_index()
data['new_item_rmse'] = np.sqrt(X_train.query('item_age<=1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data['old_item_rmse'] = np.sqrt(X_train.query('item_age>1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data = data.append([
{'date_block_num':33,
'target':X_val.target.mean() ,
'lgb_pred':X_val.lgb_pred.mean() ,
'sq_err':np.sqrt(X_val.sq_err.mean()),
'old_item_rmse':np.sqrt(X_val.query('item_age>1' ).sq_err.mean()),
'new_item_rmse':np.sqrt(X_val.query('item_age<=1' ).sq_err.mean())},
{'date_block_num':34,
'target':0,
'lgb_pred':X_test.lgb_pred.mean() ,
'sq_err':0,
'old_item_rmse':0,
'new_item_rmse':0}
],
ignore_index=True 
)
 data['date'] = [x[:7] for x in pd.date_range(start='2013-03',end='2015-09',freq='MS' ).astype('str')]+['Validation','Test']
import lightgbm as lgb
import sklearn.model_selection as model_selection
from sklearn.metrics import f1_score, make_scorer<choose_model_class>
df = pd.read_pickle('.. /input/files-top-scoring-notebook-output-exploration/df_complete.pkl')
(
 df
[df['category_id'].isin(X_test.category_id.unique())] 
.query('item_cnt>0')
.groupby('category_id')
.agg({
'category_age':'max',
'shop_id':['nunique','unique'],
'item_cnt':'sum'
})
.join(categories['category_name'])
.join(
 X_test 
.groupby('category_id')
.agg({'item_id':'nunique'})
.rename(columns={'item_id':'test_set_items'})
)
.sort_values(( 'shop_id', 'nunique')) 
.head(20)
 )
lgmodel = lgb.LGBMClassifier(metric = "",num_class = 4 )<init_hyperparams>
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('shop_id')
.agg({
'sq_err':'mean',
'target':'mean',
'lgb_pred':'mean'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('shop_id')
.agg({'test_pred':'mean'})
)
.join(shops['shop_name'])
.sort_values('sq_err', ascending=False)
.head(20)
 )
hyp_OPTaaS = { 'boosting_type': 'dart',
'colsample_bytree': 0.9843467236959204,
'learning_rate': 0.11598629586769524,
'min_child_samples': 44,
'num_leaves': 49,
'reg_alpha': 0.35397370408131534,
'reg_lambda': 0.5904910774606467,
'subsample': 0.6299872254632797,
'subsample_for_bin': 60611} 
<choose_model_class>
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('category_id')
.agg({
'sq_err':['sum','mean'],
'target':'mean',
'lgb_pred':['sum','mean'],
'item_id':'nunique'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred','item_id':'test_items'})
.groupby('category_id')
.agg({
'test_pred':['sum','mean'],
'test_items':'nunique'
}),
on='category_id' 
)
.join(categories)
.sort_values(( 'sq_err', 'mean'), ascending=False)
.head(20)
 )
model = lgb.LGBMClassifier(**hyp_OPTaaS, class_weight = 'balanced',max_depth=-1,objective = 'multiclass', n_jobs = -1, n_estimators = 100 )<train_model>
CATEGORY = 20 
(
 items[
items['item_id'].isin(X_val.item_id.unique())&
items['item_id'].isin(X_test.item_id.unique())
 ]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('item_id')
.agg({'test_pred':'mean'}),
on='item_id' 
)
.join(
 X_val 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'target':'mean',
'sq_err':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={
'lgb_pred':'val_pred',
'target':'val_target',
}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('sq_err',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )
model.fit(X, y )<predict_on_test>
CATEGORY = 20 
(
 items[items['item_id'].isin(X_test.item_id.unique())]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={'lgb_pred':'test_pred'}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('test_pred',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'item_category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )