Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<count_missing_values>
le = LabelEncoder()
pd.set_option('display.max_rows', 400)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 40)
 warnings.filterwarnings("ignore" )
print("Total number of null/nan values in train data
{}".format(train_df.isnull().sum()))<count_missing_values>
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 test.head()
print("Total number of null/nan values in test data
{}".format(test_df.isnull().sum()))<drop_column>
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 pd.DataFrame(categories.category_name.values.reshape(-1, 4))
train_data = train_df.drop(["surface_label"], axis = 1)
 train_data.columns[3:]<compute_test_metric>
categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
categories['group_id'] = le.fit_transform(categories.group_name.values)
 categories.sample(5 )
def sma(x, y, z):
sum = 0
for i in range(len(x)) :
sum +=(abs(x[i])+ abs(y[i])+ abs(z[i]))
return sum/len(x )<compute_test_metric>
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
items['item_name'] = items['item_name'].str.replace('.', '')
 for i in [r'[^\w\d\s\.]', r'\bthe\b', r'\bin\b', r'\bis\b',
r'\bfor\b', r'\bof\b', r'\bon\b', r'\band\b',
r'\bto\b', r'\bwith\b' , r'\byo\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(r'\b.\b', ' ')
 items['item_name_no_space'] = items['item_name'].str.replace(' ', '')
 items['item_name_first4'] = [x[:4] for x in items['item_name_no_space']]
items['item_name_first6'] = [x[:6] for x in items['item_name_no_space']]
items['item_name_first11'] = [x[:11] for x in items['item_name_no_space']]
del items['item_name_no_space']
items.item_name_first4 = le.fit_transform(items.item_name_first4.values)
 items.item_name_first6 = le.fit_transform(items.item_name_first6.values)
 items.item_name_first11 = le.fit_transform(items.item_name_first11.values)
 items = items.join(categories.set_index('category_id'), on='category_id')
 items.sample(10 )
train_data['sma'] = sma(train_data['angular_velocity_X'], train_data['angular_velocity_Y'], train_data['angular_velocity_Z'])
 test_df['sma'] = sma(test_df['angular_velocity_X'], test_df['angular_velocity_Y'], test_df['angular_velocity_Z'] )<feature_engineering>
dupes = items[(items.duplicated(subset=['item_name','category_id'],keep=False)) ]
dupes['in_test'] = dupes.item_id.isin(test.item_id.unique())
 dupes = dupes.groupby('item_name' ).agg({'item_id':['first','last'],'in_test':['first','last']})
 dupes = dupes[(dupes[('in_test', 'first')]==False)|(dupes[('in_test', 'last')]==False)]
temp = dupes[dupes[('in_test', 'first')]==True]
keep_first = dict(zip(temp[('item_id', 'last')], temp[('item_id', 'first')]))
temp = dupes[dupes[('in_test', 'first')]==False]
keep_second = dict(zip(temp[('item_id', 'first')], temp[('item_id', 'last')]))
item_map = {**keep_first, **keep_second}
def feat_eng(data):
df = pd.DataFrame()
data['totl_anglr_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 +
data['angular_velocity_Z'])** 0.5
data['totl_linr_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 +
data['linear_acceleration_Z'])**0.5
data['totl_xyz'] =(data['orientation_X']**2 + data['orientation_Y']**2 +
data['orientation_Z'])**0.5
data['acc_vs_vel'] = data['totl_linr_acc'] / data['totl_anglr_vel']
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_q25'] = data.groupby(['series_id'])[col].quantile(0.25)
 df[col + '_q75'] = data.groupby(['series_id'])[col].quantile(0.75)
 df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_mean_abs_chg'] = data.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df<categorify>
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 sales =(sales 
.query('0 < item_price < 50000 and 0 < item_cnt_day < 1001')
.replace({
'shop_id':{0:57, 1:58, 11:10},
'item_id':item_map
})
)
 sales = sales[sales['shop_id'].isin(test.shop_id.unique())]
sales['date'] = pd.to_datetime(sales.date,format='%d.%m.%Y')
 sales['weekday'] = sales.date.dt.dayofweek
sales['first_sale_day'] = sales.date.dt.dayofyear
sales['first_sale_day'] += 365 *(sales.date.dt.year-2013)
 sales['first_sale_day'] = sales.groupby('item_id')['first_sale_day'].transform('min' ).astype('int16')
 sales['revenue'] = sales['item_cnt_day']*sales['item_price']
X_train = feat_eng(train_data)
 test_data = feat_eng(test_df)
 print(X_train.shape)
 X_train.head()<prepare_x_and_y>
temp = sales.groupby(['shop_id','weekday'] ).agg({'item_cnt_day':'sum'} ).reset_index()
temp = pd.merge(temp, sales.groupby(['shop_id'] ).agg({'item_cnt_day':'sum'} ).reset_index() , on='shop_id', how='left')
 temp.columns = ['shop_id','weekday', 'shop_day_sales', 'shop_total_sales']
temp['day_quality'] = temp['shop_day_sales']/temp['shop_total_sales']
temp = temp[['shop_id','weekday','day_quality']]
dates = pd.DataFrame(data={'date':pd.date_range(start='2013-01-01',end='2015-11-30')})
 dates['weekday'] = dates.date.dt.dayofweek
dates['month'] = dates.date.dt.month
dates['year'] = dates.date.dt.year - 2013
dates['date_block_num'] = dates['year']*12 + dates['month'] - 1
dates['first_day_of_month'] = dates.date.dt.dayofyear
dates['first_day_of_month'] += 365 * dates['year']
dates = dates.join(temp.set_index('weekday'), on='weekday')
 dates = dates.groupby(['date_block_num','shop_id','month','year'] ).agg({'day_quality':'sum','first_day_of_month':'min'} ).reset_index()
dates.query('shop_id == 28' ).head(15 )
tsne_data = X_train
x_tsne = MinMaxScaler().fit_transform(tsne_data)
 y_tsne = y_train["surface"]
x_tsne = np.nan_to_num(x_tsne )<train_model>
sales =(sales 
.groupby(['date_block_num', 'shop_id', 'item_id'])
.agg({
'item_cnt_day':'sum',
'revenue':'sum',
'first_sale_day':'first'
})
.reset_index() 
.rename(columns={'item_cnt_day':'item_cnt'})
)
 sales.sample(5 )
def perform_model(test_data, model, X_train, y_train, X_test, y_test, class_labels, cm_normalize=True, \
print_cm=True, cm_cmap=plt.cm.Greens):
results = dict()
train_start_time = datetime.now()
print('training the model.. ')
 model.fit(X_train, y_train)
 print('Done
')
 train_end_time = datetime.now()
results['training_time'] = train_end_time - train_start_time
print('training_time(HH:MM:SS.ms)- {}
'.format(results['training_time']))
print('Predicting test data')
 test_start_time = datetime.now()
y_pred = model.predict(X_test)
 prediction = model.predict(test_data)
 test_end_time = datetime.now()
print('Done
')
 results['testing_time'] = test_end_time - test_start_time
print('testing time(HH:MM:SS:ms)- {}
'.format(results['testing_time']))
results['predicted'] = y_pred
accuracy = metrics.accuracy_score(y_true=y_test, y_pred=y_pred)
 results['accuracy'] = accuracy
print('---------------------')
 print('| Accuracy |')
 print('---------------------')
 print('
{}
'.format(accuracy))
cm = metrics.confusion_matrix(y_test, y_pred)
 results['confusion_matrix'] = cm
if print_cm:
print('--------------------')
 print('| Confusion Matrix |')
 print('--------------------')
 print('
{}'.format(cm))
plt.figure(figsize=(8,8))
plt.grid(b=False)
 plot_confusion_matrix(cm, classes=class_labels, normalize=True, title='Normalized confusion matrix', cmap = cm_cmap)
 plt.show()
print('-------------------------')
 print('| Classifiction Report |')
 print('-------------------------')
 classification_report = metrics.classification_report(y_test, y_pred)
 results['classification_report'] = classification_report
print(classification_report)
 results['model'] = model
return prediction, results 
<find_best_params>
test['date_block_num'] = 34
del test['ID']
def print_grid_search_attributes(model):
print('--------------------------')
 print('| Best Estimator |')
 print('--------------------------')
 print('
\t{}
'.format(model.best_estimator_))
print('--------------------------')
 print('| Best parameters |')
 print('--------------------------')
 print('\tParameters of best estimator :
\t{}
'.format(model.best_params_))
print('---------------------------------')
 print('| No of CrossValidation sets |')
 print('--------------------------------')
 print('
\tTotal numbre of cross validation sets: {}
'.format(model.n_splits_))
print('--------------------------')
 print('| Best Score |')
 print('--------------------------')
 print('
\tAverage Cross Validate scores of best estimator :
\t{}
'.format(model.best_score_))<prepare_x_and_y>
df = pd.concat([df,test] ).fillna(0)
 df = df.reset_index()
del df['index']
X_train = np.nan_to_num(X_train)
 test_data = np.nan_to_num(test_data )<split>
df = pd.merge(df, sales, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 df = pd.merge(df, dates, on=['date_block_num','shop_id'], how='left')
 df = pd.merge(df, items.drop(columns=['item_name','group_name','category_name']), on='item_id', how='left' )
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train["surface_label"], test_size = 0.3, random_state = 4, stratify = y_train["surface_label"], shuffle = True)
 X_train.shape, y_train.shape, X_test.shape, y_test.shape<train_on_grid>
shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops_cats = pd.DataFrame(
 np.array(list(product(*[df['shop_id'].unique() , df['category_id'].unique() ]))),
columns =['shop_id', 'category_id'] 
)
 temp = df.groupby(['category_id', 'shop_id'] ).agg({'item_cnt':'sum'} ).reset_index()
temp2 = temp.groupby('shop_id' ).agg({'item_cnt':'sum'} ).rename(columns={'item_cnt':'shop_total'})
 temp = temp.join(temp2, on='shop_id')
 temp['category_proportion'] = temp['item_cnt']/temp['shop_total']
temp = temp[['shop_id', 'category_id', 'category_proportion']]
shops_cats = pd.merge(shops_cats, temp, on=['shop_id','category_id'], how='left')
 shops_cats = shops_cats.fillna(0)
 shops_cats = shops_cats.pivot(index='shop_id', columns=['category_id'])
 kmeans = KMeans(n_clusters=7, random_state=0 ).fit(shops_cats)
 shops_cats['shop_cluster'] = kmeans.labels_.astype('int8')
 shops = shops.join(shops_cats['shop_cluster'], on='shop_id' )
labels = ['fine_concrete', 'concrete', 'soft_tiles', 'tiled', 'soft_pvc', 'hard_tiles_large_space', 'carpet', 'hard_tiles', 'wood']
parameters = {'C':[0.01, 0.1, 1, 10, 20, 30], 'penalty':['l2','l1']}
log_reg = linear_model.LogisticRegression()
log_reg_grid = GridSearchCV(log_reg, param_grid = parameters, cv = 10, verbose = 1, n_jobs = -1)
 predict_lr, log_reg_grid_results = perform_model(test_data, log_reg_grid, X_train, y_train, X_test, y_test, class_labels = labels )<train_on_grid>
shops.dropna(inplace=True)
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_type'] = 'regular'
shops.loc[shops['shop_name'].str.contains(r'tc'), 'shop_type'] = 'tc'
shops.loc[shops['shop_name'].str.contains(r'mall|center|mega'), 'shop_type'] = 'mall'
shops.loc[shops['shop_id'].isin([9,20]), 'shop_type'] = 'special'
shops.loc[shops['shop_id'].isin([12,55]), 'shop_type'] = 'online'
shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
shops.shop_city = le.fit_transform(shops.shop_city.values)
 shops.shop_type = le.fit_transform(shops.shop_type.values)
 shops.head()
param_grid = {'max_depth': np.arange(5,8,1), \
'n_estimators':np.arange(130,170,10)}
gbdt = GradientBoostingClassifier()
gbdt_grid = GridSearchCV(gbdt, param_grid=param_grid, n_jobs=-1)
 predict_gbdt, gbdt_grid_results = perform_model(test_data, gbdt_grid, X_train, y_train, X_test, y_test, class_labels=labels)
 print_grid_search_attributes(gbdt_grid_results['model'] )<train_on_grid>
df = pd.merge(df, shops.drop(columns='shop_name'), on='shop_id', how='left')
 df.head()
params = {'n_estimators': np.arange(10,201,20), 'max_depth':np.arange(3,15,2)}
rfc = RandomForestClassifier()
rfc_grid = GridSearchCV(rfc, param_grid=params, cv = 10, n_jobs=-1)
 labels = ['fine_concrete', 'concrete', 'soft_tiles', 'tiled', 'soft_pvc', 'hard_tiles_large_space', 'carpet', 'hard_tiles', 'wood']
predict_rf, rfc_grid_results = perform_model(test_data, rfc_grid, X_train, y_train, X_test, y_test, class_labels = labels)
 print_grid_search_attributes(rfc_grid_results['model'] )<save_to_csv>
df['first_sale_day'] = df.groupby('item_id')['first_sale_day'].transform('max' ).astype('int16')
 df.loc[df['first_sale_day']==0, 'first_sale_day'] = 1035
df['prev_days_on_sale'] = [max(idx)for idx in zip(df['first_day_of_month']-df['first_sale_day'],[0]*len(df)) ]
del df['first_day_of_month']
submission = pd.read_csv(".. /input/sample_submission.csv")
 submission["surface"] = predict_rf
submission["surface"] = submission["surface"].map({0:'fine_concrete', 1:'concrete', 2:'soft_tiles', 3:'tiled', 4:'soft_pvc',
5:'hard_tiles_large_space', 6:'carpet', 7:'hard_tiles', 8:'wood'})
 submission.to_csv("sample_submission.csv", index = False )<set_options>
del sales, categories, shops, shops_cats, temp, temp2, test, dupes, item_map,
df.head()
print(os.listdir(".. /input"))
style.use('ggplot')
 warnings.filterwarnings('ignore')
 py.init_notebook_mode(connected=True)
<load_from_csv>
df['item_cnt_unclipped'] = df['item_cnt']
df['item_cnt'] = df['item_cnt'].clip(0, 20 )
X_train = pd.read_csv('.. /input/X_train.csv')
 X_train.head(3 )<load_from_csv>
def downcast(df):
float_cols = [c for c in df if df[c].dtype in ["float64"]]
int_cols = [c for c in df if df[c].dtype in ['int64']]
df[float_cols] = df[float_cols].astype('float32')
 df[int_cols] = df[int_cols].astype('int16')
 return df
df = downcast(df )
y_train = pd.read_csv('.. /input/y_train.csv')
 y_train.head(3 )<load_from_csv>
df['item_age'] =(df['date_block_num'] - df.groupby('item_id')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first4_age'] =(df['date_block_num'] - df.groupby('item_name_first4')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first6_age'] =(df['date_block_num'] - df.groupby('item_name_first6')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first11_age'] =(df['date_block_num'] - df.groupby('item_name_first11')['date_block_num'].transform('min')).astype('int8')
 df['category_age'] =(df['date_block_num'] - df.groupby('category_id')['date_block_num'].transform('min')).astype('int8')
 df['group_age'] =(df['date_block_num'] - df.groupby('group_id')['date_block_num'].transform('min')).astype('int8')
 df['shop_age'] =(df['date_block_num'] - df.groupby('shop_id')['date_block_num'].transform('min')).astype('int8' )
X_test = pd.read_csv('.. /input/X_test.csv')
 X_test.head(3 )<count_values>
temp = df.query('item_cnt > 0' ).groupby(['item_id','shop_id'] ).agg({'date_block_num':'min'} ).reset_index()
temp.columns = ['item_id', 'shop_id', 'item_shop_first_sale']
df = pd.merge(df, temp, on=['item_id','shop_id'], how='left')
 df['item_shop_first_sale'] = df['item_shop_first_sale'].fillna(50)
 df['item_age_if_shop_sale'] =(df['date_block_num'] > df['item_shop_first_sale'])* df['item_age']
df['item_age_without_shop_sale'] =(df['date_block_num'] <= df['item_shop_first_sale'])* df['item_age']
del df['item_shop_first_sale']
target = y_train['surface'].value_counts().reset_index().rename(columns = {'index' : 'target'})
 target<count_missing_values>
def agg_cnt_col(df, merging_cols, new_col,aggregation):
temp = df.groupby(merging_cols ).agg(aggregation ).reset_index()
temp.columns = merging_cols + [new_col]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_cnt_col(df, ['date_block_num','item_id'],'item_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'item_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','group_id','shop_id'],'group_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','group_id'],'group_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_id'],'shop_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_city'],'city_cnt',{'item_cnt':'mean'} )
X_train.isnull().sum()<count_duplicates>
def new_item_sales(df, merging_cols, new_col):
temp =(
 df 
.query('item_age==0')
.groupby(merging_cols)['item_cnt'] 
.mean() 
.reset_index() 
.rename(columns={'item_cnt': new_col})
)
 df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = new_item_sales(df, ['date_block_num','category_id','shop_id'], 'new_items_in_cat')
 df = new_item_sales(df, ['date_block_num','category_id'], 'new_items_in_cat_all_shops' )
X_train['is_duplicate'] = X_train.duplicated()
X_train['is_duplicate'].value_counts()<drop_column>
def agg_price_col(df, merging_cols, new_col):
temp = df.groupby(merging_cols ).agg({'revenue':'sum','item_cnt_unclipped':'sum'} ).reset_index()
temp[new_col] = temp['revenue']/temp['item_cnt_unclipped']
temp = temp[merging_cols + [new_col]]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_price_col(df,['date_block_num','item_id'],'item_price')
 df = agg_price_col(df,['date_block_num','category_id'],'category_price')
 df = agg_price_col(df,['date_block_num'],'block_price' )
X_train = X_train.drop(['is_duplicate'], axis = 1 )<sort_values>
df = downcast(df )
X_train_sort = X_train.sort_values(by = ['series_id', 'measurement_number'], ascending = True)
 X_train_sort.head()<merge>
def lag_feature(df, lag, col, merge_cols):
temp = df[merge_cols + [col]]
temp = temp.groupby(merge_cols ).agg({f'{col}':'first'} ).reset_index()
temp.columns = merge_cols + [f'{col}_lag{lag}']
temp['date_block_num'] += lag
df = pd.merge(df, temp, on=merge_cols, how='left')
 df[f'{col}_lag{lag}'] = df[f'{col}_lag{lag}'].fillna(0 ).astype('float32')
 return df
df = X_train.merge(y_train, on = 'series_id', how = 'inner')
 targets =(y_train['surface'].value_counts() ).index<feature_engineering>
lag12_cols = {
'item_cnt':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops':['date_block_num', 'item_id'],
'category_cnt':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops':['date_block_num', 'category_id'],
'group_cnt':['date_block_num', 'shop_id', 'group_id'],
'group_cnt_all_shops':['date_block_num', 'group_id'],
'shop_cnt':['date_block_num', 'shop_id'],
'city_cnt':['date_block_num', 'shop_city'],
'new_items_in_cat':['date_block_num', 'shop_id', 'category_id'],
'new_items_in_cat_all_shops':['date_block_num', 'category_id']
}
for col,merge_cols in lag12_cols.items() :
df[f'{col}_lag1to12'] = 0
for i in range(1,13):
df = lag_feature(df, i, col, merge_cols)
 df[f'{col}_lag1to12'] += df[f'{col}_lag{i}']
if i > 2:
del df[f'{col}_lag{i}']
if col == 'item_cnt':
del df[f'{col}_lag1']
del df[f'{col}_lag2']
else:
del df[col]
series_dict = {}
for series in(X_train['series_id'].unique()):
series_dict[series] = X_train[X_train['series_id'] == series]<feature_engineering>
lag2_cols = {
'item_cnt_unclipped':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops_median':['date_block_num', 'item_id'],
'category_cnt_median':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops_median':['date_block_num', 'category_id']
}
for col in lag2_cols:
df = lag_feature(df, 1, col, merge_cols)
 df = lag_feature(df, 2, col, merge_cols)
 if col!='item_cnt_unclipped':
del df[col]
def filter_signal(signal, threshold=1e3):
fourier = rfft(signal)
 frequencies = rfftfreq(signal.size, d=20e-3/signal.size)
 fourier[frequencies > threshold] = 0
return irfft(fourier )<create_dataframe>
df['item_cnt_diff'] = df['item_cnt_unclipped_lag1']/df['item_cnt_lag1to12']
df['item_cnt_all_shops_diff'] = df['item_cnt_all_shops_lag1']/df['item_cnt_all_shops_lag1to12']
df['category_cnt_diff'] = df['category_cnt_lag1']/df['category_cnt_lag1to12']
df['category_cnt_all_shops_diff'] = df['category_cnt_all_shops_lag1']/df['category_cnt_all_shops_lag1to12']
X_train_denoised = X_train.copy()
X_test_denoised = X_test.copy()<groupby>
df = lag_feature(df, 1, 'category_price',['date_block_num', 'category_id'])
 df = lag_feature(df, 1, 'block_price',['date_block_num'])
 del df['category_price'], df['block_price']
for col in X_train.columns:
if col[0:3] == 'ang' or col[0:3] == 'lin':
denoised_data = X_train.groupby(['series_id'])[col].apply(lambda x: filter_signal(x))
list_denoised_data = []
for arr in denoised_data:
for val in arr:
list_denoised_data.append(val)
 X_train_denoised[col] = list_denoised_data
for col in X_test.columns:
if col[0:3] == 'ang' or col[0:3] == 'lin':
denoised_data = X_test.groupby(['series_id'])[col].apply(lambda x: filter_signal(x))
list_denoised_data = []
for arr in denoised_data:
for val in arr:
list_denoised_data.append(val)
 X_test_denoised[col] = list_denoised_data 
<feature_engineering>
df.loc[(df['item_age']>0)&(df['item_cnt_lag1to12'].isna()), 'item_cnt_lag1to12'] = 0
df.loc[(df['category_age']>0)&(df['category_cnt_lag1to12'].isna()), 'category_cnt_lag1to12'] = 0
df.loc[(df['group_age']>0)&(df['group_cnt_lag1to12'].isna()), 'group_cnt_lag1to12'] = 0
series_dict = {}
for series in(X_train_denoised['series_id'].unique()):
series_dict[series] = X_train_denoised[X_train_denoised['series_id'] == series]<normalization>
df['item_cnt_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],df['shop_age'],[12]*len(df)) ]
df['item_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],[12]*len(df)) ]
df['category_cnt_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['category_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
df['group_cnt_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],df['shop_age'],[12]*len(df)) ]
df['group_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],[12]*len(df)) ]
df['city_cnt_lag1to12'] /= [min(idx)for idx in zip(df['date_block_num'],[12]*len(df)) ]
df['shop_cnt_lag1to12'] /= [min(idx)for idx in zip(df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
def quaternion_to_euler(qx,qy,qz,qw):
sinr_cosp = +2.0 *(qw * qx + qy + qz)
 cosr_cosp = +1.0 - 2.0 *(qx * qx + qy * qy)
 roll = math.atan2(sinr_cosp, cosr_cosp)
 sinp = +2.0 *(qw * qy - qz * qx)
 if(math.fabs(sinp)>= 1):
pitch = copysign(M_PI/2, sinp)
 else:
pitch = math.asin(sinp)
 siny_cosp = +2.0 *(qw * qz + qx * qy)
 cosy_cosp = +1.0 - 2.0 *(qy * qy + qz * qz)
 yaw = math.atan2(siny_cosp, cosy_cosp)
 return roll, pitch, yaw<concatenate>
df = downcast(df )
def eular_angle(data):
x, y, z, w = data['orientation_X'].tolist() , data['orientation_Y'].tolist() , data['orientation_Z'].tolist() , data['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 data['euler_x'] = nx
data['euler_y'] = ny
data['euler_z'] = nz
return data<train_model>
def past_information(df, merging_cols, new_col, aggregation):
temp = []
for i in range(1,35):
block = df.query(f'date_block_num < {i}' ).groupby(merging_cols ).agg(aggregation ).reset_index()
block.columns = merging_cols + [new_col]
block['date_block_num'] = i
block = block[block[new_col]>0]
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=['date_block_num']+merging_cols, how='left')
 return df
df = past_information(df, ['item_id'],'last_item_price',{'item_price':'last'})
 df = past_information(df, ['shop_id','item_id'],'item_cnt_sum_alltime',{'item_cnt':'sum'})
 df = past_information(df, ['item_id'],'item_cnt_sum_alltime_allshops',{'item_cnt':'sum'})
 del df['revenue'], df['item_cnt_unclipped'], df['item_price']
data = eular_angle(X_train_denoised)
 test = eular_angle(X_test_denoised)
 print(data.shape, test.shape )<feature_engineering>
df['relative_price_item_block_lag1'] = df['last_item_price']/df['block_price_lag1']
def fe_eng1(data):
data['total_angular_vel'] =(data['angular_velocity_X']**2 + data['angular_velocity_Y']**2 + data['angular_velocity_Z']**2)** 0.5
data['total_linear_acc'] =(data['linear_acceleration_X']**2 + data['linear_acceleration_Y']**2 + data['linear_acceleration_Z']**2)**0.5
data['total_orientation'] =(data['orientation_X']**2 + data['orientation_Y']**2 + data['orientation_Z']**2)**0.5
data['acc_vs_vel'] = data['total_linear_acc'] / data['total_angular_vel']
data['total_angle'] =(data['euler_x'] ** 2 + data['euler_y'] ** 2 + data['euler_z'] ** 2)** 5
data['angle_vs_acc'] = data['total_angle'] / data['total_linear_acc']
data['angle_vs_vel'] = data['total_angle'] / data['total_angular_vel']
return data<create_dataframe>
df['item_cnt_per_day_alltime'] =(df['item_cnt_sum_alltime']/df['prev_days_on_sale'] ).fillna(0)
 df['item_cnt_per_day_alltime_allshops'] =(df['item_cnt_sum_alltime_allshops']/df['prev_days_on_sale'] ).fillna(0 )
def fe_eng2(data):
df = pd.DataFrame()
for col in data.columns:
if col in ['row_id','series_id','measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_median'] = data.groupby(['series_id'])[col].median()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_range'] = df[col + '_max'] - df[col + '_min']
df[col + '_maxtoMin'] = df[col + '_max'] / df[col + '_min']
df[col + '_mad'] = data.groupby(['series_id'])[col].apply(lambda x: np.median(np.abs(np.diff(x))))
df[col + '_abs_max'] = data.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 df[col + '_abs_avg'] =(df[col + '_abs_min'] + df[col + '_abs_max'])/2
return df<categorify>
gc.collect()
df = downcast(df )
data.fillna(0, inplace = True)
 data.replace(-np.inf, 0, inplace = True)
 data.replace(np.inf, 0, inplace = True)
 test.fillna(0, inplace = True)
 test.replace(-np.inf, 0, inplace = True)
 test.replace(np.inf, 0, inplace = True )<categorify>
def matching_name_cat_age(df,n,all_shops):
temp_cols = [f'same_name{n}catage_cnt','date_block_num', f'item_name_first{n}','item_age','category_id']
if all_shops:
temp_cols[0] += '_all_shops'
else:
temp_cols += ['shop_id']
temp = []
for i in range(1,35):
block =(
 df 
.query(f'date_block_num < {i}')
.groupby(temp_cols[2:])
.agg({'item_cnt':'mean'})
.reset_index() 
.rename(columns={'item_cnt':temp_cols[0]})
)
 block = block[block[temp_cols[0]]>0]
block['date_block_num'] = i
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=temp_cols[1:], how='left')
 return df
for n in [4,6,11]:
for all_shops in [True,False]:
df = matching_name_cat_age(df,n,all_shops )
le = LabelEncoder()
y_train['surface'] = le.fit_transform(y_train['surface'] )<define_variables>
df = downcast(df)
 int8_cols = [
'item_cnt','month','group_id','shop_type',
'shop_city','shop_id','date_block_num','category_id',
'item_age',
]
int16_cols = [
'item_id','item_name_first4',
'item_name_first6','item_name_first11'
]
for col in int8_cols:
df[col] = df[col].astype('int8')
 for col in int16_cols:
df[col] = df[col].astype('int16' )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=60)
 predicted = np.zeros(( test.shape[0],9))
measured= np.zeros(( data.shape[0]))
score = 0<predict_on_test>
def nearby_item_data(df,col):
if col in ['item_cnt_unclipped_lag1','item_cnt_lag1to12']:
cols = ['date_block_num', 'shop_id', 'item_id']
temp = df[cols + [col]]
else:
cols = ['date_block_num', 'item_id']
temp = df.groupby(cols ).agg({col:'first'} ).reset_index() [cols + [col]]
temp.columns = cols + [f'below_{col}']
temp['item_id'] += 1
df = pd.merge(df, temp, on=cols, how='left')
 temp.columns = cols + [f'above_{col}']
temp['item_id'] -= 2
df = pd.merge(df, temp, on=cols, how='left')
 return df
item_cols = ['item_cnt_unclipped_lag1','item_cnt_lag1to12',
'item_cnt_all_shops_lag1','item_cnt_all_shops_lag1to12']
for col in item_cols:
df = nearby_item_data(df,col)
 del temp
for times,(trn_idx, val_idx)in enumerate(folds.split(data.values,y_train['surface'].values)) :
model = RandomForestClassifier(n_estimators=700, n_jobs = -1)
 model.fit(data.iloc[trn_idx],y_train['surface'][trn_idx])
 measured[val_idx] = model.predict(data.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(data.iloc[val_idx],y_train['surface'][val_idx])
 print("Fold: {} score: {}".format(times,model.score(data.iloc[val_idx],y_train['surface'][val_idx])))
 gc.collect()<compute_test_metric>
results = Counter()
items['item_name'].str.split().apply(results.update)
 words = []
cnts = []
for key, value in results.items() :
words.append(key)
 cnts.append(value)
 counts = pd.DataFrame({'word':words,'count':cnts})
 common_words = counts.query('count>200' ).word.to_list()
for word in common_words:
items[f'{word}_in_name'] = items['item_name'].str.contains(word ).astype('int8')
 drop_cols = [
'item_id','category_id','item_name','item_name_first4',
'item_name_first6','item_name_first11',
'category_name','group_name','group_id'
]
items = items.drop(columns=drop_cols )
print('Average score', score / folds.n_splits )<compute_test_metric>
df = df.join(items, on='item_id' )
confusion_matrix(measured,y_train['surface'] )<filter>
def binary_encode(df, letters, cols):
encoder = ce.BinaryEncoder(cols=[f'item_name_first{letters}'], return_df=True)
 temp = encoder.fit_transform(df[f'item_name_first{letters}'])
 df = pd.concat([df,temp], axis=1)
 del df[f'item_name_first{letters}_0']
name_cols = [f'item_name_first{letters}_{x}' for x in range(1,cols)]
df[name_cols] = df[name_cols].astype('int8')
 return df
df = binary_encode(df, 11, 15)
 del df['item_name_first4'], df['item_name_first6']
less_important_features = feature_importances.loc[feature_importances['importance'] < 0.0025]
print('There are {0} features their importance value is less then 0.0025'.format(less_important_features.shape[0]))<drop_column>
df.to_pickle('df_complete.pkl' )
for i, col in enumerate(less_important_features.index):
data = data.drop(columns = [col], axis = 1)
 test = test.drop(columns = [col], axis = 1)
 data.shape, test.shape<find_best_model_class>
%reset -f
predicted = np.zeros(( test.shape[0],9))
measured= np.zeros(( data.shape[0]))
score = 0
for times,(trn_idx, val_idx)in enumerate(folds.split(data.values,y_train['surface'].values)) :
model = RandomForestClassifier(n_estimators=700, n_jobs = -1)
 model.fit(data.iloc[trn_idx],y_train['surface'][trn_idx])
 measured[val_idx] = model.predict(data.iloc[val_idx])
 predicted += model.predict_proba(test)/folds.n_splits
score += model.score(data.iloc[val_idx],y_train['surface'][val_idx])
 print("Fold: {} score: {}".format(times,model.score(data.iloc[val_idx],y_train['surface'][val_idx])))
 gc.collect()<compute_test_metric>
pd.set_option('display.max_rows', 160)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 30)
 warnings.filterwarnings("ignore" )
print('Average score', score / folds.n_splits )<save_to_csv>
df = pd.read_pickle('.. /input/feature-engineering-lightgbm-exploring-performance/df_complete.pkl')
 X_train = df[~df.date_block_num.isin([0,1,33,34])]
y_train = X_train['item_cnt']
del X_train['item_cnt']
X_val = df[df['date_block_num']==33]
y_val = X_val['item_cnt']
del X_val['item_cnt']
X_test = df[df['date_block_num']==34].drop(columns='item_cnt')
 X_test = X_test.reset_index()
del X_test['index']
del df
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['surface'] = le.inverse_transform(predicted.argmax(axis=1))
submission.to_csv('rs_surface_submission6.csv', index=False)
 submission.head(10 )<set_options>
def build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features):
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_val = lgb.Dataset(X_val, y_val)
 model = lgb.train(params=params, train_set=lgb_train, valid_sets=(lgb_train, lgb_val), verbose_eval=50,
categorical_feature=cat_features)
 return model
def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn<load_from_csv>
params = {
'objective': 'rmse',
'metric': 'rmse',
'num_leaves': 1023,
'min_data_in_leaf':10,
'feature_fraction':0.7,
'learning_rate': 0.01,
'num_rounds': 1000,
'early_stopping_rounds': 30,
'seed': 1
}
cat_features = ['category_id','month','shop_id','shop_city']
lgb_model = build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features)
x = pd.read_csv('.. /input/X_train.csv')
 y = pd.read_csv('.. /input/y_train.csv')
 test = pd.read_csv('.. /input/X_test.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv' )<count_unique_values>
submission = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv')
 submission['item_cnt_month'] = lgb_model.predict(X_test ).clip(0,20)
 submission[['ID', 'item_cnt_month']].to_csv('initial_lgb_submission.csv', index=False )
x['series_id'].nunique()<filter>
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
for i in [r'[^\w\d\s]', r'\bthe\b', r'\bin\b', r'\bfor\b', r'\bof\b', r'\bd\b', r'\bis\b', r'\bon\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(' ', '')
 items = items.join(categories.set_index('category_id'), on='category_id')
 shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
x[x.isnull().any(axis=1)]<filter>
X_train['lgb_pred'] = lgb_model.predict(X_train ).clip(0,20)
 X_train['target'] = y_train
X_train['sq_err'] =(X_train['lgb_pred']-X_train['target'])**2
X_val['lgb_pred'] = lgb_model.predict(X_val ).clip(0,20)
 X_val['target'] = y_val
X_val['sq_err'] =(X_val['lgb_pred']-X_val['target'])**2
X_test['lgb_pred'] = lgb_model.predict(X_test ).clip(0,20 )
y[y.isnull().any(axis=1)]<filter>
data = X_train.groupby('date_block_num' ).agg({'lgb_pred':'mean','target':'mean','sq_err':'mean'} ).reset_index()
data['new_item_rmse'] = np.sqrt(X_train.query('item_age<=1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data['old_item_rmse'] = np.sqrt(X_train.query('item_age>1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data = data.append([
{'date_block_num':33,
'target':X_val.target.mean() ,
'lgb_pred':X_val.lgb_pred.mean() ,
'sq_err':np.sqrt(X_val.sq_err.mean()),
'old_item_rmse':np.sqrt(X_val.query('item_age>1' ).sq_err.mean()),
'new_item_rmse':np.sqrt(X_val.query('item_age<=1' ).sq_err.mean())},
{'date_block_num':34,
'target':0,
'lgb_pred':X_test.lgb_pred.mean() ,
'sq_err':0,
'old_item_rmse':0,
'new_item_rmse':0}
],
ignore_index=True 
)
 data['date'] = [x[:7] for x in pd.date_range(start='2013-03',end='2015-09',freq='MS' ).astype('str')]+['Validation','Test']
test[test.isnull().any(axis=1)]<categorify>
df = pd.read_pickle('.. /input/feature-engineering-lightgbm-exploring-performance/df_complete.pkl')
(
 df
[df['category_id'].isin(X_test.category_id.unique())] 
.query('item_cnt>0')
.groupby('category_id')
.agg({
'category_age':'max',
'shop_id':['nunique','unique'],
'item_cnt':'sum'
})
.join(categories['category_name'])
.join(
 X_test 
.groupby('category_id')
.agg({'item_id':'nunique'})
.rename(columns={'item_id':'test_set_items'})
)
.sort_values(( 'shop_id', 'nunique')) 
.head(20)
 )
le = LabelEncoder()
le.fit(y['surface'])
 y['surface'] = le.transform(y['surface'] )<prepare_x_and_y>
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('shop_id')
.agg({
'sq_err':'mean',
'target':'mean',
'lgb_pred':'mean'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('shop_id')
.agg({'test_pred':'mean'})
)
.join(shops['shop_name'])
.sort_values('sq_err', ascending=False)
.head(20)
 )
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def fe(df):
df['total_angular_velocity'] =(df['angular_velocity_X'] ** 2 + df['angular_velocity_Y'] ** 2 + df['angular_velocity_Z'] ** 2)** 0.5
df['total_linear_acceleration'] =(df['linear_acceleration_X'] ** 2 + df['linear_acceleration_Y'] ** 2 + df['linear_acceleration_Z'] ** 2)** 0.5
df['total_xyz'] =(df['orientation_X']**2 + df['orientation_Y']**2 +df['orientation_Z'])**0.5
df['acc_vs_vel'] = df['total_linear_acceleration'] / df['total_angular_velocity']
temp_df = pd.DataFrame()
for col in df.columns[3:]:
temp_df[col + '_mean'] = df.groupby(['series_id'])[col].mean()
temp_df[col + '_median'] = df.groupby(['series_id'])[col].median()
temp_df[col + '_max'] = df.groupby(['series_id'])[col].max()
temp_df[col + '_min'] = df.groupby(['series_id'])[col].min()
temp_df[col + '_std'] = df.groupby(['series_id'])[col].std()
temp_df[col + '_range'] = temp_df[col + '_max'] - temp_df[col + '_min']
temp_df[col + '_maxtoMin'] = temp_df[col + '_max'] / temp_df[col + '_min']
temp_df[col + '_mean_abs_chg'] = df.groupby(['series_id'])[col].apply(lambda x: np.mean(np.abs(np.diff(x))))
temp_df[col + '_abs_min'] = df.groupby(['series_id'])[col].apply(lambda x: np.min(np.abs(x)))
 temp_df[col + '_abs_max'] = df.groupby(['series_id'])[col].apply(lambda x: np.max(np.abs(x)))
 temp_df[col + '_abs_avg'] =(temp_df[col + '_abs_min'] + temp_df[col + '_abs_max'])/2
return temp_df<feature_engineering>
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('category_id')
.agg({
'sq_err':['sum','mean'],
'target':'mean',
'lgb_pred':['sum','mean'],
'item_id':'nunique'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred','item_id':'test_items'})
.groupby('category_id')
.agg({
'test_pred':['sum','mean'],
'test_items':'nunique'
}),
on='category_id' 
)
.join(categories)
.sort_values(( 'sq_err', 'mean'), ascending=False)
.head(20)
 )
train=fe(x)
 train.head()<drop_column>
CATEGORY = 20 
(
 items[
items['item_id'].isin(X_val.item_id.unique())&
items['item_id'].isin(X_test.item_id.unique())
 ]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('item_id')
.agg({'test_pred':'mean'}),
on='item_id' 
)
.join(
 X_val 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'target':'mean',
'sq_err':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={
'lgb_pred':'val_pred',
'target':'val_target',
}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('sq_err',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )
test_df=fe(test)
 test_df.head()<categorify>
CATEGORY = 20 
(
 items[items['item_id'].isin(X_test.item_id.unique())]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={'lgb_pred':'test_pred'}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('test_pred',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'item_category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )