Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<choose_model_class>
le = LabelEncoder()
pd.set_option('display.max_rows', 400)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 40)
 warnings.filterwarnings("ignore" )
model = lgb.LGBMClassifier()
default_params = model.get_params()
del default_params['n_estimators']
cv_results = lgb.cv(default_params, train_set, num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS, seed = 42 )<train_model>
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 test.head()
print('The maximum validation ROC AUC was: {:.5f} with a standard deviation of {:.5f}.'.format(cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('The optimal number of boosting rounds(estimators)was {}.'.format(len(cv_results['auc-mean'])) )<import_modules>
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 pd.DataFrame(categories.category_name.values.reshape(-1, 4))
from sklearn.metrics import roc_auc_score<compute_train_metric>
categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
categories['group_id'] = le.fit_transform(categories.group_name.values)
 categories.sample(5 )
model.n_estimators = len(cv_results['auc-mean'])
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
baseline_auc = roc_auc_score(test_labels, preds)
 print('The baseline model scores {:.5f} ROC AUC on the test set.'.format(baseline_auc))<compute_train_metric>
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
items['item_name'] = items['item_name'].str.replace('.', '')
 for i in [r'[^\w\d\s\.]', r'\bthe\b', r'\bin\b', r'\bis\b',
r'\bfor\b', r'\bof\b', r'\bon\b', r'\band\b',
r'\bto\b', r'\bwith\b' , r'\byo\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(r'\b.\b', ' ')
 items['item_name_no_space'] = items['item_name'].str.replace(' ', '')
 items['item_name_first4'] = [x[:4] for x in items['item_name_no_space']]
items['item_name_first6'] = [x[:6] for x in items['item_name_no_space']]
items['item_name_first11'] = [x[:11] for x in items['item_name_no_space']]
del items['item_name_no_space']
items.item_name_first4 = le.fit_transform(items.item_name_first4.values)
 items.item_name_first6 = le.fit_transform(items.item_name_first6.values)
 items.item_name_first11 = le.fit_transform(items.item_name_first11.values)
 items = items.join(categories.set_index('category_id'), on='category_id')
 items.sample(10 )
def objective(hyperparameters, iteration):

 if 'n_estimators' in hyperparameters.keys() :
del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 10000, nfold = N_FOLDS,
early_stopping_rounds = 100, metrics = 'auc', seed = 42)
 score = cv_results['auc-mean'][-1]
estimators = len(cv_results['auc-mean'])
 hyperparameters['n_estimators'] = estimators
return [score, hyperparameters, iteration]<compute_test_metric>
dupes = items[(items.duplicated(subset=['item_name','category_id'],keep=False)) ]
dupes['in_test'] = dupes.item_id.isin(test.item_id.unique())
 dupes = dupes.groupby('item_name' ).agg({'item_id':['first','last'],'in_test':['first','last']})
 dupes = dupes[(dupes[('in_test', 'first')]==False)|(dupes[('in_test', 'last')]==False)]
temp = dupes[dupes[('in_test', 'first')]==True]
keep_first = dict(zip(temp[('item_id', 'last')], temp[('item_id', 'first')]))
temp = dupes[dupes[('in_test', 'first')]==False]
keep_second = dict(zip(temp[('item_id', 'first')], temp[('item_id', 'last')]))
item_map = {**keep_first, **keep_second}
score, params, iteration = objective(default_params, 1)
 print('The cross-validation ROC AUC was {:.5f}.'.format(score))<choose_model_class>
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 sales =(sales 
.query('0 < item_price < 50000 and 0 < item_cnt_day < 1001')
.replace({
'shop_id':{0:57, 1:58, 11:10},
'item_id':item_map
})
)
 sales = sales[sales['shop_id'].isin(test.shop_id.unique())]
sales['date'] = pd.to_datetime(sales.date,format='%d.%m.%Y')
 sales['weekday'] = sales.date.dt.dayofweek
sales['first_sale_day'] = sales.date.dt.dayofyear
sales['first_sale_day'] += 365 *(sales.date.dt.year-2013)
 sales['first_sale_day'] = sales.groupby('item_id')['first_sale_day'].transform('min' ).astype('int16')
 sales['revenue'] = sales['item_cnt_day']*sales['item_price']
model = lgb.LGBMModel()
model.get_params()<define_search_space>
temp = sales.groupby(['shop_id','weekday'] ).agg({'item_cnt_day':'sum'} ).reset_index()
temp = pd.merge(temp, sales.groupby(['shop_id'] ).agg({'item_cnt_day':'sum'} ).reset_index() , on='shop_id', how='left')
 temp.columns = ['shop_id','weekday', 'shop_day_sales', 'shop_total_sales']
temp['day_quality'] = temp['shop_day_sales']/temp['shop_total_sales']
temp = temp[['shop_id','weekday','day_quality']]
dates = pd.DataFrame(data={'date':pd.date_range(start='2013-01-01',end='2015-11-30')})
 dates['weekday'] = dates.date.dt.dayofweek
dates['month'] = dates.date.dt.month
dates['year'] = dates.date.dt.year - 2013
dates['date_block_num'] = dates['year']*12 + dates['month'] - 1
dates['first_day_of_month'] = dates.date.dt.dayofyear
dates['first_day_of_month'] += 365 * dates['year']
dates = dates.join(temp.set_index('weekday'), on='weekday')
 dates = dates.groupby(['date_block_num','shop_id','month','year'] ).agg({'day_quality':'sum','first_day_of_month':'min'} ).reset_index()
dates.query('shop_id == 28' ).head(15 )
param_grid = {
'boosting_type': ['gbdt', 'goss', 'dart'],
'num_leaves': list(range(20, 150)) ,
'learning_rate': list(np.logspace(np.log10(0.005), np.log10(0.5), base = 10, num = 1000)) ,
'subsample_for_bin': list(range(20000, 300000, 20000)) ,
'min_child_samples': list(range(20, 500, 5)) ,
'reg_alpha': list(np.linspace(0, 1)) ,
'reg_lambda': list(np.linspace(0, 1)) ,
'colsample_bytree': list(np.linspace(0.6, 1, 10)) ,
'subsample': list(np.linspace(0.5, 1, 100)) ,
'is_unbalance': [True, False]
}<define_variables>
sales =(sales 
.groupby(['date_block_num', 'shop_id', 'item_id'])
.agg({
'item_cnt_day':'sum',
'revenue':'sum',
'first_sale_day':'first'
})
.reset_index() 
.rename(columns={'item_cnt_day':'item_cnt'})
)
 sales.sample(5 )
a = 0
b = 0
for x in param_grid['learning_rate']:
if x >= 0.005 and x < 0.05:
a += 1
elif x >= 0.05 and x < 0.5:
b += 1
print('There are {} values between 0.005 and 0.05'.format(a))
print('There are {} values between 0.05 and 0.5'.format(b))<create_dataframe>
test['date_block_num'] = 34
del test['ID']
random_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 grid_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)) )<count_unique_values>
df = pd.concat([df,test] ).fillna(0)
 df = df.reset_index()
del df['index']
com = 1
for x in param_grid.values() :
com *= len(x)
 print('There are {} combinations'.format(com))<train_model>
df = pd.merge(df, sales, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 df = pd.merge(df, dates, on=['date_block_num','shop_id'], how='left')
 df = pd.merge(df, items.drop(columns=['item_name','group_name','category_name']), on='item_id', how='left' )
print('This would take {:.0f} years to finish.'.format(( 100 * com)/(60 * 60 * 24 * 365)) )<train_on_grid>
shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops_cats = pd.DataFrame(
 np.array(list(product(*[df['shop_id'].unique() , df['category_id'].unique() ]))),
columns =['shop_id', 'category_id'] 
)
 temp = df.groupby(['category_id', 'shop_id'] ).agg({'item_cnt':'sum'} ).reset_index()
temp2 = temp.groupby('shop_id' ).agg({'item_cnt':'sum'} ).rename(columns={'item_cnt':'shop_total'})
 temp = temp.join(temp2, on='shop_id')
 temp['category_proportion'] = temp['item_cnt']/temp['shop_total']
temp = temp[['shop_id', 'category_id', 'category_proportion']]
shops_cats = pd.merge(shops_cats, temp, on=['shop_id','category_id'], how='left')
 shops_cats = shops_cats.fillna(0)
 shops_cats = shops_cats.pivot(index='shop_id', columns=['category_id'])
 kmeans = KMeans(n_clusters=7, random_state=0 ).fit(shops_cats)
 shops_cats['shop_cluster'] = kmeans.labels_.astype('int8')
 shops = shops.join(shops_cats['shop_cluster'], on='shop_id' )
grid_results = grid_search(param_grid)
 print('The best validation score was {:.5f}'.format(grid_results.loc[0, 'score']))
print('
The best hyperparameters were:')
 pprint.pprint(grid_results.loc[0, 'params'] )<train_on_grid>
shops.dropna(inplace=True)
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_type'] = 'regular'
shops.loc[shops['shop_name'].str.contains(r'tc'), 'shop_type'] = 'tc'
shops.loc[shops['shop_name'].str.contains(r'mall|center|mega'), 'shop_type'] = 'mall'
shops.loc[shops['shop_id'].isin([9,20]), 'shop_type'] = 'special'
shops.loc[shops['shop_id'].isin([12,55]), 'shop_type'] = 'online'
shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
shops.shop_city = le.fit_transform(shops.shop_city.values)
 shops.shop_type = le.fit_transform(shops.shop_type.values)
 shops.head()
grid_search_params = grid_results.loc[0, 'params']
model = lgb.LGBMClassifier(**grid_search_params, random_state=42)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('The best model from grid search scores {:.5f} ROC AUC on the test set.'.format(roc_auc_score(test_labels, preds)) )<set_options>
df = pd.merge(df, shops.drop(columns='shop_name'), on='shop_id', how='left')
 df.head()
pd.options.display.max_colwidth = 1000
grid_results['params'].values<init_hyperparams>
df['first_sale_day'] = df.groupby('item_id')['first_sale_day'].transform('max' ).astype('int16')
 df.loc[df['first_sale_day']==0, 'first_sale_day'] = 1035
df['prev_days_on_sale'] = [max(idx)for idx in zip(df['first_day_of_month']-df['first_sale_day'],[0]*len(df)) ]
del df['first_day_of_month']
random.seed(50)
 random_params = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
random_params['subsample'] = 1.0 if random_params['boosting_type'] == 'goss' else random_params['subsample']
random_params<create_dataframe>
del sales, categories, shops, shops_cats, temp, temp2, test, dupes, item_map,
df.head()
def random_search(param_grid, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 for i in range(MAX_EVALS):
hyperparameters = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
hyperparameters['subsample'] = 1.0 if hyperparameters['boosting_type'] == 'goss' else hyperparameters['subsample']
eval_results = objective(hyperparameters, i)
 results.loc[i, :] = eval_results
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results<train_model>
df['item_cnt_unclipped'] = df['item_cnt']
df['item_cnt'] = df['item_cnt'].clip(0, 20 )
random_results = random_search(param_grid)
 print('The best validation score was {:.5f}'.format(random_results.loc[0, 'score']))
print('
The best hyperparameters were:')
 pprint.pprint(random_results.loc[0, 'params'] )<find_best_model_class>
def downcast(df):
float_cols = [c for c in df if df[c].dtype in ["float64"]]
int_cols = [c for c in df if df[c].dtype in ['int64']]
df[float_cols] = df[float_cols].astype('float32')
 df[int_cols] = df[int_cols].astype('int16')
 return df
df = downcast(df )
random_search_params = random_results.loc[0, 'params']
model = lgb.LGBMClassifier(**random_search_params, random_state = 42)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('The best model from random search scores {:.5f} ROC AUC on the test set.'.format(roc_auc_score(test_labels, preds)) )<save_to_csv>
df['item_age'] =(df['date_block_num'] - df.groupby('item_id')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first4_age'] =(df['date_block_num'] - df.groupby('item_name_first4')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first6_age'] =(df['date_block_num'] - df.groupby('item_name_first6')['date_block_num'].transform('min')).astype('int8')
 df['item_name_first11_age'] =(df['date_block_num'] - df.groupby('item_name_first11')['date_block_num'].transform('min')).astype('int8')
 df['category_age'] =(df['date_block_num'] - df.groupby('category_id')['date_block_num'].transform('min')).astype('int8')
 df['group_age'] =(df['date_block_num'] - df.groupby('group_id')['date_block_num'].transform('min')).astype('int8')
 df['shop_age'] =(df['date_block_num'] - df.groupby('shop_id')['date_block_num'].transform('min')).astype('int8' )
out_file = 'random_search_trials.csv'
of_connection = open(out_file, 'w')
 writer = csv.writer(of_connection)
 headers = ['score', 'hyperparameters', 'iteration']
writer.writerow(headers)
 of_connection.close()<save_to_csv>
temp = df.query('item_cnt > 0' ).groupby(['item_id','shop_id'] ).agg({'date_block_num':'min'} ).reset_index()
temp.columns = ['item_id', 'shop_id', 'item_shop_first_sale']
df = pd.merge(df, temp, on=['item_id','shop_id'], how='left')
 df['item_shop_first_sale'] = df['item_shop_first_sale'].fillna(50)
 df['item_age_if_shop_sale'] =(df['date_block_num'] > df['item_shop_first_sale'])* df['item_age']
df['item_age_without_shop_sale'] =(df['date_block_num'] <= df['item_shop_first_sale'])* df['item_age']
del df['item_shop_first_sale']
def random_search(param_grid, out_file, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 for i in range(MAX_EVALS):
random_params = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
random_params['subsample'] = 1.0 if random_params['boosting_type'] == 'goss' else random_params['subsample']
eval_results = objective(random_params, i)
 results.loc[i, :] = eval_results
of_connection = open(out_file, 'a')
 writer = csv.writer(of_connection)
 writer.writerow(eval_results)
 of_connection.close()
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results<save_to_csv>
def agg_cnt_col(df, merging_cols, new_col,aggregation):
temp = df.groupby(merging_cols ).agg(aggregation ).reset_index()
temp.columns = merging_cols + [new_col]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_cnt_col(df, ['date_block_num','item_id'],'item_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'item_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id','shop_id'],'category_cnt_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','category_id'],'category_cnt_all_shops_median',{'item_cnt':'median'})
 df = agg_cnt_col(df, ['date_block_num','group_id','shop_id'],'group_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','group_id'],'group_cnt_all_shops',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_id'],'shop_cnt',{'item_cnt':'mean'})
 df = agg_cnt_col(df, ['date_block_num','shop_city'],'city_cnt',{'item_cnt':'mean'} )
def grid_search(param_grid, out_file, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 keys, values = zip(*param_grid.items())
 i = 0
for v in itertools.product(*values):
parameters = dict(zip(keys, v))
parameters['subsample'] = 1.0 if parameters['boosting_type'] == 'goss' else parameters['subsample']
eval_results = objective(parameters, i)
 results.loc[i, :] = eval_results
i += 1
of_connection = open(out_file, 'a')
 writer = csv.writer(of_connection)
 writer.writerow(eval_results)
 of_connection.close()
if i > MAX_EVALS:
break
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results<load_from_csv>
def new_item_sales(df, merging_cols, new_col):
temp =(
 df 
.query('item_age==0')
.groupby(merging_cols)['item_cnt'] 
.mean() 
.reset_index() 
.rename(columns={'item_cnt': new_col})
)
 df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = new_item_sales(df, ['date_block_num','category_id','shop_id'], 'new_items_in_cat')
 df = new_item_sales(df, ['date_block_num','category_id'], 'new_items_in_cat_all_shops' )
random_results = pd.read_csv('.. /input/home-credit-model-tuning/random_search_trials_1000.csv')
 grid_results = pd.read_csv('.. /input/home-credit-model-tuning/grid_search_trials_1000.csv' )<categorify>
def agg_price_col(df, merging_cols, new_col):
temp = df.groupby(merging_cols ).agg({'revenue':'sum','item_cnt_unclipped':'sum'} ).reset_index()
temp[new_col] = temp['revenue']/temp['item_cnt_unclipped']
temp = temp[merging_cols + [new_col]]
df = pd.merge(df, temp, on=merging_cols, how='left')
 return df
df = agg_price_col(df,['date_block_num','item_id'],'item_price')
 df = agg_price_col(df,['date_block_num','category_id'],'category_price')
 df = agg_price_col(df,['date_block_num'],'block_price' )
grid_results['hyperparameters'] = grid_results['hyperparameters'].map(ast.literal_eval)
 random_results['hyperparameters'] = random_results['hyperparameters'].map(ast.literal_eval )<choose_model_class>
df = downcast(df )
def evaluate(results, name):

 results = results.sort_values('score', ascending = False ).reset_index(drop = True)
 print('The highest cross validation score from {} was {:.5f} found on iteration {}.'.format(name, results.loc[0, 'score'], results.loc[0, 'iteration']))
hyperparameters = results.loc[0, 'hyperparameters']
model = lgb.LGBMClassifier(**hyperparameters)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('ROC AUC from {} on test data = {:.5f}.'.format(name, roc_auc_score(test_labels, preds)))
 hyp_df = pd.DataFrame(columns = list(results.loc[0, 'hyperparameters'].keys()))
for i, hyp in enumerate(results['hyperparameters']):
hyp_df = hyp_df.append(pd.DataFrame(hyp, index = [0]),
ignore_index = True)
 hyp_df['iteration'] = results['iteration']
hyp_df['score'] = results['score']
return hyp_df<find_best_params>
def lag_feature(df, lag, col, merge_cols):
temp = df[merge_cols + [col]]
temp = temp.groupby(merge_cols ).agg({f'{col}':'first'} ).reset_index()
temp.columns = merge_cols + [f'{col}_lag{lag}']
temp['date_block_num'] += lag
df = pd.merge(df, temp, on=merge_cols, how='left')
 df[f'{col}_lag{lag}'] = df[f'{col}_lag{lag}'].fillna(0 ).astype('float32')
 return df
grid_hyp = evaluate(grid_results, name = 'grid search' )<find_best_params>
lag12_cols = {
'item_cnt':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops':['date_block_num', 'item_id'],
'category_cnt':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops':['date_block_num', 'category_id'],
'group_cnt':['date_block_num', 'shop_id', 'group_id'],
'group_cnt_all_shops':['date_block_num', 'group_id'],
'shop_cnt':['date_block_num', 'shop_id'],
'city_cnt':['date_block_num', 'shop_city'],
'new_items_in_cat':['date_block_num', 'shop_id', 'category_id'],
'new_items_in_cat_all_shops':['date_block_num', 'category_id']
}
for col,merge_cols in lag12_cols.items() :
df[f'{col}_lag1to12'] = 0
for i in range(1,13):
df = lag_feature(df, i, col, merge_cols)
 df[f'{col}_lag1to12'] += df[f'{col}_lag{i}']
if i > 2:
del df[f'{col}_lag{i}']
if col == 'item_cnt':
del df[f'{col}_lag1']
del df[f'{col}_lag2']
else:
del df[col]
random_hyp = evaluate(random_results, name = 'random search' )<set_options>
lag2_cols = {
'item_cnt_unclipped':['date_block_num', 'shop_id', 'item_id'],
'item_cnt_all_shops_median':['date_block_num', 'item_id'],
'category_cnt_median':['date_block_num', 'shop_id', 'category_id'],
'category_cnt_all_shops_median':['date_block_num', 'category_id']
}
for col in lag2_cols:
df = lag_feature(df, 1, col, merge_cols)
 df = lag_feature(df, 2, col, merge_cols)
 if col!='item_cnt_unclipped':
del df[col]
alt.renderers.enable('notebook' )<prepare_output>
df['item_cnt_diff'] = df['item_cnt_unclipped_lag1']/df['item_cnt_lag1to12']
df['item_cnt_all_shops_diff'] = df['item_cnt_all_shops_lag1']/df['item_cnt_all_shops_lag1to12']
df['category_cnt_diff'] = df['category_cnt_lag1']/df['category_cnt_lag1to12']
df['category_cnt_all_shops_diff'] = df['category_cnt_all_shops_lag1']/df['category_cnt_all_shops_lag1to12']
random_hyp['search'] = 'random'
grid_hyp['search'] = 'grid'
hyp = random_hyp.append(grid_hyp)
 hyp.head()<find_best_params>
df = lag_feature(df, 1, 'category_price',['date_block_num', 'category_id'])
 df = lag_feature(df, 1, 'block_price',['date_block_num'])
 del df['category_price'], df['block_price']
best_grid_hyp = grid_hyp.iloc[grid_hyp['score'].idxmax() ].copy()
best_random_hyp = random_hyp.iloc[random_hyp['score'].idxmax() ].copy()<compute_test_metric>
df.loc[(df['item_age']>0)&(df['item_cnt_lag1to12'].isna()), 'item_cnt_lag1to12'] = 0
df.loc[(df['category_age']>0)&(df['category_cnt_lag1to12'].isna()), 'category_cnt_lag1to12'] = 0
df.loc[(df['group_age']>0)&(df['group_cnt_lag1to12'].isna()), 'group_cnt_lag1to12'] = 0
print('Average validation score of grid search = {:.5f}.'.format(np.mean(grid_hyp['score'])))
 print('Average validation score of random search = {:.5f}.'.format(np.mean(random_hyp['score'])) )<data_type_conversions>
df['item_cnt_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],df['shop_age'],[12]*len(df)) ]
df['item_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['item_age'],[12]*len(df)) ]
df['category_cnt_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['category_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
df['group_cnt_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],df['shop_age'],[12]*len(df)) ]
df['group_cnt_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['group_age'],[12]*len(df)) ]
df['city_cnt_lag1to12'] /= [min(idx)for idx in zip(df['date_block_num'],[12]*len(df)) ]
df['shop_cnt_lag1to12'] /= [min(idx)for idx in zip(df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],df['shop_age'],[12]*len(df)) ]
df['new_items_in_cat_all_shops_lag1to12'] /= [min(idx)for idx in zip(df['category_age'],[12]*len(df)) ]
random_hyp['score'] = random_hyp['score'].astype(float)
 best_random_hyp = random_hyp.loc[0, :].copy()<load_from_csv>
df = downcast(df )
train = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_train.csv')
 test = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_test.csv')
 test_ids = test['SK_ID_CURR']
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test = test.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )<train_on_grid>
def past_information(df, merging_cols, new_col, aggregation):
temp = []
for i in range(1,35):
block = df.query(f'date_block_num < {i}' ).groupby(merging_cols ).agg(aggregation ).reset_index()
block.columns = merging_cols + [new_col]
block['date_block_num'] = i
block = block[block[new_col]>0]
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=['date_block_num']+merging_cols, how='left')
 return df
df = past_information(df, ['item_id'],'last_item_price',{'item_price':'last'})
 df = past_information(df, ['shop_id','item_id'],'item_cnt_sum_alltime',{'item_cnt':'sum'})
 df = past_information(df, ['item_id'],'item_cnt_sum_alltime_allshops',{'item_cnt':'sum'})
 del df['revenue'], df['item_cnt_unclipped'], df['item_price']
train_set = lgb.Dataset(train, label = train_labels)
 hyperparameters = dict(**random_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set,
num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS )<compute_test_metric>
df['relative_price_item_block_lag1'] = df['last_item_price']/df['block_price_lag1']
print('The cross validation score on the full dataset = {:.5f} with std: {:.5f}.'.format(
 cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )<train_model>
df['item_cnt_per_day_alltime'] =(df['item_cnt_sum_alltime']/df['prev_days_on_sale'] ).fillna(0)
 df['item_cnt_per_day_alltime_allshops'] =(df['item_cnt_sum_alltime_allshops']/df['prev_days_on_sale'] ).fillna(0 )
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]<save_to_csv>
gc.collect()
df = downcast(df )
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_simple_features_random.csv', index = False )<set_options>
def matching_name_cat_age(df,n,all_shops):
temp_cols = [f'same_name{n}catage_cnt','date_block_num', f'item_name_first{n}','item_age','category_id']
if all_shops:
temp_cols[0] += '_all_shops'
else:
temp_cols += ['shop_id']
temp = []
for i in range(1,35):
block =(
 df 
.query(f'date_block_num < {i}')
.groupby(temp_cols[2:])
.agg({'item_cnt':'mean'})
.reset_index() 
.rename(columns={'item_cnt':temp_cols[0]})
)
 block = block[block[temp_cols[0]]>0]
block['date_block_num'] = i
temp.append(block)
 temp = pd.concat(temp)
 df = pd.merge(df, temp, on=temp_cols[1:], how='left')
 return df
for n in [4,6,11]:
for all_shops in [True,False]:
df = matching_name_cat_age(df,n,all_shops )
%matplotlib inline<import_modules>
df = downcast(df)
 int8_cols = [
'item_cnt','month','group_id','shop_type',
'shop_city','shop_id','date_block_num','category_id',
'item_age',
]
int16_cols = [
'item_id','item_name_first4',
'item_name_first6','item_name_first11'
]
for col in int8_cols:
df[col] = df[col].astype('int8')
 for col in int16_cols:
df[col] = df[col].astype('int16' )
from sklearn.model_selection import GridSearchCV
from skopt import BayesSearchCV<import_modules>
def nearby_item_data(df,col):
if col in ['item_cnt_unclipped_lag1','item_cnt_lag1to12']:
cols = ['date_block_num', 'shop_id', 'item_id']
temp = df[cols + [col]]
else:
cols = ['date_block_num', 'item_id']
temp = df.groupby(cols ).agg({col:'first'} ).reset_index() [cols + [col]]
temp.columns = cols + [f'below_{col}']
temp['item_id'] += 1
df = pd.merge(df, temp, on=cols, how='left')
 temp.columns = cols + [f'above_{col}']
temp['item_id'] -= 2
df = pd.merge(df, temp, on=cols, how='left')
 return df
item_cols = ['item_cnt_unclipped_lag1','item_cnt_lag1to12',
'item_cnt_all_shops_lag1','item_cnt_all_shops_lag1to12']
for col in item_cols:
df = nearby_item_data(df,col)
 del temp
import lightgbm as lgb<import_modules>
results = Counter()
items['item_name'].str.split().apply(results.update)
 words = []
cnts = []
for key, value in results.items() :
words.append(key)
 cnts.append(value)
 counts = pd.DataFrame({'word':words,'count':cnts})
 common_words = counts.query('count>200' ).word.to_list()
for word in common_words:
items[f'{word}_in_name'] = items['item_name'].str.contains(word ).astype('int8')
 drop_cols = [
'item_id','category_id','item_name','item_name_first4',
'item_name_first6','item_name_first11',
'category_name','group_name','group_id'
]
items = items.drop(columns=drop_cols )
import gc<import_modules>
df = df.join(items, on='item_id' )
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, MinMaxScaler<find_best_params>
def binary_encode(df, letters, cols):
encoder = ce.BinaryEncoder(cols=[f'item_name_first{letters}'], return_df=True)
 temp = encoder.fit_transform(df[f'item_name_first{letters}'])
 df = pd.concat([df,temp], axis=1)
 del df[f'item_name_first{letters}_0']
name_cols = [f'item_name_first{letters}_{x}' for x in range(1,cols)]
df[name_cols] = df[name_cols].astype('int8')
 return df
df = binary_encode(df, 11, 15)
 del df['item_name_first4'], df['item_name_first6']
def status_print(optim_result):

 all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)
 best_params = pd.Series(bayes_cv_tuner.best_params_)
 print('Model
Best ROC-AUC: {}
Best params: {}
'.format(
 len(all_models),
np.round(bayes_cv_tuner.best_score_, 4),
bayes_cv_tuner.best_params_ 
))
clf_name = bayes_cv_tuner.estimator.__class__.__name__
all_models.to_csv(clf_name+"_cv_results.csv" )<load_from_csv>
df.to_pickle('df_complete.pkl' )
data = pd.read_csv(".. /input/application_train.csv")
 test = pd.read_csv('.. /input/application_test.csv')
 prev = pd.read_csv('.. /input/previous_application.csv')
 buro = pd.read_csv('.. /input/bureau.csv')
 buro_balance = pd.read_csv('.. /input/bureau_balance.csv')
 credit_card = pd.read_csv('.. /input/credit_card_balance.csv')
 POS_CASH = pd.read_csv('.. /input/POS_CASH_balance.csv')
 payments = pd.read_csv('.. /input/installments_payments.csv')
 lgbm_submission = pd.read_csv('.. /input/sample_submission.csv' )<prepare_x_and_y>
%reset -f
y = data["TARGET"]
del data["TARGET"]<define_variables>
pd.set_option('display.max_rows', 160)
 pd.set_option('display.max_columns', 160)
 pd.set_option('display.max_colwidth', 30)
 warnings.filterwarnings("ignore" )
categorical_feature = [col for col in data.columns if data[col].dtype == 'object']<categorify>
df = pd.read_pickle('.. /input/files-top-scoring-notebook-output-exploration/df_complete.pkl')
 X_train = df[~df.date_block_num.isin([0,1,33,34])]
y_train = X_train['item_cnt']
del X_train['item_cnt']
X_val = df[df['date_block_num']==33]
y_val = X_val['item_cnt']
del X_val['item_cnt']
X_test = df[df['date_block_num']==34].drop(columns='item_cnt')
 X_test = X_test.reset_index()
del X_test['index']
del df
one_hot_df = pd.concat([data, test])
 one_hot_df = pd.get_dummies(one_hot_df, columns=categorical_feature )<split>
def build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features):
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_val = lgb.Dataset(X_val, y_val)
 model = lgb.train(params=params, train_set=lgb_train, valid_sets=(lgb_train, lgb_val), verbose_eval=50,
categorical_feature=cat_features)
 return model
data = one_hot_df.iloc[:data.shape[0],:]
test = one_hot_df.iloc[data.shape[0]:,]<groupby>
params = {
'objective': 'rmse',
'metric': 'rmse',
'num_leaves': 1023,
'min_data_in_leaf':10,
'feature_fraction':0.7,
'learning_rate': 0.01,
'num_rounds': 1000,
'early_stopping_rounds': 30,
'seed': 1
}
cat_features = ['category_id','month','shop_id','shop_city']
lgb_model = build_lgb_model(params, X_train, X_val, y_train, y_val, cat_features)
 lgb_model.save_model('initial_lgb_model.txt' )
buro_grouped_size = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].size()
buro_grouped_max = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].max()
buro_grouped_min = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].min()<groupby>
submission = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv')
 submission['item_cnt_month'] = lgb_model.predict(X_test ).clip(0,20)
 submission[['ID', 'item_cnt_month']].to_csv('initial_lgb_submission.csv', index=False )
buro_counts = buro_balance.groupby("SK_ID_BUREAU")["STATUS"].value_counts(normalize=False)
 buro_counts_unstacked = buro_counts.unstack("STATUS")
<feature_engineering>
categories = pd.read_csv('.. /input/predict-future-sales-eng-translation/categories.csv')
 categories['group_name'] = categories['category_name'].str.extract(r'(^[\w\s]*)')
 categories['group_name'] = categories['group_name'].str.strip()
items = pd.read_csv('.. /input/predict-future-sales-eng-translation/items.csv')
 items['item_name'] = items['item_name'].str.lower()
for i in [r'[^\w\d\s]', r'\bthe\b', r'\bin\b', r'\bfor\b', r'\bof\b', r'\bd\b', r'\bis\b', r'\bon\b']:
items['item_name'] = items['item_name'].str.replace(i, ' ')
 items['item_name'] = items['item_name'].str.replace(' ', '')
 items = items.join(categories.set_index('category_id'), on='category_id')
 shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops['shop_name'] = shops['shop_name'].str.lower()
shops['shop_name'] = shops['shop_name'].str.replace(r'[^\w\d\s]', ' ')
 shops['shop_city'] = shops['shop_name'].str.split().str[0]
shops.loc[shops['shop_id'].isin([12,55]), 'shop_city'] = 'online'
buro_counts_unstacked.columns = ['STATUS_0', 'STATUS_1','STATUS_2','STATUS_3','STATUS_4','STATUS_5','STATUS_C','STATUS_X',]
buro_counts_unstacked["MONTHS_COUNT"] = buro_grouped_size
buro_counts_unstacked["MONTHS_MIN"] = buro_grouped_min
buro_counts_unstacked["MONTHS_MAX"] = buro_grouped_max<merge>
X_train['lgb_pred'] = lgb_model.predict(X_train ).clip(0,20)
 X_train['target'] = y_train
X_train['sq_err'] =(X_train['lgb_pred']-X_train['target'])**2
X_val['lgb_pred'] = lgb_model.predict(X_val ).clip(0,20)
 X_val['target'] = y_val
X_val['sq_err'] =(X_val['lgb_pred']-X_val['target'])**2
X_test['lgb_pred'] = lgb_model.predict(X_test ).clip(0,20 )
buro = buro.join(buro_counts_unstacked, how='left', on='SK_ID_BUREAU' )<categorify>
data = X_train.groupby('date_block_num' ).agg({'lgb_pred':'mean','target':'mean','sq_err':'mean'} ).reset_index()
data['new_item_rmse'] = np.sqrt(X_train.query('item_age<=1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data['old_item_rmse'] = np.sqrt(X_train.query('item_age>1' ).groupby('date_block_num' ).agg({'sq_err':'mean'} ).sq_err)
 data = data.append([
{'date_block_num':33,
'target':X_val.target.mean() ,
'lgb_pred':X_val.lgb_pred.mean() ,
'sq_err':np.sqrt(X_val.sq_err.mean()),
'old_item_rmse':np.sqrt(X_val.query('item_age>1' ).sq_err.mean()),
'new_item_rmse':np.sqrt(X_val.query('item_age<=1' ).sq_err.mean())},
{'date_block_num':34,
'target':0,
'lgb_pred':X_test.lgb_pred.mean() ,
'sq_err':0,
'old_item_rmse':0,
'new_item_rmse':0}
],
ignore_index=True 
)
 data['date'] = [x[:7] for x in pd.date_range(start='2013-03',end='2015-09',freq='MS' ).astype('str')]+['Validation','Test']
prev_cat_features = [pcol for pcol in prev.columns if prev[pcol].dtype == "object"]
prev = pd.get_dummies(prev, columns=prev_cat_features)
<groupby>
df = pd.read_pickle('.. /input/files-top-scoring-notebook-output-exploration/df_complete.pkl')
(
 df
[df['category_id'].isin(X_test.category_id.unique())] 
.query('item_cnt>0')
.groupby('category_id')
.agg({
'category_age':'max',
'shop_id':['nunique','unique'],
'item_cnt':'sum'
})
.join(categories['category_name'])
.join(
 X_test 
.groupby('category_id')
.agg({'item_id':'nunique'})
.rename(columns={'item_id':'test_set_items'})
)
.sort_values(( 'shop_id', 'nunique')) 
.head(20)
 )
avg_prev = prev.groupby("SK_ID_CURR" ).mean() 
<groupby>
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('shop_id')
.agg({
'sq_err':'mean',
'target':'mean',
'lgb_pred':'mean'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('shop_id')
.agg({'test_pred':'mean'})
)
.join(shops['shop_name'])
.sort_values('sq_err', ascending=False)
.head(20)
 )
cnt_prev = prev[["SK_ID_CURR", "SK_ID_PREV"]].groupby("SK_ID_CURR" ).count()
avg_prev["nb_app"] = cnt_prev["SK_ID_PREV"]
del avg_prev["SK_ID_PREV"]<categorify>
(
X_val
[X_val['category_id'].isin(X_test.category_id.unique())] 
.groupby('category_id')
.agg({
'sq_err':['sum','mean'],
'target':'mean',
'lgb_pred':['sum','mean'],
'item_id':'nunique'
})
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred','item_id':'test_items'})
.groupby('category_id')
.agg({
'test_pred':['sum','mean'],
'test_items':'nunique'
}),
on='category_id' 
)
.join(categories)
.sort_values(( 'sq_err', 'mean'), ascending=False)
.head(20)
 )
buro_cat_features = [bcol for bcol in buro.columns if buro[bcol].dtype == 'object']
buro = pd.get_dummies(buro, columns=buro_cat_features)
 avg_buro = buro.groupby('SK_ID_CURR' ).mean()
avg_buro['buro_count'] = buro[['SK_ID_BUREAU', 'SK_ID_CURR']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
del avg_buro['SK_ID_BUREAU']<categorify>
CATEGORY = 20 
(
 items[
items['item_id'].isin(X_val.item_id.unique())&
items['item_id'].isin(X_test.item_id.unique())
 ]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.rename(columns={'lgb_pred':'test_pred'})
.groupby('item_id')
.agg({'test_pred':'mean'}),
on='item_id' 
)
.join(
 X_val 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'target':'mean',
'sq_err':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={
'lgb_pred':'val_pred',
'target':'val_target',
}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('sq_err',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )
le = LabelEncoder()
POS_CASH['NAME_CONTRACT_STATUS'] = le.fit_transform(POS_CASH['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
nunique_status2.head(10 )<count_unique_values>
CATEGORY = 20 
(
 items[items['item_id'].isin(X_test.item_id.unique())]
[['category_id','category_name','item_id','item_name']] 
.join(
 X_test 
.groupby('item_id')
.agg({
'lgb_pred':'mean',
'same_name4catage_cnt_all_shops':'first',
'new_items_in_cat_all_shops_lag1to12':'first',
'item_cnt_all_shops_lag1':'first',
'category_cnt_all_shops_lag1':'first',
'item_cnt_sum_alltime_allshops':'first',
'prev_days_on_sale':'first'
})
.rename(columns={'lgb_pred':'test_pred'}),
on='item_id' 
)
.query(f'category_id=={CATEGORY}')
.sort_values('test_pred',ascending=False)
.rename(columns={
'same_name4catage_cnt_all_shops':'name4mean',
'new_items_in_cat_all_shops_lag1to12':'new_in_cat_mean',
'item_cnt_all_shops_lag1':'item_cnt_lag1',
'category_cnt_all_shops_lag1':'cat_cnt_lag1',
'item_category_id':'cat',
'item_cnt_sum_alltime_allshops':'item_cnt_alltime'
})
.head(20)
 )
<categorify><EOS> 
submission_1 = pd.read_csv('.. /input/blend-boosting-best-score-on-predict-future-sales/submission_blend_1.csv')
 submission_2 = pd.read_csv('initial_lgb_submission.csv')
 submission_1['item_cnt_month'] = np.median(np.concatenate([np.expand_dims(submission_1['item_cnt_month'], axis = 1), np.expand_dims(submission_2['item_cnt_month'], axis = 1)], axis = 1), axis = 1)
 submission_1[['ID', 'item_cnt_month']].to_csv('final_sub.csv', index=False )