Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<split>
warnings.filterwarnings(action='ignore')
 data_path = '/kaggle/input/competitive-data-science-predict-future-sales/'
sales_train = pd.read_csv(data_path + 'sales_train.csv')
 shops = pd.read_csv(data_path + 'shops.csv')
 items = pd.read_csv(data_path + 'items.csv')
 item_categories = pd.read_csv(data_path + 'item_categories.csv')
 test = pd.read_csv(data_path + 'test.csv')
 submission = pd.read_csv(data_path + 'sample_submission.csv' )
n_folds = 5
k_fold = KFold(n_splits=n_folds, shuffle=False, random_state=50)
 feature_importances_values = np.zeros(data.shape[1])
 test_predictions = np.zeros(test.shape[0])
 out_of_fold = np.zeros(data.shape[0])
 valid_scores = []
train_scores = []
for train_indices, test_indices in k_fold.split(data):
train_data, train_y = data.iloc[train_indices], y.iloc[train_indices]
test_data, test_y = data.iloc[test_indices], y.iloc[test_indices]
model = lgb.LGBMClassifier(application="binary", boosting_type=lgbm_params["boosting"],
learning_rate=lgbm_params["learning_rate"], n_estimators=lgbm_params["n_estimators"],
num_leaves=lgbm_params["num_leaves"],max_depth=lgbm_params["max_depth"],
reg_lambda=lgbm_params['reg_lambda'],reg_alpha=lgbm_params["reg_alpha"],
drop_rate=lgbm_params["drop_rate"], random_state=50)
 model.fit(train_data, train_y, eval_metric='auc', eval_set=[(test_data, test_y),(train_data, train_y)],
eval_names=['valid', 'train'], early_stopping_rounds=100, verbose=200)
 best_iteration = model.best_iteration_
feature_importances_values += model.feature_importances_ / k_fold.n_splits
test_predictions += model.predict_proba(test, num_iteration=best_iteration)[:,1]/k_fold.n_splits
out_of_fold[test_indices] = model.predict_proba(test_data, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, train_data, test_data
gc.collect() 
<save_to_csv>
def downcast(df, verbose=True):
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
dtype_name = df[col].dtype.name
if dtype_name == 'object':
pass
elif dtype_name == 'bool':
df[col] = df[col].astype('int8')
 elif dtype_name.startswith('int')or(df[col].round() == df[col] ).all() :
df[col] = pd.to_numeric(df[col], downcast='integer')
 else:
df[col] = pd.to_numeric(df[col], downcast='float')
 end_mem = df.memory_usage().sum() / 1024**2
if verbose:
print('{:.1f}% compressed'.format(100 *(start_mem - end_mem)/ start_mem))
return df
all_df = [sales_train, shops, items, item_categories, test]
for df in all_df:
df = downcast(df )
submission = pd.DataFrame({'SK_ID_CURR': test["SK_ID_CURR"], 'TARGET': test_predictions})
 submission.to_csv("submissions.csv", index=False )<import_modules>
sales_train = sales_train[sales_train['item_price'] > 0]
sales_train = sales_train[sales_train['item_price'] < 50000]
sales_train = sales_train[sales_train['item_cnt_day'] > 0]
sales_train = sales_train[sales_train['item_cnt_day'] < 1000]
import gc<import_modules>
sales_train.loc[sales_train['shop_id'] == 0, 'shop_id'] = 57
sales_train.loc[sales_train['shop_id'] == 1, 'shop_id'] = 58
sales_train.loc[sales_train['shop_id'] == 10, 'shop_id'] = 11
sales_train.loc[sales_train['shop_id'] == 39, 'shop_id'] = 40
test.loc[test['shop_id'] == 0, 'shop_id'] = 57
test.loc[test['shop_id'] == 1, 'shop_id'] = 58
test.loc[test['shop_id'] == 10, 'shop_id'] = 11
test.loc[test['shop_id'] == 39, 'shop_id'] = 40
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import LabelEncoder, MinMaxScaler<load_from_csv>
shops['city'] = shops['shop_name'].apply(lambda x: x.split() [0] )
data = pd.read_csv(".. /input/application_train.csv")
 test = pd.read_csv('.. /input/application_test.csv')
 prev = pd.read_csv('.. /input/previous_application.csv')
 buro = pd.read_csv('.. /input/bureau.csv')
 buro_balance = pd.read_csv('.. /input/bureau_balance.csv')
 credit_card = pd.read_csv('.. /input/credit_card_balance.csv')
 POS_CASH = pd.read_csv('.. /input/POS_CASH_balance.csv')
 payments = pd.read_csv('.. /input/installments_payments.csv')
 lgbm_submission = pd.read_csv('.. /input/sample_submission.csv' )<prepare_x_and_y>
shops.loc[shops['city'] =='!Якутск', 'city'] = 'Якутск'
y = data["TARGET"]
del data["TARGET"]<define_variables>
label_encoder = LabelEncoder()
shops['city'] = label_encoder.fit_transform(shops['city'] )
categorical_feature = [col for col in data.columns if data[col].dtype == 'object']<categorify>
shops = shops.drop('shop_name', axis=1)
 shops.head()
one_hot_df = pd.concat([data, test])
 one_hot_df = pd.get_dummies(one_hot_df, columns=categorical_feature )<split>
items = items.drop(['item_name'], axis=1 )
data = one_hot_df.iloc[:data.shape[0],:]
test = one_hot_df.iloc[data.shape[0]:,]<groupby>
items['first_sale_date'] = sales_train.groupby('item_id' ).agg({'date_block_num': 'min'})['date_block_num']
items.head()
buro_grouped_size = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].size()
buro_grouped_max = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].max()
buro_grouped_min = buro_balance.groupby("SK_ID_BUREAU")["MONTHS_BALANCE"].min()<groupby>
items[items['first_sale_date'].isna() ]
buro_counts = buro_balance.groupby("SK_ID_BUREAU")["STATUS"].value_counts(normalize=False)
 buro_counts_unstacked = buro_counts.unstack("STATUS")
<feature_engineering>
items['first_sale_date'] = items['first_sale_date'].fillna(34 )
buro_counts_unstacked.columns = ['STATUS_0', 'STATUS_1','STATUS_2','STATUS_3','STATUS_4','STATUS_5','STATUS_C','STATUS_X',]
buro_counts_unstacked["MONTHS_COUNT"] = buro_grouped_size
buro_counts_unstacked["MONTHS_MIN"] = buro_grouped_min
buro_counts_unstacked["MONTHS_MAX"] = buro_grouped_max<merge>
item_categories['category'] = item_categories['item_category_name'].apply(lambda x: x.split() [0] )
buro = buro.join(buro_counts_unstacked, how='left', on='SK_ID_BUREAU' )<categorify>
item_categories['category'].value_counts()
prev_cat_features = [pcol for pcol in prev.columns if prev[pcol].dtype == "object"]
prev = pd.get_dummies(prev, columns=prev_cat_features)
 prev.head(10 )<groupby>
def make_etc(x):
if len(item_categories[item_categories['category']==x])>= 5:
return x
else:
return 'etc'
item_categories['category'] = item_categories['category'].apply(make_etc )
avg_prev = prev.groupby("SK_ID_CURR" ).mean()
avg_prev.head(10 )<groupby>
label_encoder = LabelEncoder()
item_categories['category'] = label_encoder.fit_transform(item_categories['category'])
 item_categories = item_categories.drop('item_category_name', axis=1 )
cnt_prev = prev[["SK_ID_CURR", "SK_ID_PREV"]].groupby("SK_ID_CURR" ).count()
avg_prev["nb_app"] = cnt_prev["SK_ID_PREV"]
del avg_prev["SK_ID_PREV"]<categorify>
group = sales_train.groupby(idx_features ).agg({'item_cnt_day': 'sum',
'item_price': 'mean'})
 group = group.reset_index()
group = group.rename(columns={'item_cnt_day': 'item_cnt_month', 'item_price': 'item_price_mean'})
 train = train.merge(group, on=idx_features, how='left')
 train.head()
buro_cat_features = [bcol for bcol in buro.columns if buro[bcol].dtype == 'object']
buro = pd.get_dummies(buro, columns=buro_cat_features)
 avg_buro = buro.groupby('SK_ID_CURR' ).mean()
avg_buro['buro_count'] = buro[['SK_ID_BUREAU', 'SK_ID_CURR']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
del avg_buro['SK_ID_BUREAU']<categorify>
del group
gc.collect() ;
le = LabelEncoder()
POS_CASH['NAME_CONTRACT_STATUS'] = le.fit_transform(POS_CASH['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
nunique_status2.head(10 )<count_unique_values>
group = sales_train.groupby(idx_features ).agg({'item_cnt_day': 'count'})
 group = group.reset_index()
group = group.rename(columns={'item_cnt_day': 'item_count'})
 train = train.merge(group, on=idx_features, how='left')
 del group, sales_train
gc.collect()
train.head()
POS_CASH['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
POS_CASH['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
POS_CASH.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True )<categorify>
test['date_block_num'] = 34
all_data = pd.concat([train, test.drop('ID', axis=1)],
ignore_index=True,
keys=idx_features)
 all_data = all_data.fillna(0)
 all_data.head()
le =LabelEncoder()
credit_card['NAME_CONTRACT_STATUS'] = le.fit_transform(credit_card['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
credit_card['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
credit_card['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
credit_card.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True)
<groupby>
all_data = all_data.merge(shops, on='shop_id', how='left')
 all_data = all_data.merge(items, on='item_id', how='left')
 all_data = all_data.merge(item_categories, on='item_category_id', how='left')
 all_data = downcast(all_data)
 del shops, items, item_categories
gc.collect() ;
avg_payments = payments.groupby("SK_ID_CURR" ).mean()
avg_payments2 = payments.groupby("SK_ID_CURR" ).max()
avg_payments3 = payments.groupby("SK_ID_CURR" ).min()<drop_column>
def resumetable(df):
print(f'Data Shape: {df.shape}')
 summary = pd.DataFrame(df.dtypes, columns=['Dtypes'])
 summary['Null'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First_values'] = df.loc[0].values
summary['Second_values'] = df.loc[1].values
summary['Third_values'] = df.loc[2].values
return summary
del avg_payments["SK_ID_PREV"]<merge>
def add_mean_features(df, mean_features, idx_features):
assert(idx_features[0] == 'date_block_num')and \
len(idx_features)in [2, 3]
if len(idx_features)== 2:
feature_name = idx_features[1] + '_mean_sales'
else:
feature_name = idx_features[1] + '_' + idx_features[2] + '_mean_sales'
group = df.groupby(idx_features ).agg({'item_cnt_month': 'mean'})
 group = group.reset_index()
group = group.rename(columns={'item_cnt_month': feature_name})
 df = df.merge(group, on=idx_features, how='left')
 df = downcast(df, False)
 mean_features.append(feature_name)
 del group
gc.collect()
return df, mean_features
data = data.merge(right=avg_prev.reset_index() , how="left", on="SK_ID_CURR")
 test = test.merge(right=avg_prev.reset_index() , how="left", on="SK_ID_CURR")
<merge>
item_mean_features = []
all_data, item_mean_features = add_mean_features(df=all_data,
mean_features=item_mean_features,
idx_features=['date_block_num', 'item_id'])
 all_data, item_mean_features = add_mean_features(df=all_data,
mean_features=item_mean_features,
idx_features=['date_block_num', 'item_id', 'city'] )
data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR' )<merge>
shop_mean_features = []
all_data, shop_mean_features = add_mean_features(df=all_data,
mean_features=shop_mean_features,
idx_features=['date_block_num', 'shop_id', 'item_category_id'] )
data = data.merge(right=POS_CASH.groupby("SK_ID_CURR" ).mean().reset_index() , how="left", on="SK_ID_CURR")
 test = test.merge(right=POS_CASH.groupby("SK_ID_CURR" ).mean().reset_index() , how="left", on="SK_ID_CURR" )<merge>
def add_lag_features(df, lag_features_to_clip, idx_features,
lag_feature, nlags=3, clip=False):
df_temp = df[idx_features + [lag_feature]].copy()
for i in range(1, nlags+1):
lag_feature_name = lag_feature +'_lag' + str(i)
 df_temp.columns = idx_features + [lag_feature_name]
df_temp['date_block_num'] += i
df = df.merge(df_temp.drop_duplicates() ,
on=idx_features,
how='left')
 df[lag_feature_name] = df[lag_feature_name].fillna(0)
 if clip:
lag_features_to_clip.append(lag_feature_name)
 df = downcast(df, False)
 del df_temp
gc.collect()
return df, lag_features_to_clip
data = data.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(credit_card.groupby('SK_ID_CURR' ).mean().reset_index() , how='left', on='SK_ID_CURR')
<merge>
lag_features_to_clip = []
idx_features = ['date_block_num', 'shop_id', 'item_id']
all_data, lag_features_to_clip = add_lag_features(df=all_data,
lag_features_to_clip=lag_features_to_clip,
idx_features=idx_features,
lag_feature='item_cnt_month',
nlags=3,
clip=True )
data = data.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments.reset_index() , how='left', on='SK_ID_CURR')
<merge>
all_data, lag_features_to_clip = add_lag_features(df=all_data,
lag_features_to_clip=lag_features_to_clip,
idx_features=idx_features,
lag_feature='item_count',
nlags=3)
 all_data, lag_features_to_clip = add_lag_features(df=all_data,
lag_features_to_clip=lag_features_to_clip,
idx_features=idx_features,
lag_feature='item_price_mean',
nlags=3 )
data = data.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments2.reset_index() , how='left', on='SK_ID_CURR' )<merge>
X_test_temp = all_data[all_data['date_block_num'] == 34]
X_test_temp[item_mean_features].sum()
data = data.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_payments3.reset_index() , how='left', on='SK_ID_CURR' )<filter>
for item_mean_feature in item_mean_features:
all_data, lag_features_to_clip = add_lag_features(df=all_data,
lag_features_to_clip=lag_features_to_clip,
idx_features=idx_features,
lag_feature=item_mean_feature,
nlags=3)
 all_data = all_data.drop(item_mean_features, axis=1 )
test = test[test.columns[data.isnull().mean() < 0.85]]
data = data[data.columns[data.isnull().mean() < 0.85]]<define_variables>
for shop_mean_feature in shop_mean_features:
all_data, lag_features_to_clip = add_lag_features(df=all_data,
lag_features_to_clip=lag_features_to_clip,
idx_features=['date_block_num', 'shop_id', 'item_category_id'],
lag_feature=shop_mean_feature,
nlags=3)
 all_data = all_data.drop(shop_mean_features, axis=1 )
excluded_feats = ["SK_ID_CURR"]
features = [f for f in data.columns if f not in excluded_feats]<define_variables>
all_data = all_data.drop(all_data[all_data['date_block_num'] < 3].index )
folds = KFold(n_splits = 4, shuffle=True, random_state=546789)
 oof_preds = np.zeros(data.shape[0])
 sub_preds = np.zeros(test.shape[0] )<split>
all_data['item_cnt_month_lag_mean'] = all_data[['item_cnt_month_lag1',
'item_cnt_month_lag2',
'item_cnt_month_lag3']].mean(axis=1 )
for n_fold,(trn_idx, val_idx)in enumerate(folds.split(data)) :
trn_x, trn_y = data[features].iloc[trn_idx], y.iloc[trn_idx]
val_x, val_y = data[features].iloc[val_idx], y.iloc[val_idx]
clf = XGBClassifier(
 objective="binary:logistic",
booster="gbtree",
eval_metric = "auc",
nthread = 4,
eta = 0.05,
gamma = 0,
max_depth = 6,
subsample=0.7,
colsample_bytree = 0.7,
colsample_bylevel = 0.675,
min_child_weight = 22,
alpha = 0,
random_state = 42,
nrounds = 2000 
)
 clf.fit(trn_x, trn_y, eval_set=[(trn_x, trn_y),(val_x, val_y)], verbose=10, early_stopping_rounds=30)
 oof_preds[val_idx] = clf.predict_proba(val_x)[:, 1]
sub_preds += clf.predict_proba(test[features])[:, 1] / folds.n_splits
print("Fold %2d AUC : %.6f" %(n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))
 del clf, trn_x, trn_y, val_x, val_y
gc.collect()<compute_test_metric>
all_data[lag_features_to_clip + ['item_cnt_month', 'item_cnt_month_lag_mean']] = all_data[lag_features_to_clip +['item_cnt_month', 'item_cnt_month_lag_mean']].clip(0, 20 )
print('Full AUC score %.6f' % roc_auc_score(y, oof_preds))
test['TARGET'] = sub_preds<save_to_csv>
all_data['lag_grad1'] = all_data['item_cnt_month_lag1']/all_data['item_cnt_month_lag2']
all_data['lag_grad1'] = all_data['lag_grad1'].replace([np.inf, -np.inf],
np.nan ).fillna(0)
 all_data['lag_grad2'] = all_data['item_cnt_month_lag2']/all_data['item_cnt_month_lag3']
all_data['lag_grad2'] = all_data['lag_grad2'].replace([np.inf, -np.inf],
np.nan ).fillna(0 )
test[["SK_ID_CURR","TARGET"]].to_csv('xgb_submission_esi.csv', index=False, float_format='%.8f' )<set_options>
all_data['brand_new'] = all_data['first_sale_date'] == all_data['date_block_num']
warnings.simplefilter(action='ignore', category=FutureWarning)
 py.init_notebook_mode(connected=True)
 %matplotlib inline
cf.go_offline() 
<categorify>
all_data['duration_after_first_sale'] = all_data['date_block_num'] - all_data['first_sale_date']
all_data = all_data.drop('first_sale_date', axis=1 )
def one_hot_encoder(df, nan_as_category = True):
original_columns = list(df.columns)
 categorical_columns = [col for col in df.columns if df[col].dtype == 'object']
df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)
 new_columns = [c for c in df.columns if c not in original_columns]
return df, new_columns<define_variables>
all_data['month'] = all_data['date_block_num']%12
num_rows = None
nan_as_category = True<train_model>
X_train = all_data[all_data['date_block_num'] < 33]
X_train = X_train.drop(['item_cnt_month'], axis=1)
 X_valid = all_data[all_data['date_block_num'] == 33]
X_valid = X_valid.drop(['item_cnt_month'], axis=1)
 X_test = all_data[all_data['date_block_num'] == 34]
X_test = X_test.drop(['item_cnt_month'], axis=1)
 y_train = all_data[all_data['date_block_num'] < 33]['item_cnt_month']
y_valid = all_data[all_data['date_block_num'] == 33]['item_cnt_month']
del all_data
gc.collect() ;
print("Start Train Test................." )<load_from_csv>
params = {'metric': 'rmse',
'num_leaves': 255,
'learning_rate': 0.005,
'feature_fraction': 0.75,
'bagging_fraction': 0.75,
'bagging_freq': 5,
'force_col_wise' : True,
'random_state': 10}
cat_features = ['shop_id', 'city', 'item_category_id', 'category', 'month']
dtrain = lgb.Dataset(X_train, y_train)
 dvalid = lgb.Dataset(X_valid, y_valid)
 lgb_model = lgb.train(params=params,
train_set=dtrain,
num_boost_round=1500,
valid_sets=(dtrain, dvalid),
early_stopping_rounds=150,
categorical_feature=cat_features,
verbose_eval=100 )
df = pd.read_csv('.. /input/application_train.csv', nrows= num_rows)
 test_df = pd.read_csv('.. /input/application_test.csv', nrows= num_rows)
 print("Train samples: {}, test samples: {}".format(len(df), len(test_df)))
 df = df.append(test_df ).reset_index()
del test_df
gc.collect()<categorify>
preds = lgb_model.predict(X_test ).clip(0,20)
 submission['item_cnt_month'] = preds
submission.to_csv('submission.csv', index=False )
<feature_engineering><EOS> 
del X_train, y_train, X_valid, y_valid, X_test, lgb_model, dtrain, dvalid
gc.collect() ;