Kaggle competition: Santander Customer Transaction Prediction
data_dir = ".. /input/competitive-data-science-predict-future-sales/"
trans_file = "sales_train.csv"
out_file = "test.csv"
items_file = "items.csv"
if LOCATION_KAGGLE:
categs_file = "item_categories.csv"
shops_file = "shops.csv"
else:
categs_file = "item_categories-translated.csv"
shops_file = "shops-translated.csv"
item_names_file = "items-translated.csv" 
<load_from_csv>
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 submit_data = pd.read_csv('.. /input/sample_submission.csv')
 features = [c for c in train_data.columns if c not in ['ID_code', 'target']]
target = train_data['target']
print("Data is ready!" )
df_trans = pd.read_csv(data_dir+trans_file)
 if SHIFT_1MONTH:
df_trans['date_block_num'] += 1
df_out = pd.read_csv(data_dir+out_file)
 df_items = pd.read_csv(data_dir+items_file)
 df_categs = pd.read_csv(data_dir+categs_file)
 df_shops = pd.read_csv(data_dir+shops_file)
 if LOCATION_KAGGLE == False:
df_item_names = pd.read_csv(data_dir+item_names_file)
 df_items['item_name'] = df_item_names['item_name'] 
<count_unique_values>
tar = target
num_months = len(df_trans['date_block_num'].unique())
 print("
Full Transactions has {} entries covering {} months.
 ".format(len(df_trans),num_months))
print("The number of unique values for each column:")
 df_trans.nunique()<count_values>
train_data = train_data.drop(["ID_code"], axis=1)
 test_data = test_data.drop(["ID_code"], axis=1 )
<train_model>
target0 = train_data[target==0]
target1 = train_data[target==1]
<count_unique_values>
for i in range(0,199):
ch = 'var_' + str(i)
 target0[ch] = np.random.permutation(target0[ch])
 for i in range(0,199):
ch = 'var_' + str(i)
 target1[ch] = np.random.permutation(target1[ch] )
if SHIFT_1MONTH:
print(" - - - SHIFT_1MONTH - - -
Finding all future_items")
 out_items = np.sort(df_out.item_id.unique())
 num_out_items = len(out_items)
 print("
Number of items in Output: {}".format(num_out_items))
trans_items = np.sort(df_trans.item_id.unique())
 print("Number of items in FULL Transactions: {}".format(len(trans_items)))
 future_items = []
for item in out_items:
if item not in trans_items:
future_items.append(item)
 if len(future_items)> 0:
print("There are {} future_items which are missing from the FULL Transactions data:".format(
 len(future_items)) ,"
")
 else:
print("
All of the Test Items are in the Train data.
 ")
 plt.hist(future_items, bins=50)
 plt.xlabel("item_id")
 plt.title("Future Items missing from the FULL Transactions")
 plt.show()<define_variables>
target0 = target0.append(target1 )
if SHIFT_1MONTH and LOCATION_KAGGLE:
df_month = df_trans[df_trans['date_block_num'] == 34].copy()
df_val = df_out.copy()
out_vals = []
for iout in df_val.index:
this_shop = df_val.loc[iout,'shop_id']
this_item = df_val.loc[iout,'item_id']
out_vals.append(df_month.loc[(df_month['shop_id'] == this_shop)& 
(df_month['item_id'] == this_item),'item_cnt_day'].sum())
 df_val['item_cnt_month'] = np.array(out_vals)
 clip_level = 20
num_over20 = sum(df_val['item_cnt_month'] > clip_level)
 print("
Check for over-20 values, found/clipped {} of them.".format(num_over20))
if num_over20 > 0:
df_val.loc[df_val['item_cnt_month'] > clip_level, 'item_cnt_month'] = clip_level
print("
Sum of output item_cnt_month: {:.0f}, average count-per-month per shop-item: {:.3f}
".format(
 df_val['item_cnt_month'].sum() , df_val['item_cnt_month'].sum() /(len(df_val))))
df_val[['ID','item_cnt_month']].to_csv("submission_Oct15.csv", index=None)
 del df_month, df_val, out_vals
gc.collect()<drop_column>
train_data = train_data.append(target0 )
if DOWNSEL_14MONS:
df_trans =(df_trans[(( df_trans['date_block_num'] >= 20)& 
(df_trans['date_block_num'] <= 33)) ] ).reset_index().drop('index',axis=1)
 elif DOWNSEL_26MONS:
df_trans =(df_trans[(( df_trans['date_block_num'] >= 8)& 
(df_trans['date_block_num'] <= 33)) ] ).reset_index().drop('index',axis=1)
 else:
df_trans =(df_trans[(( df_trans['date_block_num'] <= 33)) ] ).reset_index().drop('index',axis=1)
 num_months = len(df_trans['date_block_num'].unique())
 print("Using {} months of data.".format(num_months))<feature_engineering>
target = train_data['target']
df_trans['item_cnt_clip'] = df_trans['item_cnt_day']
df_trans.loc[df_trans['item_cnt_clip'] > 20, 'item_cnt_clip'] = 20<merge>
train_data = train_data.drop(["target"], axis=1)
df_trans = pd.merge(df_trans, df_items[['item_id','item_category_id']].set_index('item_id'), left_on='item_id',
right_index=True, how='left', sort=False)
 df_out = pd.merge(df_out, df_items[['item_id','item_category_id']].set_index('item_id'), left_on='item_id',
right_index=True, how='left', sort=False )<data_type_conversions>
%%time
train_data = pd.DataFrame(scale(train_data.values), columns=train_data.columns, index=train_data.index)
 test_data = pd.DataFrame(scale(test_data.values), columns=test_data.columns, index=test_data.index )
df_trans['log10_price'] = np.log10(np.abs(df_trans['item_price']))
print("Min and Max of the log10 of item price: {:.2f} , {:.2f}".format(df_trans['log10_price'].min() ,
df_trans['log10_price'].max()))
df_trans['item_price3'] =(( df_trans['item_price'] > 300.00 ).astype(int)+ 
(df_trans['item_price'] > 1000.00 ).astype(int))<feature_engineering>
params = {}
params['bagging_freq'] = 5
params['bagging_fraction'] = 0.0331
params['random_state'] = 42
params['learning_rate'] = 0.0123
params['boost_from_average'] = False
params['boosting_type'] = 'gbdt'
params['feature_fraction'] = 0.045
params['objective'] = 'binary'
params['metric'] = 'auc'
params['min_data_in_leaf'] = 80
params['num_leaves'] = 13
params['num_threads'] = 8
params['tree_learner'] = 'serial'
params['max_depth'] = -1
params['min_sum_hessian_in_leaf'] = 10.0
params['verbosity'] = 1
params['bagging_seed'] = 42
params['seed'] = 42
df_trans['date_dt'] = pd.to_datetime(df_trans.date, format='%d.%m.%Y')
 df_trans['day'] = df_trans.date_dt.dt.day
df_trans['month'] = df_trans.date_dt.dt.month
df_trans['year'] = df_trans.date_dt.dt.year<count_unique_values>
num_folds = 10
features = [c for c in test_data.columns if c not in ['ID_code', 'target']]
folds = StratifiedKFold(n_splits=num_folds,shuffle=True, random_state=42)
 oof = np.zeros(len(train_data))
getVal = np.zeros(len(train_data))
predictions = np.zeros(len(tar))
print(predictions.shape)
 feature_importance_df = pd.DataFrame()
print("The selected Transactions have {} entries covering {} months.".format(len(df_trans),num_months))
print("The number of unique values for each column:")
 df_trans.nunique()<predict_on_test>
%%time
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data.values, target.values)) :
X_train, y_train = train_data.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train_data.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = X_train.values, y_train.values
X_tr = pd.DataFrame(X_tr)
 print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 predictions += clf.predict(test_data[features], num_iteration=clf.best_iteration)/ folds.n_splits
<define_variables><EOS> 
submit_data['target'] = pd.DataFrame(predictions)
 submit_data.to_csv("LGBM.csv", index=False )