Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<count_unique_values>
warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.simplefilter(action='ignore', category=DeprecationWarning)
 %matplotlib inline
print(os.listdir(".. /input"))
train_categories=sorted(train_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(train_categories)) + ' categories in the training set :')
 print(train_categories)
 print()
test_categories=sorted(test_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(test_categories)) + ' categories in the test set :')
 print(test_categories)
 print()
print('number of supercategories : '+str(item_categories['item_supercategory_name'].nunique()))
print()
for supercat in item_categories['item_supercategory_name'].unique() :
print(supercat + ' : ' +str(item_categories.loc[item_categories['item_supercategory_name']==supercat,'item_category_id'].nunique()))
print(item_categories.loc[item_categories['item_supercategory_name']==supercat,['item_category_id','item_category_name']])
 print()
item_categories
del train_categories, test_categories<categorify>
train_df = pd.read_csv(".. /input/train.csv")
 train_df.head()
train_df['item_category_id']=train_df['item_id'].map(items['item_category_id'] )<drop_column>
def null_values(df):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() / len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 mis_val_table_ren_columns = mis_val_table_ren_columns[
mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
 '% of Total Values', ascending=False ).round(1)
 print("Dataframe has " + str(df.shape[1])+ " columns.
 "
"There are " + str(mis_val_table_ren_columns.shape[0])+
" columns that have missing values.")
 return mis_val_table_ren_columns
print('category 8 : ' + str(item_categories.loc[8,'item_category_name']))
print('category 80 : ' + str(item_categories.loc[80,'item_category_name']))
print('category 81 : ' + str(item_categories.loc[81,'item_category_name']))
print('category 82 : ' + str(item_categories.loc[82,'item_category_name']))
train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==8)].index.values,axis=0,inplace=True)
 train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==80)].index.values,axis=0,inplace=True)
 train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==81)].index.values,axis=0,inplace=True)
 train_df.drop(train_df.loc[(train_df['item_id'].map(items['item_category_id'])==82)].index.values,axis=0,inplace=True)
 item_categories.drop(8,axis=0,inplace=True)
 item_categories.drop(80,axis=0,inplace=True)
 item_categories.drop(81,axis=0,inplace=True)
 item_categories.drop(82,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==8].index.values,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==80].index.values,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==81].index.values,axis=0,inplace=True)
 items.drop(items.loc[items['item_category_id']==82].index.values,axis=0,inplace=True )<categorify>
test_df = pd.read_csv(".. /input/test.csv")
 test_df.head()
item_categories['item_supercategory_id']=item_categories['item_supercategory_name'].map({'Игровые консоли':0,'Игры':1,'Аксессуары':2,'Доставка товара':3,'Прием денежных средств для 1С-Онлайн':4,'Карты оплаты':5,'Кино':6,'Книги':7,'Музыка':8,'Подарки':9,'Программы':10,'Misc':11})
 item_categories['item_category_console_id']=item_categories['item_category_console'].map({console:i for i,console in enumerate(consoles+['None'])} )<define_variables>
var_features = ["var_0", "var_15", "var_22", "var_25", "var_39", "var_45", "var_52", "var_68",
"var_85", "var_99", "var_112", "var_135", "var_147", "var_158", "var_160",
"var_169", "var_173", "var_180", "var_192", "var_199"]
original_cols=['item_category_name','item_supercategory_name','item_category_console','item_category_is_digital']
label_cols=['item_category_id','item_supercategory_id','item_category_console_id']
item_categories=item_categories[original_cols+label_cols]<categorify>
X = np.array(train_df.loc[:, train_df.columns != 'target'])
 y = np.array(train_df.loc[:, train_df.columns == 'target'])
 print('Shape of X: {}'.format(X.shape))
print('Shape of y: {}'.format(y.shape))
for col in original_cols:
train_df[col]=train_df['item_category_id'].map(item_categories[col])
 del original_cols
del label_cols<count_unique_values>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=0, stratify=y)
 print("X_train dataset: ", X_train.shape)
 print("y_train dataset: ", y_train.shape)
 print("X_valid dataset: ", X_valid.shape)
 print("y_valid dataset: ", y_valid.shape )
train_categories=sorted(train_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(train_categories)) + ' categories in the training set :')
 print(train_categories)
 print()
test_categories=sorted(test_df['item_id'].map(items['item_category_id'] ).unique())
 print(str(len(test_categories)) + ' categories in the test set :')
 print(test_categories)
 print()
print('number of supercategories : '+str(item_categories['item_supercategory_name'].nunique()))
print()
for supercat in item_categories['item_supercategory_name'].unique() :
print(supercat + ' : ' +str(item_categories.loc[item_categories['item_supercategory_name']==supercat,'item_category_id'].nunique()))
print(item_categories.loc[item_categories['item_supercategory_name']==supercat,['item_category_id','item_category_name']])
 print()
print('number of consoles : '+str(item_categories['item_category_console'].nunique()))
print()
for console in item_categories['item_category_console'].unique() :
print(console + ' : ' +str(item_categories.loc[item_categories['item_category_console']==console,'item_category_id'].nunique()))
print(item_categories.loc[item_categories['item_category_console']==console,['item_category_id','item_category_name']])
 print()
item_categories
del train_categories, test_categories<set_options>
params = {
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.04,
'learning_rate': 0.008,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}
lg_train = lgb.Dataset(X_train, label= y_train.ravel())
 lg_valid = lgb.Dataset(X_valid, label=y_valid.ravel())
 model = lgb.train(params, lg_train, 10000, valid_sets=[lg_valid],
early_stopping_rounds=1000, verbose_eval=1000)
gc.collect()<drop_column>
pred_val_lgb = model.predict(X_valid, num_iteration=model.best_iteration )
print('number of realisation with negative prices : '+str(( train_df['item_price']<0 ).sum()))
train_df.drop(train_df['item_price'].idxmin() ,axis=0,inplace=True)
 print('number of realisation with negative prices : '+str(( train_df['item_price']<0 ).sum()))<set_options>
pred_test_lgb = model.predict(test_df_new.values, num_iteration=model.best_iteration )
gc.collect()<data_type_conversions>
sub_df_lgb = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_lgb["target"] = pred_test_lgb
sub_df_lgb.to_csv("lgb_submission.csv", index=False)
downcast_dtypes(train_df)
 train_df['month_id']=train_df['month_id'].astype(np.int8)
 train_df['shop_id']=train_df['shop_id'].astype(np.int8)
 downcast_dtypes(shops)
 shops['shop_id']=shops['shop_id'].astype(np.int8)
 downcast_dtypes(items)
 items['item_category_id']=items['item_category_id'].astype(np.int8)
 downcast_dtypes(item_categories)
 item_categories['item_category_id']=item_categories['item_category_id'].astype(np.int8)
 gc.collect()<load_pretrained>
xgb_model = xgb.XGBClassifier(max_depth=2, learning_rate=0.02, n_estimators=3000, booster="gbtree",
objective="binary:logistic", min_child_weight=1, subsample = 0.8,
colsample_bytree = 0.6, random_state=1234 )
create_directory(os.path.join(DATA_FOLDER, 'cleaned'))
train_df.to_pickle(os.path.join(DATA_FOLDER, 'cleaned/train.pkl'))
shops.to_pickle(os.path.join(DATA_FOLDER,'cleaned/shops.pkl'))
items.to_pickle(os.path.join(DATA_FOLDER,'cleaned/items.pkl'))
item_categories.to_pickle(os.path.join(DATA_FOLDER,'cleaned/item_categories.pkl'))<drop_column>
xgb_model.fit(X_train, y_train.ravel() , eval_set=[(X_valid, y_valid.ravel())], eval_metric="auc",
early_stopping_rounds=1000, verbose=True )
del train_df
del shops
del items
del item_categories
del test_df
gc.collect()<define_variables>
pred_test_xgb = xgb_model.predict_proba(test_df_new.values)[:, 1]
reset_variable_space<load_pretrained>
pred_test_xgb_new = xgb_model.predict(test_df_new.values )
train_df=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/train.pkl'))
shops=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/shops.pkl'))
items=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/items.pkl'))
item_categories=pd.read_pickle(os.path.join(DATA_FOLDER,'cleaned/item_categories.pkl'))<drop_column>
sub_df_xgb = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_xgb["target"] = pred_test_xgb
sub_df_xgb.to_csv("xgb_submission.csv", index=False)
 sub_df_xgb.head()
train_df.drop('day_id',axis=1,inplace=True )<data_type_conversions>
cb_model = CatBoostClassifier(iterations=3000, learning_rate=0.02, depth=2, objective="Logloss" )
col_agg = ['month_id','shop_id','item_id']
train_agg=train_df.groupby(col_agg ).agg(item_quantity=pd.NamedAgg(column='item_quantity',aggfunc='sum'))
train_agg.reset_index(inplace=True)
 train_agg['month_id'] = train_agg['month_id'].astype(np.int8)
 train_agg['shop_id'] = train_agg['shop_id'].astype(np.int8)
 train_agg['item_id'] = train_agg['item_id'].astype(np.int16)
 train_agg['item_quantity'].clip(0,20,inplace=True)
 train_agg<data_type_conversions>
cb_model.fit(X_train, y_train.ravel() , eval_set=[(X_valid, y_valid.ravel())],
early_stopping_rounds=1000 )
test_df=pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'test.csv'))
test_df['ID']=test_df['ID'].astype(np.int32)
 test_df['shop_id']=test_df['shop_id'].astype(np.int8)
 test_df['item_id']=test_df['item_id'].astype(np.int16)
 test_df['month_id']=34
test_df=test_df[col_agg]<feature_engineering>
pred_test_cb = cb_model.predict(test_df_new.values, prediction_type="Probability")[:, 1]
pred_test_cb[0:10]
ts = time.time()
train_X = []
for i in range(0,34):
sales = train_df.loc[train_df['month_id']==i]
train_X.append(np.array(list(product([i], sales['shop_id'].unique() , sales['item_id'].unique())) , dtype='int16'))
train_X = pd.DataFrame(np.vstack(train_X), columns=col_agg)
 train_X = pd.concat([train_X,test_df],ignore_index=True)
 train_X['month_id'] = train_X['month_id'].astype(np.int8)
 train_X['shop_id'] = train_X['shop_id'].astype(np.int8)
 train_X['item_id'] = train_X['item_id'].astype(np.int16)
 train_X.sort_values(by=col_agg,inplace=True)
 print('time : ' +str(time.time() - ts))
print()
print(train_X.info(null_counts=True))
del sales
gc.collect()<categorify>
sub_df_cb = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_cb["target"] = pred_test_cb
sub_df_cb.to_csv("cb_submission.csv", index=False)
 sub_df_cb.head()
ts = time.time()
train_X=train_X.join(train_agg.set_index(col_agg),on=col_agg)
 train_X['item_quantity'].fillna(0,inplace=True)
 print('time : ' +str(time.time() - ts))<set_options>
pred_ensemble = 0.4 * pred_test_lgb + 0.3 * pred_test_xgb + 0.3 * pred_test_cb
pred_ensemble[0:10]
del col_agg
gc.collect()<feature_engineering>
sub_df_ensemble = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df_ensemble["target"] = pred_ensemble
sub_df_ensemble.to_csv("ensemble_submission.csv", index=False)
 sub_df_ensemble.head()