Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<predict_on_test>
warnings.filterwarnings('ignore')
 init_notebook_mode()
val_pred_pred = final_model.predict(val_predictions)
 test_pred_pred = final_model.predict(test_predictions )<compute_test_metric>
%time
gc.enable()
seed = 127
np.random.seed(seed )
print('Rmse:', np.sqrt(mean_squared_error(y_valid, val_pred_pred)) )<save_to_csv>
%time
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )
submission = pd.DataFrame(test_df['ID'], columns=['ID'])
 submission['item_cnt_month'] = test_pred_pred.clip(0., 20.)
submission.to_csv('submission_stack.csv', index=False)
 submission.head(10 )<set_options>
def missingcount(count, samples):

 return(int(count)/samples)* 100
def missingPercent(df):

 count_df = pd.DataFrame(df.isnull().sum(axis=0 ).reset_index())
 count_df.columns = ['FEATURE', 'MISSING_COUNT']
count_df["MISSING_%"] = count_df["MISSING_COUNT"].map(lambda x: missingcount(x, df.shape[0]))
return count_df.set_index('FEATURE' ).T
pd.set_option('display.max_rows', 500)
 %matplotlib inline 
<data_type_conversions>
missingPercent(train_df )
def downcast_dtypes(df):
float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype in ["int64", "int32"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df<define_variables>
labels=[str(x)for x in list(train_df.target.unique())]
values=[(len(train_df[train_df.target == 0])/len(train_df)) *100,(len(train_df[train_df.target > 0])/len(train_df)) *100]
sample_percentage_per_class(labels,values )
RAW_DATA_FOLDER = '/kaggle/input/competitive-data-science-predict-future-sales/'
DATA_FOLDER = '/kaggle/working/'<set_options>
rf = RandomForestClassifier(max_depth=10, verbose=3, n_estimators=20)
 parameters = {'n_estimators': [120, 100], 'max_depth':[3,5,]}
grid = GridSearchCV(rf, parameters, cv=3, n_jobs=-1, verbose=3, scoring=make_scorer(roc_auc_score))
loaded=%who_ls
loaded.append('loaded' )<drop_column>
df_train_FS=train_df
grid.fit(df_train_FS.drop(["target",'ID_code'], axis=1 ).values, df_train_FS.target.values )
all_vars=%who_ls
all_vars.append('all_vars')
 for var in list(set(all_vars)-set(loaded)) :
exec('del '+var)
 del var<set_options>
grid.best_score_
%macro reset_variable_space 6
loaded.append('reset_variable_space' )<load_from_csv>
X=df_train_FS.drop(["target",'ID_code'], axis=1)
 Y=df_train_FS.target
train_df = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'sales_train.csv'))
items = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'items.csv'))
item_categories = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'item_categories.csv'))
shops = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'shops.csv'))
test_df = pd.read_csv(os.path.join(RAW_DATA_FOLDER, 'test.csv'))<count_unique_values>
x_train,x_test,y_train, y_test = train_test_split(X, Y, random_state=41)
 rfmodel=RandomForestClassifier(max_depth=10, n_estimators=20,verbose=0 ).fit(x_train, y_train )
print('train_df :' + str(train_df.shape))
print()
train_df.info(null_counts=True)
 print('_'*40)
 print()
print('test set :' + str(test_df.shape))
print()
test_df.info(null_counts=True)
 print('_'*40)
 print('_'*40)
 print()
print('number of different items in training set : '+str(train_df['item_id'].nunique())+ ' / ' + str(items.item_id.nunique()))
print('number of different items in test set : '+str(test_df['item_id'].nunique())+ ' / ' + str(items.item_id.nunique()))
print('number of different items ONLY in test set : '+str(test_df.loc[~test_df['item_id'].isin(train_df['item_id']),'item_id'].nunique()))
print()
print('number of different shops in training set : '+str(train_df['shop_id'].nunique())+ ' / ' + str(shops.shop_id.nunique()))
print('number of different shops in test set : '+str(test_df['shop_id'].nunique())+ ' / ' + str(shops.shop_id.nunique()))<rename_columns>
perm = PermutationImportance(rfmodel, random_state=41, scoring=make_scorer(roc_auc_score)).fit(x_test,y_test )
train_df.rename({'date_block_num':'month_id', 'item_cnt_day':'item_quantity'},axis=1,inplace=True )<data_type_conversions>
eli5.show_weights(perm, feature_names = x_test.columns.tolist() , top=80 )
train_df['date']=pd.DataFrame(pd.to_datetime(train_df['date'],format="%d.%m.%Y"))<feature_engineering>
y = train_df.target
train_df= train_df.iloc[:, 2:]
train_df['day_id']=(train_df['date']-train_df['date'].min() ).dt.days<drop_column>
ids = test_df.ID_code.to_frame()
test_df.drop('ID_code', inplace=True, axis=1 )
print('percentage of realisations that represent returned articles : ' +str(round(( train_df['item_quantity']<0 ).sum() /(train_df['item_quantity']>=0 ).sum() *100,2)) + ' %')
 train_df.drop(train_df[train_df['item_quantity']<0].index,axis=0,inplace=True )<count_duplicates>
%%time
idx = features = train_df.columns.values[2:202]
for df in [test_df, train_df]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )
print('shop 39 : ' + shops.shop_name[39])
 print('shop 40 : ' + shops.shop_name[40])
 print()
tmp=train_df[(train_df['shop_id']==39)|(train_df['shop_id']==40)].drop('month_id',axis=1)
 print('
print(tmp['shop_id'].value_counts())
 print()
print('
tmpa=train_df[(train_df['shop_id']==39)].drop('month_id',axis=1)
 tmpb=train_df[(train_df['shop_id']==40)].drop('month_id',axis=1)
 print('duplicates date/item(shop 39): ' + str(tmpa.duplicated(subset=['date','item_id'],keep=False ).sum()))
print('duplicates date/item(shop 40): ' + str(tmpb.duplicated(subset=['date','item_id'],keep=False ).sum()))
print('...but the reunion of both has many...')
 print('duplicates date/item : ' + str(tmp.duplicated(subset=['date','item_id'],keep=False ).sum()))
print('duplicates date/item/price : ' + str(tmp.duplicated(subset=['date','item_id','item_price'],keep=False ).sum()))
print('duplicates date/item/price/quantity : ' + str(tmp.duplicated(subset=['date','item_id','item_price','item_quantity'],keep=False ).sum()))
print('...so all duplicates are exactly paired between the two shops(39,40)!
print()
print('
print('shop 39 : ' + tmp[tmp.shop_id==39].date.min().strftime("%d.%m.%Y")+ ' --- ' + tmp[tmp.shop_id==39].date.max().strftime("%d.%m.%Y"))
print('shop 40 : ' + tmp[tmp.shop_id==40].date.min().strftime("%d.%m.%Y")+ ' --- ' + tmp[tmp.shop_id==40].date.max().strftime("%d.%m.%Y"))
print()
print('
print('
tmpr=tmp[tmp['date']<tmpb['date'].max() ]
print(tmpr['shop_id'].value_counts())
 print()
print('
pa=np.zeros(22170)
 pa[tmpa['item_id'].unique() ]=1
pa.astype(int)
 pb=np.zeros(22170)
 pb[tmpb['item_id'].unique() ]=1
pb.astype(int)
 print('number of different items sold in shop 39 : ' + str(sum(pa ).astype(int)))
 print('number of different items sold in shop 40 : ' + str(sum(pb ).astype(int)))
 print('number of different items sold in both shops : ' + str(sum(pa*pb ).astype(int)))
 print()
print("
print("
print()
del tmp,tmpa,tmpb,tmpr,pa,pb<drop_column>
sc = StandardScaler(seed, with_std=True)
 sc.fit(train_df)
 train_df= sc.transform(train_df)
 test_df = sc.transform(test_df )
shops.drop(0,axis=0,inplace=True)
 shops.drop(1,axis=0,inplace=True)
 shops.drop(11,axis=0,inplace=True)
 shops.drop(40,axis=0,inplace=True)
 train_df.loc[train_df['shop_id']==0,'shop_id']=57
train_df.loc[train_df['shop_id']==1,'shop_id']=58
train_df.loc[train_df['shop_id']==11,'shop_id']=10
train_df.loc[train_df['shop_id']==40,'shop_id']=39<groupby>
params = {
'tree_learner': 'serial',
'objective': 'binary',
'learning_rate': 0.00742,
'num_leaves': 3,
'bagging_freq': 5,
'bagging_fraction': 0.33,
'boost_from_average':True,
'boost': 'gbrt',
'feature_fraction': 0.04,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 85,
'min_sum_hessian_in_leaf': 0.80,
'num_threads': 16,
'verbosity': 1
}
train_df=train_df.groupby(list(train_df.columns.drop('item_quantity')) ,as_index=False ).sum()
train_df.info(null_counts=True )<count_duplicates>
num_round = 250000
folds = StratifiedKFold(n_splits=7, shuffle=True, random_state=seed)
 lstCV=folds.split(train_df, y )
print('Remaining duplicates in the training set : ' + str(train_df.duplicated(subset=['date','shop_id','item_id','item_price'],keep=False ).sum()))<count_unique_values>
train_pred = np.zeros(len(train_df))
test_pred = np.zeros(len(test_df))
shops['city'] = shops['shop_name'].str.extract('(\S+)\s', expand=False)
 print('number of cities in the dataset : ' + str(shops.city.nunique()))
print('number of null cities : ' + str(shops.city.isnull().sum()))
shops.city.unique()<categorify>
%time
for fold_,(trn_idx, val_idx)in enumerate(lstCV):
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df[trn_idx], label=y[trn_idx])
 val_data = lgb.Dataset(train_df[val_idx], label=y[val_idx])
 clf = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3500)
 train_pred[val_idx] = clf.predict(train_df[val_idx], num_iteration=clf.best_iteration)
 test_pred += clf.predict(test_df, num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(y, train_pred)) )
shops['city_id']=pd.factorize(shops['city'])[0]<merge>
sub_df = pd.DataFrame({"ID_code": ids["ID_code"]})
 sub_df["target"] = test_pred
sub_df.to_csv("submission.csv", index=False )