Kaggle competition: Santander Customer Transaction Prediction
pandarallel.initialize()<load_from_csv>
%matplotlib inline
train_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 item_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 item_cat_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 shop_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv' )<load_from_csv>
santander_data = pd.read_csv('.. /input/train.csv')
 santander_data_test = pd.read_csv('.. /input/test.csv')
 santander_submission = pd.read_csv('.. /input/sample_submission.csv')
 print('Training df shape',santander_data.shape)
 print('Test df shape',santander_data_test.shape )
exch_rate = pd.read_csv('.. /input/1c-params/exchange_rate.csv', sep=',', parse_dates=['<DATE>'])
 exch_rate = exch_rate.drop(['<TICKER>', '<PER>', '<TIME>'], axis=1)
 exch_rate = exch_rate.rename(columns={'<DATE>':'date', '<CLOSE>':'rate'} )<feature_engineering>
label_df = santander_data['target']
train_df['date'] = train_df['date'].parallel_apply(dateutil.parser.parse, dayfirst=True )<categorify>
santander_data.drop(['ID_code','target'], axis=1, inplace=True)
 santander_data_test.drop('ID_code', axis=1, inplace=True)
 santander_data.head(5 )
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x<feature_engineering>
len_train = len(santander_data)
 len_train
item_df['item_name'] = item_df['item_name'].apply(name_correction )<feature_engineering>
merged = pd.concat([santander_data, santander_data_test] )
item_cat_df['new_cats'] =(item_cat_df['item_category_name'] 
.str.split(' - ' ).apply(lambda x: x[0]))<feature_engineering>
idx = features = merged.columns.values[0:200]
for df in [merged]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1 )
item_cat_df['new_subcats'] =(item_cat_df['item_category_name'] 
.str.split(' - ' ).apply(lambda x: x[-1]))<feature_engineering>
train_df = santander_data
X_test = santander_data_test
del santander_data
del santander_data_test
gc.collect()
shop_df['city'] =(shop_df['shop_name'].apply(name_correction)
.str.split(' ' ).apply(lambda x: x[0]))<feature_engineering>
skf_three= StratifiedKFold(n_splits=5, shuffle=True, random_state=2319 )
shop_df.loc[shop_df['city']=='н', 'city'] = 'нижний_новгород'<feature_engineering>
param = {'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1}
shop_df['shop_type'] =(shop_df['shop_name'].apply(name_correction)
.str.split(' ' ).apply(lambda x: x[1] if(len(x)>1)else 'other'))<feature_engineering>
oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(len(X_test))
feats = [f for f in train_df.columns]
for n_fold,(train_idx, valid_idx)in enumerate(skf_three.split(train_df[feats], label_df)) :
trn_data = lgb.Dataset(train_df.iloc[train_idx][feats], label=label_df.iloc[train_idx])
 val_data = lgb.Dataset(train_df.iloc[valid_idx][feats], label=label_df.iloc[valid_idx])
 clf = lgb.train(param, trn_data,1000000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3500)
 oof_preds[valid_idx] = clf.predict(train_df.iloc[valid_idx][feats], num_iteration=clf.best_iteration)
 sub_preds += clf.predict(X_test[feats], num_iteration=clf.best_iteration)/ 5
print('Full AUC score %.6f' % roc_auc_score(label_df, oof_preds))
pred3=sub_preds
<feature_engineering><EOS> 
santander_submission['target'] = pred3
santander_submission.to_csv('submission23.csv', index=False )