Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<define_search_space>
!pip install -U vega_datasets notebook vega
lr = 1e-04<train_model>
%matplotlib inline
pd.options.display.precision = 15
warnings.filterwarnings("ignore")
 %matplotlib inline
alt.renderers.enable('notebook')
 %env JOBLIB_TEMP_FOLDER=/tmp
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()<find_best_params>
def prepare_altair() :

 vega_url = 'https://cdn.jsdelivr.net/npm/vega@' + v5.SCHEMA_VERSION
vega_lib_url = 'https://cdn.jsdelivr.net/npm/vega-lib'
vega_lite_url = 'https://cdn.jsdelivr.net/npm/vega-lite@' + alt.SCHEMA_VERSION
vega_embed_url = 'https://cdn.jsdelivr.net/npm/vega-embed@3'
noext = "?noext"
paths = {
'vega': vega_url + noext,
'vega-lib': vega_lib_url + noext,
'vega-lite': vega_lite_url + noext,
'vega-embed': vega_embed_url + noext
}
workaround = f
return workaround
def add_autoincrement(render_func):
cache = {}
def wrapped(chart, id="vega-chart", autoincrement=True):
if autoincrement:
if id in cache:
counter = 1 + cache[id]
cache[id] = counter
else:
cache[id] = 0
actual_id = id if cache[id] == 0 else id + '-' + str(cache[id])
 else:
if id not in cache:
cache[id] = 0
actual_id = id
return render_func(chart, id=actual_id)
 return wrapped
@add_autoincrement
def render(chart, id="vega-chart"):

 chart_str =
return HTML(
 chart_str.format(
 id=id,
chart=json.dumps(chart)if isinstance(chart, dict)else chart.to_json(indent=None)
)
)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
c_prec = df[col].apply(lambda x: np.finfo(x ).precision ).max()
if c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max and c_prec == np.finfo(np.float32 ).precision:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
@jit
def fast_auc(y_true, y_prob):

 y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
def eval_auc(y_true, y_pred):

 return 'auc', fast_auc(y_true, y_pred), True
def group_mean_log_mae(y_true, y_pred, types, floor=1e-9):

 maes =(y_true-y_pred ).abs().groupby(types ).mean()
return np.log(maes.map(lambda x: max(x, floor)) ).mean()
def train_model_regression(X, X_test, y, params, folds=None, model_type='lgb', eval_metric='mae', columns=None, plot_feature_importance=False, model=None,
verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3):

 columns = X.columns if columns is None else columns
X_test = X_test[columns]
splits = folds.split(X)if splits is None else splits
n_splits = folds.n_splits if splits is None else n_folds
metrics_dict = {'mae': {'lgb_metric_name': 'mae',
'catboost_metric_name': 'MAE',
'sklearn_scoring_function': metrics.mean_absolute_error},
'group_mae': {'lgb_metric_name': 'mae',
'catboost_metric_name': 'MAE',
'scoring_function': group_mean_log_mae},
'mse': {'lgb_metric_name': 'mse',
'catboost_metric_name': 'MSE',
'sklearn_scoring_function': metrics.mean_squared_error}
}
result_dict = {}
oof = np.zeros(len(X))
prediction = np.zeros(len(X_test))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(splits):
if verbose:
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 if type(X)== np.ndarray:
X_train, X_valid = X[columns][train_index], X[columns][valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMRegressor(**params, n_estimators = n_estimators, n_jobs = -1)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],
verbose=verbose, early_stopping_rounds=early_stopping_rounds)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=20000, evals=watchlist, early_stopping_rounds=200, verbose_eval=verbose, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)
 print(f'Fold {fold_n}.{eval_metric}: {score:.4f}.')
 print('')
 y_pred = model.predict(X_test ).reshape(-1,)
 if model_type == 'cat':
model = CatBoostRegressor(iterations=20000, eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,
loss_function=metrics_dict[eval_metric]['catboost_metric_name'])
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 oof[valid_index] = y_pred_valid.reshape(-1,)
 if eval_metric != 'group_mae':
scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
else:
scores.append(metrics_dict[eval_metric]['scoring_function'](y_valid, y_pred_valid, X_valid['type']))
prediction += y_pred
if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
return result_dict
def train_model_classification(X, X_test, y, params, folds, model_type='lgb', eval_metric='auc', columns=None, plot_feature_importance=False, model=None,
verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3, averaging='usual', n_jobs=-1):

 columns = X.columns if columns is None else columns
n_splits = folds.n_splits if splits is None else n_folds
X_test = X_test[columns]
metrics_dict = {'auc': {'lgb_metric_name': eval_auc,
'catboost_metric_name': 'AUC',
'sklearn_scoring_function': metrics.roc_auc_score},
}
result_dict = {}
if averaging == 'usual':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
elif averaging == 'rank':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 if type(X)== np.ndarray:
X_train, X_valid = X[columns][train_index], X[columns][valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMClassifier(**params, n_estimators=n_estimators, n_jobs = n_jobs)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],
verbose=verbose, early_stopping_rounds=early_stopping_rounds)
 y_pred_valid = model.predict_proba(X_valid)[:, 1]
y_pred = model.predict_proba(X_test, num_iteration=model.best_iteration_)[:, 1]
if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=n_estimators, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=verbose, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)
 print(f'Fold {fold_n}.{eval_metric}: {score:.4f}.')
 print('')
 y_pred = model.predict_proba(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=n_estimators, eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,
loss_function=Logloss)
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 if averaging == 'usual':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += y_pred.reshape(-1, 1)
 elif averaging == 'rank':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += pd.Series(y_pred ).rank().values.reshape(-1, 1)
 if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
result_dict['top_columns'] = cols
return result_dict
workaround = prepare_altair()
HTML("".join((
"<script>",
workaround,
"</script>", 
)) )
learn.unfreeze()
learn.lr_find()<train_model>
folder_path = '.. /input/'
train_identity = pd.read_csv(f'{folder_path}train_identity.csv')
 train_transaction = pd.read_csv(f'{folder_path}train_transaction.csv')
 test_identity = pd.read_csv(f'{folder_path}test_identity.csv')
 test_transaction = pd.read_csv(f'{folder_path}test_transaction.csv')
 sub = pd.read_csv(f'{folder_path}sample_submission.csv')
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )
learn.fit_one_cycle(10,slice(1e-4,1e-3))<compute_test_metric>
del train_identity, train_transaction, test_identity, test_transaction
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score<compute_test_metric>
print(f'There are {train.isnull().any().sum() } columns in train dataset with missing values.' )
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta<predict_on_test>
one_value_cols = [col for col in train.columns if train[col].nunique() <= 1]
one_value_cols_test = [col for col in test.columns if test[col].nunique() <= 1]
one_value_cols == one_value_cols_test
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )<prepare_x_and_y>
print(f'There are {len(one_value_cols)} columns in train dataset with one unique value.')
 print(f'There are {len(one_value_cols_test)} columns in test dataset with one unique value.' )
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )<load_from_csv>
train['id_03'].value_counts(dropna=False, normalize=True ).head()
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()<data_type_conversions>
train['id_11'].value_counts(dropna=False, normalize=True ).head()
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )<save_to_csv>
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_addr2'] = train['D15'] / train.groupby(['addr2'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_addr2'] = test['D15'] / test.groupby(['addr2'])['D15'].transform('std' )
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )<save_to_csv>
train[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = train['P_emaildomain'].str.split('.', expand=True)
 train[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = train['R_emaildomain'].str.split('.', expand=True)
 test[['P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3']] = test['P_emaildomain'].str.split('.', expand=True)
 test[['R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']] = test['R_emaildomain'].str.split('.', expand=True )
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )<import_modules>
many_null_cols = [col for col in train.columns if train[col].isnull().sum() / train.shape[0] > 0.9]
many_null_cols_test = [col for col in test.columns if test[col].isnull().sum() / test.shape[0] > 0.9]
import gc
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.metrics import matthews_corrcoef
from operator import itemgetter<load_from_csv>
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
big_top_value_cols_test = [col for col in test.columns if test[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
def get_station_ohe() :
directory = '.. /input/'
trainfile = 'train_date.csv'
testfile = 'test_date.csv'
features = None
subset = None
train_date_part = pd.read_csv(directory + trainfile, nrows=10000)
 date_cols = train_date_part.drop('Id', axis=1 ).count().reset_index().sort_values(by=0, ascending=False)
 date_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])
 date_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()
stations = list([f.split('_')[1] for f in date_cols ])
 stations = sorted(stations,key= lambda x: int(x[1:]))
for i, chunk in enumerate(pd.read_csv(directory + trainfile,
usecols=['Id'] + date_cols,
chunksize=50000,
low_memory=False)) :
if features is None:
features = list(chunk.columns)
 features.remove('Id')
 chunk.columns = ['Id'] + stations
chunk['start_station'] = -1
chunk['end_station'] = -1
for s in stations:
chunk[s] = 1 *(chunk[s] >= 0)
 id_not_null = chunk[chunk[s] == 1].Id
chunk.loc[(chunk['start_station']== -1)&(chunk.Id.isin(id_not_null)) ,'start_station'] = int(s[1:])
 chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])
 subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
for i, chunk in enumerate(pd.read_csv(directory + testfile,
usecols=['Id'] + date_cols,
chunksize=50000,
low_memory=False)) :
chunk.columns = ['Id'] + stations
chunk['start_station'] = -1
chunk['end_station'] = -1
for s in stations:
chunk[s] = 1 *(chunk[s] >= 0)
 id_not_null = chunk[chunk[s] == 1].Id
chunk.loc[(chunk['start_station']== -1)&(chunk.Id.isin(id_not_null)) ,'start_station'] = int(s[1:])
 chunk.loc[chunk.Id.isin(id_not_null),'end_station'] = int(s[1:])
 subset = pd.concat([subset, chunk])
 del chunk
gc.collect()
return subset,stations,date_cols 
<drop_column>
cols_to_drop = list(set(many_null_cols + many_null_cols_test + big_top_value_cols + big_top_value_cols_test + one_value_cols+ one_value_cols_test))
cols_to_drop.remove('isFraud')
 len(cols_to_drop )
station_ohe,stations,date_cols = get_station_ohe()<feature_engineering>
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
station_ohe['path_len'] = station_ohe[stations].sum(axis=1 )<load_from_csv>
cat_cols = ['id_12', 'id_13', 'id_14', 'id_15', 'id_16', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_23', 'id_24', 'id_25', 'id_26', 'id_27', 'id_28', 'id_29',
'id_30', 'id_31', 'id_32', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo', 'ProductCD', 'card4', 'card6', 'M4','P_emaildomain',
'R_emaildomain', 'card1', 'card2', 'card3', 'card5', 'addr1', 'addr2', 'M1', 'M2', 'M3', 'M5', 'M6', 'M7', 'M8', 'M9',
'P_emaildomain_1', 'P_emaildomain_2', 'P_emaildomain_3', 'R_emaildomain_1', 'R_emaildomain_2', 'R_emaildomain_3']
for col in cat_cols:
if col in train.columns:
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
def get_date_features() :
directory = '.. /input/'
trainfile = 'train_date.csv'
for i, chunk in enumerate(pd.read_csv(directory + trainfile,
chunksize=1,
low_memory=False)) :
features = list(chunk.columns)
 del chunk
break
seen = np.zeros(52)
 rv = []
for f in features:
if f == 'Id' or 'S24' in f or 'S25' in f:
rv.append(f)
 continue
station = int(f.split('_')[1][1:])
 if seen[station]:
continue
seen[station] = 1
rv.append(f)
 return rv
usefuldatefeatures = get_date_features()<load_from_csv>
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'TransactionDT', 'TransactionID'], axis=1)
 y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT', 'TransactionID'], axis=1)
 del train
test = test[["TransactionDT", 'TransactionID']]
def create_new_feats() :
directory = '.. /input/'
trainfile = 'train_date.csv'
testfile = 'test_date.csv'
features = None
subset = None
for i, chunk in enumerate(pd.read_csv(directory + trainfile,
usecols=usefuldatefeatures,
chunksize=50000,
low_memory=False)) :
if features is None:
features = list(chunk.columns)
 features.remove('Id')
 week_duration = 1679
df_mindate_chunk = chunk[['Id']].copy()
df_mindate_chunk['mindate'] = chunk[features].min(axis=1 ).values
df_mindate_chunk['maxdate'] = chunk[features].max(axis=1 ).values
df_mindate_chunk['duration'] = df_mindate_chunk['maxdate'] - df_mindate_chunk['mindate']
df_mindate_chunk['part_week'] =(( df_mindate_chunk['mindate'].values * 100)% week_duration ).astype(np.int64)
 df_mindate_chunk['min_time_station'] = chunk[features].idxmin(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 df_mindate_chunk['max_time_station'] = chunk[features].idxmax(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 if subset is None:
subset = df_mindate_chunk.copy()
else:
subset = pd.concat([subset, df_mindate_chunk])
 del chunk
gc.collect()
for i, chunk in enumerate(pd.read_csv(directory + testfile,
usecols=usefuldatefeatures,
chunksize=50000,
low_memory=False)) :
df_mindate_chunk = chunk[['Id']].copy()
df_mindate_chunk['mindate'] = chunk[features].min(axis=1 ).values
df_mindate_chunk['maxdate'] = chunk[features].max(axis=1 ).values
df_mindate_chunk['duration'] = df_mindate_chunk['maxdate'] - df_mindate_chunk['mindate']
df_mindate_chunk['part_week'] =(( df_mindate_chunk['mindate'].values * 100)% week_duration ).astype(np.int64)
 df_mindate_chunk['min_time_station'] = chunk[features].idxmin(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 df_mindate_chunk['max_time_station'] = chunk[features].idxmax(axis = 1 ).apply(lambda s: int(s.split('_')[1][1:])if s is not np.nan else -1)
 subset = pd.concat([subset, df_mindate_chunk])
 del chunk
gc.collect()
return subset<feature_engineering>
def clean_inf_nan(df):
return df.replace([np.inf, -np.inf], np.nan)
 X = clean_inf_nan(X)
 X_test = clean_inf_nan(X_test )
new_features = create_new_feats()<sort_values>
gc.collect()
new_features.sort_values(by=['mindate', 'Id'], inplace=True )<feature_engineering>
n_fold = 5
folds = TimeSeriesSplit(n_splits=n_fold)
 folds = KFold(n_splits=5 )
new_features['mindate_id_diff'] = new_features.Id.diff()<concatenate>
params = {'num_leaves': 256,
'min_child_samples': 79,
'objective': 'binary',
'max_depth': 13,
'learning_rate': 0.03,
"boosting_type": "gbdt",
"subsample_freq": 3,
"subsample": 0.9,
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3,
'reg_lambda': 0.3,
'colsample_bytree': 0.9,
}
result_dict_lgb = train_model_classification(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb', eval_metric='auc', plot_feature_importance=True,
verbose=500, early_stopping_rounds=200, n_estimators=5000, averaging='usual', n_jobs=-1 )
midr = np.full_like(new_features.mindate_id_diff.values, np.nan )<feature_engineering>
sub['isFraud'] = result_dict_lgb['prediction']
sub.to_csv('submission.csv', index=False )
<feature_engineering><EOS> 
pd.DataFrame(result_dict_lgb['oof'] ).to_csv('lgb_oof.csv', index=False )