Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<train_model>
import os
import gc
from operator import methodcaller
import numpy as np
import pandas as pd
from sklearn.preprocessing import RobustScaler as RobustScaler
from scipy.stats import skew, norm
from scipy.stats import boxcox_normmax, boxcox
from scipy.special import boxcox1p
import lightgbm as lgb
from lightgbm import LGBMClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
import matplotlib.pyplot as plt
import seaborn as sns
print('start handle attempt_no...' )<data_type_conversions>
g_enable_log = True
def log(log_str):
if g_enable_log:
print(log_str)
 def g() :
return gc.collect()
def delete(*obj_list):
for obj in obj_list:
del obj
gc_cnt = g()
if gc_cnt > 0:
log("unreachable_obj_found: {}".format(gc_cnt))
def init_robust_boxcox() :
def robust_boxcox(data:pd.Series, lmbda=None):
if lmbda is None:
transformed, lmbda = boxcox(1 + data)
 if lmbda <= 0:
lmbda = 0
transformed = np.log1p(data)
 return(transformed, lmbda)
 else:
if lmbda <= 0:
lmbda = 0
transformed = np.log1p(data)
 return(transformed, lmbda)
 else:
transformed = boxcox(1 + data, lmbda)
 return(transformed, lmbda)
 def robust_inv_boxcox(data:pd.Series, lmbda):
if lmbda <= 0:
return np.expm1(data)
 else:
transformed = inv_boxcox(data, lmbda)- 1
return transformed
return robust_boxcox, robust_inv_boxcox
robust_boxcox, robust_inv_boxcox = init_robust_boxcox()
@contextmanager
def timer_memory(name):
t0 = time.time()
yield
print(f'Memory: {(psutil.Process(os.getpid() ).memory_info().rss/2**30):.02f}GB')
 print(f'{name} done in {time.time() -t0:.0f}s' )
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()<data_type_conversions>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )<data_type_conversions>
g_scaler_dict = {}
g_boxcox_lmbda_dict = {}
def box_cox_trans(df, fea_name, sv_policy):

 df[fea_name] = df[fea_name].astype('float64'); g()
df.loc[df[fea_name] <= 0,(fea_name)] = 0.000001
if sv_policy == 'new_and_save':
df[fea_name], lmbda = robust_boxcox(df[fea_name]); g()
g_boxcox_lmbda_dict[fea_name] = lmbda
elif sv_policy == 'reuse':
lmbda = g_boxcox_lmbda_dict[fea_name]
df[fea_name], lmbda = robust_boxcox(df[fea_name], lmbda); g()
else:
df[fea_name], lmbda = robust_boxcox(df[fea_name]); g()
log("\tboxcox lmbda: {}".format(lmbda))
return df[fea_name].astype('float32')
 def scaler_trans(df, fea_name, sv_policy, scaler):

 df[fea_name] = df[fea_name].astype('float64'); g()
if sv_policy == 'new_and_save':
df[[fea_name]] = scaler.fit_transform(df[[fea_name]]); g()
g_scaler_dict[fea_name] = scaler
elif sv_policy == 'reuse':
df[[fea_name]] = g_scaler_dict[fea_name].transform(df[[fea_name]]); g()
else:
df[[fea_name]] = scaler.fit_transform(df[[fea_name]]); g()
return df[fea_name].astype('float32' )
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32' )<data_type_conversions>
def add_features(df, is_boxcox=False, is_scaler=False, save_transformer='reuse'):

 sv = save_transformer
bc = is_boxcox
s = None;
if is_scaler:
s = RobustScaler()
df = add_grp_nxt_clk_intv(df, ['ip','os','device','app'], bc=None,scl=s,sv=sv);g()
df = add_grp_count(df, ['dd','hh','app','channel'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['ip','device','os'], 'app', 'cumcount', 'uint32', bc=bc,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','device'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['ip'], 'channel', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
df = add_grp_stat(df, ['ip'], 'app', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
df = add_grp_nxt_clk_intv(df, ['ip','channel'], bc=None,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','hh','device'], bc=bc,scl=s,sv=sv);g()
df = add_grp_nxt_clk_intv(df, ['ip','app','channel'], bc=None,scl=s,sv=sv);g()
df = add_grp_count(df, ['app','channel'], bc=bc,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','app'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['ip','hh'], 'app', 'nunique','uint16',bc=bc,scl=s,sv=sv);g()
df = add_grp_count(df, ['ip','hh','app'], bc=bc,scl=s,sv=sv);g()
df = add_grp_stat(df, ['app'], 'channel', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
df = add_grp_stat(df, ['hh','app'], 'channel', 'nunique', 'uint16',bc=None,scl=None,sv=sv);g()
return df
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32' )<feature_engineering>
g_categorical_features = ['app', 'device', 'os', 'channel', 'hh']
g_non_train_columns = ['click_time', 'dd', 'ip']
def get_file_spec(is_test_file):
dtypes = {'ip':'uint32','app':'uint16','device':'uint8','os':'uint16',
'channel':'uint16','is_attributed':'int8','click_id':'int32'}
date_columns = ['click_time']
test_file_columns = ['click_time','ip','app','device','os','channel','click_id']
train_file_columns = ['click_time','ip','app','device','os','channel','is_attributed']
if is_test_file:
return dtypes, date_columns, test_file_columns
else:
return dtypes, date_columns, train_file_columns
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True )<data_type_conversions>
def read_data_file(file_path, is_test_file):
print('read file [is_test_file={}]: {}'.format(is_test_file, file_path))
dtypes, date_columns, file_columns = get_file_spec(is_test_file)
 df = pd.read_csv(file_path, parse_dates=date_columns,usecols=file_columns,dtype=dtypes)
 df['dd'] = pd.to_datetime(df.click_time ).dt.day.astype('uint8')
 df['hh'] = pd.to_datetime(df.click_time ).dt.hour.astype('uint8')
 g()
return df
def process_ip_bucket(ip_bucket, is_down_sample, majority_multiply, tsfm_sv_policy):
print('---------- process bucket: {} -----------'.format(ip_bucket))
train_file_path = '/kaggle/input/train/train_{}.csv'.format(ip_bucket)
 test_file_path = '/kaggle/input/test/test_{}.csv'.format(ip_bucket)
 df_test = read_data_file(test_file_path, is_test_file=True)
 df_train_vldt = read_data_file(train_file_path, is_test_file=False)
 print('read files: {}; {}'.format(train_file_path, test_file_path))
df_train_vldt['click_id'] = -1
df_test['is_attributed'] = -1
df_full = pd.concat([df_train_vldt, df_test], sort=False)
 train_vldt_len = len(df_train_vldt)
 delete(df_train_vldt, df_test)
 ', df_full.head(n=5000000 ).nunique()); g()
print('add features: ')
 df_full = add_features(df_full, save_transformer=tsfm_sv_policy); g()
print('drop non-training-columns: {}'.format(g_non_train_columns))
df_full.drop(g_non_train_columns, axis=1, inplace=True); g()
print('split data set: ')
 vldt_len = min(g_vldt_set_size // g_ip_bkt_num, train_vldt_len // 5)
 df_train = df_full[: train_vldt_len - vldt_len]
df_vldt = df_full[train_vldt_len - vldt_len : train_vldt_len]
df_test = df_full[train_vldt_len :]
print('drop temporary columns: ')
 print('\tdf_train['click_id'].value_counts() ={}'.format(df_train['click_id'].value_counts()))
print('\tdf_vldt['click_id'].value_counts() ={}'.format(df_vldt['click_id'].value_counts()))
print('\tdf_test['is_attributed'].value_counts() ={}'.format(df_test['is_attributed'].value_counts()))
df_train.drop(['click_id'], axis=1, inplace=True)
 df_vldt.drop(['click_id'], axis=1, inplace=True)
 df_test.drop(['is_attributed'], axis=1, inplace=True)
 print('shape: vldt={}; test={}'.format(df_vldt.shape, df_test.shape))
print('shape: train(before downsample)={}'.format(df_train.shape))
g()
if is_down_sample == True:
print('down_sample: majority_multiply={}'.format(majority_multiply))
df_train = random_down_sample(df_train, majority_multiply)
 print('gc.collect(warnning is OK): ')
 g()
print('shape(after downsample)={}'.format(df_train.shape))
return df_train, df_vldt, df_test
train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32' )<data_type_conversions>
def prep_data_set_full_data() :
log_template="append bkt {}: train.shape={}; vldt.shape={}; test.shape={}"
df_train, df_vldt, df_test = process_ip_bucket(
 ip_bucket=0, is_down_sample=g_is_down_sample,
majority_multiply=g_majority_multiply,
tsfm_sv_policy='new_and_save')
 print(log_template.format(0, df_train.shape, df_vldt.shape, df_test.shape))
for bkt_id in range(1,g_ip_bkt_num):
train_bkt, vldt_bkt, test_bkt = process_ip_bucket(
 ip_bucket=bkt_id, is_down_sample=g_is_down_sample,
majority_multiply=g_majority_multiply,
tsfm_sv_policy='reuse'); g()
df_train = df_train.append(train_bkt, ignore_index=True); g()
df_vldt = df_vldt.append(vldt_bkt, ignore_index=True); g()
df_test = df_test.append(test_bkt, ignore_index=True); g()
delete(train_bkt, vldt_bkt, test_bkt)
 print(log_template.format(bkt_id, df_train.shape, df_vldt.shape, df_test.shape))
df_test.set_index('click_id', drop=True, inplace=True)
 df_test.sort_index(axis=0, inplace=True)
 return df_train, df_vldt, df_test
def feature_target_split(df, target_col_name, inplace=False):
y = df[target_col_name]; g()
if True == inplace:
df.drop(target_col_name, axis=1, inplace=True); g()
X = df
else:
X = df.drop(target_col_name, axis=1, inplace=True); g()
return X, y
def prep_feature_target_full_data() :
df_train, df_vldt, df_test = prep_data_set_full_data()
X_train, y_train = feature_target_split(df_train, target_col_name = 'is_attributed', inplace = True)
 X_vldt, y_vldt = feature_target_split(df_vldt, target_col_name = 'is_attributed', inplace = True)
 delete(df_train, df_vldt)
 print('----: prep_feature_target_full_data :-----')
 print('-- X_train={}; y_train={}; X_vldt={}; y_vldt={}; df_test={}'.format(
 X_train.shape, y_train.shape, X_vldt.shape, y_vldt.shape, df_test.shape))
print('-- features :', X_train.columns.values.tolist())
 print('-- categorical :', g_fit_params['categorical_feature'])
 print('-- y_train.value_counts
:', y_train.value_counts())
 print('-- y_vldt.value_countsl
:', y_vldt.value_counts())
 return X_train, y_train, X_vldt, y_vldt, df_test
<data_type_conversions>
def default_model() :
lgb_default = LGBMClassifier()
return lgb_default.set_params(
 objective = 'binary',
metric = 'auc',
boosting_type = 'gbdt',
verbose = 1,
nthread = 4,
iid = False,
two_round = True 
)
 def gbtd_base_001() :
return default_model().set_params(
 subsample = 0.8,
subsample_freq = 1,
subsample_for_bin = 200000,
colsample_bytree = 0.8,
learning_rate = 0.08,
num_leaves = 105,
max_depth = 7,
min_split_gain = 0.3,
max_bin = 255,
reg_alpha = 0.3,
n_estimators = 2500 
)
 g_base_models = {
'gbdt_base_001' : gbtd_base_001()
}
g_search_params = {
'gbdt_base_001_exp_002' : {'min_sum_hessian_in_leaf':[0.001, 0.01, 0.05, 0.1]},
'gbdt_base_001_exp_005' : {'min_split_gain':[0.3, 0.4, 0.5]},
'gbdt_base_001_exp_010' : {'num_leaves':[90,105], 'min_split_gain':[0.3,0.4]}
}
def update_data_balancing_param(lgb_model, y_value_counts, majority_val=0, minority_val=1):
print('y_value_counts:
{}'.format(y_value_counts))
if g_scale_pos_weight is None:
print(f'use global config: scale_pos_weight={g_scale_pos_weight}')
 lgb_model.set_params(scale_pos_weight = g_scale_pos_weight)
 else :
unbalance_degree = y_value_counts[majority_val] / y_value_counts[minority_val]
print(f'majority_count/minority_count={unbalance_degree}')
 if unbalance_degree > 1.5 or unbalance_degree < 0.66:
print(f'unbalance_dgree')
 lgb_model.set_params(is_unbalance = True)
 return lgb_model
def get_model_and_search_params(base_model_id, search_params_id):
return g_base_models[base_model_id], g_search_params[search_params_id]
', search_params )
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
<data_type_conversions>
g_fit_params = {
'categorical_feature' : g_categorical_features,
'early_stopping_rounds' : 25,
'verbose' : 10,
'eval_metric' : 'auc'
}
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )<data_type_conversions>
g_base_model = g_base_models['gbdt_base_001']
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )<data_type_conversions>
with timer_memory('prep_feature_target_full_data'):
X_train, y_train, g_X_vldt, g_y_vldt, g_df_test = prep_feature_target_full_data()
<data_type_conversions>
with timer_memory('fit_model'):
g_model_fitted = fit_model(X_train, y_train, g_X_vldt, g_y_vldt, g_base_model )
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )<feature_engineering>
def predict_and_submit(model_fitted, num_iteration):
sub = pd.DataFrame()
sub['click_id'] = g_df_test.index
sub['click_id'] = sub['click_id'].astype('int')
 pred_prob = model_fitted.predict_proba(X=g_df_test, num_iteration=num_iteration)
 sub['is_attributed'] = pred_prob[:,1].reshape(-1,1)
 sub.to_csv('submit.csv', index=False, float_format='%.9f')
 print(f'num_iteration: {num_iteration}')
 print(f'sub.shape: {sub.shape}')
 return sub
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']<data_type_conversions>
with timer_memory('predict_and_sumit'):
submit = predict_and_submit(g_model_fitted, g_model_fitted.best_iteration_)
 submit.head()
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )<data_type_conversions>
g_y_pred_proba_vldt = g_model_fitted.predict_proba(X=g_X_vldt, num_iteration=g_model_fitted.best_iteration_)[:,1]
g_y_pred_label_vldt = g_model_fitted.predict(X=g_X_vldt, num_iteration=g_model_fitted.best_iteration_)
 g_y_vldt.value_counts()
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
<set_options>
report = classification_report(g_y_vldt, g_y_pred_label_vldt, target_names=['is_not_attributed','is_attributed'])
 print(report )
del cum
gc.collect()<data_type_conversions>
df_plot = random_down_sample(
 g_X_vldt.merge(g_y_vldt, left_index=True, right_index=True),
majority_multiply=g_majority_multiply, target_col_name='is_attributed',
minority_val = 1, majority_val = 0); g()
df_plot['is_attributed'].value_counts()
<data_type_conversions><EOS> 
def grid_search(X_train, y_train, X_vldt, y_vldt, base_model, param_grid):
estimator_lst,grid_point_lst,precision_lst,recall_lst,f1_lst,auc_lst=[],[],[],[],[],[]
for grid_point in list(ParameterGrid(param_grid)) :
print(f'----- grid_point:
{grid_point}')
 base_model = base_model.set_params(**grid_point)
 base_model = base_model.set_params(silent = True)
 base_model = base_model.set_params(verbosity = 0)
 print('update_data_balancing_param:')
 base_model = update_data_balancing_param(base_model, y_train.value_counts())
 print(f'model parameters:
{base_model.get_params() }')
 print('cat_fea: {}'.format(g_fit_params['categorical_feature']))
print('fea: {}'.format(X_train.columns))
print(type(X_train.columns))
print(X_train.columns.get_loc('app'))
cat_fea_indices = list(map(lambda col:X_train.columns.get_loc(col), g_fit_params['categorical_feature']))
fit_params_copy = g_fit_params.copy()
fit_params_copy['categorical_feature'] = cat_fea_indices
fit_params_copy['verbose'] = -1
print(f'fit_params_copy:
{fit_params_copy}')
 fitted = base_model.fit(
 X_train, y_train, eval_set = [(X_vldt, y_vldt)], **fit_params_copy); g()
y_pred = fitted.predict(X_vldt)
 grid_point_lst.append(grid_point)
 estimator_lst.append(fitted)
 precision_lst.append(precision_score(y_vldt, y_pred))
recall_lst.append(recall_score(y_vldt, y_pred))
f1_lst.append(f1_score(y_vldt, y_pred))
auc_lst.append(roc_auc_score(y_vldt, y_pred))
score_dict = pd.DataFrame(data={
'precision':precision_lst,
'recall':recall_lst,
'f1':f1_lst,
'auc':auc_lst
}, index=grid_point_lst)
 estimator_dict = pd.DataFrame(data={'estimator':estimator_lst}, index=grid_point_lst)
 return score_dict, estimator_dict
def run_grid_search(base_model_id, exp_params_id):
with timer_memory('grid_search %'.format(exp_params_id)) :
lgb_model, search_params = get_model_and_search_params(base_model_id, exp_params_id)
 score_dict, estimator_dict = grid_search(X_train, y_train, g_X_vldt, g_y_vldt, lgb_model, search_params)
 return score_dict