Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<data_type_conversions>
FILENO= 24
debug=0
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
def do_count(df, group_cols, agg_type='uint32', show_max=False, show_agg=True):
agg_name='{}count'.format('_'.join(group_cols))
if show_agg:
print("
Aggregating by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols][group_cols].groupby(group_cols ).size().rename(agg_name ).to_frame().reset_index()
df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()<categorify>
def do_countuniq(df, group_cols, counted, agg_type='uint32', show_max=False, show_agg=True):
agg_name= '{}_by_{}_countuniq'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Counting unqiue ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()<categorify>
def do_cumcount(df, group_cols, counted,agg_type='uint16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_cumcount'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Cumulative count by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()
df[agg_name]=gp.values
del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<feature_engineering>
def do_mean(df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_mean'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Calculating mean of ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<feature_engineering>
def do_var(df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_var'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Calculating variance of ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
<define_variables>
if debug:
print('*** debug parameter set: this is a test run for debugging purposes ***')
 def lgb_modelfit_nocv(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc',
feval=None, early_stopping_rounds=50, num_boost_round=3000, verbose_eval=10, categorical_features=None):
lgb_params = {
'boosting_type': 'gbdt',
'objective': objective,
'metric':metrics,
'learning_rate': 0.05,
'num_leaves': 31,
'max_depth': -1,
'min_child_samples': 20,
'max_bin': 255,
'subsample': 0.6,
'subsample_freq': 0,
'colsample_bytree': 0.3,
'min_child_weight': 5,
'subsample_for_bin': 200000,
'min_split_gain': 0,
'reg_alpha': 0,
'reg_lambda': 0,
'nthread': 8,
'verbose': 0,
}
lgb_params.update(params)
 print("preparing validation datasets")
 xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values,
feature_name=predictors,
categorical_feature=categorical_features 
)
 xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values,
feature_name=predictors,
categorical_feature=categorical_features 
)
 del dtrain
gc.collect()
del dvalid
gc.collect()
evals_results = {}
bst1 = lgb.train(lgb_params,
xgtrain,
valid_sets=[ xgvalid],
valid_names=['valid'],
evals_result=evals_results,
num_boost_round=num_boost_round,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=10,
feval=feval)
 print("
Model Report")
 print("bst1.best_iteration: ", bst1.best_iteration)
 print(metrics+":", evals_results['valid'][metrics][bst1.best_iteration-1])
 return(bst1,bst1.best_iteration )
iter_test = env.iter_test()
prior_test_df = None
prev_test_df1 = None<define_search_space>
def DO(frm,to,fileno):
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint8',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32',
}
print('loading train data...',frm,to)
 train_df = pd.read_csv(".. /input/train.csv", parse_dates=['click_time'], skiprows=range(1,frm), nrows=to-frm, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])
 print('loading test data...')
 if debug:
test_df = pd.read_csv(".. /input/test.csv", nrows=100000, parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
 else:
test_df = pd.read_csv(".. /input/test.csv", parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
 len_train = len(train_df)
 train_df=train_df.append(test_df)
 del test_df
gc.collect()
train_df['hour'] = pd.to_datetime(train_df.click_time ).dt.hour.astype('int8')
 train_df['day'] = pd.to_datetime(train_df.click_time ).dt.day.astype('int8')
 train_df = do_next_Click(train_df,agg_suffix='nextClick', agg_type='float32'); gc.collect()
train_df = do_prev_Click(train_df,agg_suffix='prevClick', agg_type='float32'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'channel'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'device', 'os'], 'app'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'day'], 'hour'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'app'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'app'], 'os'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'device'); gc.collect()
train_df = do_countuniq(train_df, ['app'], 'channel'); gc.collect()
train_df = do_cumcount(train_df, ['ip'], 'os'); gc.collect()
train_df = do_cumcount(train_df, ['ip', 'device', 'os'], 'app'); gc.collect()
train_df = do_count(train_df, ['ip', 'day', 'hour']); gc.collect()
train_df = do_count(train_df, ['ip', 'app']); gc.collect()
train_df = do_count(train_df, ['ip', 'app', 'os']); gc.collect()
train_df = do_var(train_df, ['ip', 'app', 'os'], 'hour'); gc.collect()
del train_df['day']
gc.collect()
gc.collect()
print('
Before appending predictors...
 ',sorted(predictors))
target = 'is_attributed'
word= ['app','device','os', 'channel', 'hour']
for feature in word:
if feature not in predictors:
predictors.append(feature)
 categorical = ['app', 'device', 'os', 'channel', 'hour']
print('
After appending predictors...
 ',sorted(predictors))
test_df = train_df[len_train:]
val_df = train_df[(len_train-val_size):len_train]
train_df = train_df[:(len_train-val_size)]
print("
train size: ", len(train_df))
print("
valid size: ", len(val_df))
print("
test size : ", len(test_df))
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id'].astype('int')
 gc.collect()
print("Training...")
 start_time = time.time()
params = {
'learning_rate': 0.10,
'num_leaves': 7,
'max_depth': 3,
'min_child_samples': 100,
'max_bin': 100,
'subsample': 0.7,
'subsample_freq': 1,
'colsample_bytree': 0.9,
'min_child_weight': 0,
'scale_pos_weight':200
} 
(bst,best_iteration)= lgb_modelfit_nocv(params,
train_df,
val_df,
predictors,
target,
objective='binary',
metrics='auc',
early_stopping_rounds=30,
verbose_eval=True,
num_boost_round=1000,
categorical_features=categorical)
 print('[{}]: model training time'.format(time.time() - start_time))
del train_df
del val_df
gc.collect()
ax = lgb.plot_importance(bst, max_num_features=300)
 plt.savefig('test%d.png'%(fileno), dpi=600, bbox_inches='tight')
 plt.show()
print("Predicting...")
 sub['is_attributed'] = bst.predict(test_df[predictors],num_iteration=best_iteration)
 sub.to_csv('sub_it%d.csv'%(fileno),index=False,float_format='%.9f')
 print("done...")
 return sub
<feature_engineering><EOS> 
nrows=184903891-1
nchunk=25000000
val_size=2500000
frm=nrows-85000000
if debug:
frm=0
nchunk=100000
val_size=10000
to=frm+nchunk
sub=DO(frm,to,FILENO )