Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<categorify>
FILENO= 1
debug=0
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
<data_type_conversions>
def do_agg(df, group_cols, agg_type='uint8', show_max=False, show_agg=True):
agg_name='{}_agg'.format('_'.join(group_cols))
if show_agg:
print("
Aggregating by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols].groupby(group_cols ).size().rename(agg_name ).to_frame().reset_index()
df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
<drop_column>
def do_count(df, group_cols, counted, agg_type='uint8', show_max=False, show_agg=True):
agg_name= '{}_by_{}_count'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Counting ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].count().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
del content_agg
del bundle_agg
del part_agg
gc.collect()<define_variables>
def do_countuniq(df, group_cols, counted, agg_type='uint8', show_max=False, show_agg=True):
agg_name= '{}_by_{}_countuniq'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Counting unqiue ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
<init_hyperparams>
def do_cumcount(df, group_cols, counted,agg_type='uint16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_cumcount'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Cumulative count by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()
df[agg_name]=gp.values
del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df)) 
<drop_column>
def do_mean(df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_mean'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Calculating mean of ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
def do_var(df, group_cols, counted, agg_type='float16', show_max=False, show_agg=True):
agg_name= '{}_by_{}_var'.format(( '_'.join(group_cols)) ,(counted))
if show_agg:
print("
Calculating variance of ", counted, " by ", group_cols , '...and saved in', agg_name)
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 predictors.append(agg_name)
 gc.collect()
return(df )
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.Booster(model_file='.. /input/savemodel/lgbm_model.txt')
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect() 
<choose_model_class>
if debug:
print('*** debug parameter set: this is a test run for debugging purposes ***')
 def DO(frm,to,fileno):
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint8',
'os' : 'uint8',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32',
}
print('loading train data...',frm,to)
 train_df = pd.read_csv(".. /input/train.csv", parse_dates=['click_time'], skiprows=range(1,frm), nrows=to-frm, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])
 print('
loading test data...
 ')
 if debug:
test_df = pd.read_csv(".. /input/test.csv", nrows=100000, parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
 else:
test_df = pd.read_csv(".. /input/test.csv", parse_dates=['click_time'], dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])
 len_train = len(train_df)
 train_df=train_df.append(test_df)
 del test_df
gc.collect()
print('
Extracting new features...
 ')
 train_df['hour'] = pd.to_datetime(train_df.click_time ).dt.hour.astype('int8')
 train_df['day'] = pd.to_datetime(train_df.click_time ).dt.day.astype('int8')
 train_df['minute'] = pd.to_datetime(train_df.click_time ).dt.minute.astype('int8')
 train_df = do_next_Click(train_df,agg_suffix='nextClick', agg_type='uint32'); gc.collect()
train_df = do_prev_Click(train_df,agg_suffix='prevClick', agg_type='uint32'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'channel'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'os'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'hour'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'minute'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'app'); gc.collect()
train_df = do_countuniq(train_df, ['ip'], 'device'); gc.collect()
train_df = do_countuniq(train_df, ['app'], 'channel'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'device'], 'channel'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'device', 'os'], 'channel'); gc.collect()
train_df = do_countuniq(train_df, ['ip','app', 'os'], 'channel'); gc.collect()
train_df = do_countuniq(train_df, ['ip', 'device', 'os'], 'app'); gc.collect()
train_df = do_countuniq(train_df, ['ip','app', 'os', 'device'], 'channel'); gc.collect()
train_df = do_cumcount(train_df, ['ip'], 'os'); gc.collect()
train_df = do_cumcount(train_df, ['ip', 'device', 'os'], 'app'); gc.collect()
train_df = do_count(train_df, ['ip','app', 'os'], 'channel'); gc.collect()
train_df = do_count(train_df, ['ip','app', 'os', 'device'], 'channel'); gc.collect()
train_df = do_agg(train_df, ['ip', 'day', 'hour']); gc.collect()
train_df = do_agg(train_df, ['ip', 'app']); gc.collect()
train_df = do_agg(train_df, ['ip', 'app', 'os']); gc.collect()
del train_df['day']
gc.collect()
del train_df['minute']
gc.collect()
print('
Before appending predictors...
 ', predictors)
 print('
Before appending predictors length...', len(predictors))
target = 'is_attributed'
word= ['app','device','os', 'channel', 'hour']
for feature in word:
if feature not in predictors:
predictors.append(feature)
 predictors_sorted= sorted(list(set(predictors)))
 categorical = ['app', 'device', 'os', 'channel', 'hour']
print('
After appending predictors...
 ',predictors_sorted)
 print('
After appending predictors length...', len(predictors_sorted))
test_df = train_df[len_train:]
gc.collect()
val_df = train_df[(len_train-val_size):len_train]
gc.collect()
train_df = train_df[:(len_train-val_size)]
gc.collect()
print("train size: ", len(train_df))
print("valid size: ", len(val_df))
print("test size : ", len(test_df))
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id'].astype('int')
 gc.collect()
print("
Training...")
 start_time = time.time()
xgb_params ={'eta': 0.3,
'max_depth': 6,
'learning_rate': 0.3,
'n_estimators': 500,
'silent': False,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'nthread':24,
'njobs': -1,
'gamma': 5.103973694670875e-08,
'max_delta_step': 20,
'min_child_weight': 4,
'subsample': 0.7,
'colsample_bylevel': 0.1,
'colsample_bytree': 0.7,
'reg_alpha': 1e-09,
'reg_lambda': 1000.0,
'scale_pos_weight': 499.99999999999994,
'random_state': 84,
' tree_method':'approx'
}
xgtrain = xgb.DMatrix(train_df[predictors_sorted].values, label=train_df[target].values)
 xgvalid = xgb.DMatrix(val_df[predictors_sorted].values, label=val_df[target].values)
 del train_df
del val_df
gc.collect()
trained_model = xgb.train(xgb_params,xgtrain, 300,[(xgvalid, 'valid')],
maximize=True, early_stopping_rounds=50,verbose_eval=10)
 xgbtest=xgb.DMatrix(test_df[predictors_sorted].values)
 print("
Model Report")
 print("bst1.best_iteration: ", trained_model.best_iteration)
 print('[{}]: Training time for XGB'.format(time.time() - start_time))
ax = xgb.plot_importance(trained_model, max_num_features=300)
 plt.gcf().savefig('test%d.png'%(fileno), dpi=600)
 plt.show()
print("Predicting...")
 sub['is_attributed'] = trained_model.predict(xgbtest,ntree_limit=trained_model.best_ntree_limit)
 sub.to_csv('sub_it%d.csv'%(fileno),index=False,float_format='%.9f')
 print("done...")
 return sub
<data_type_conversions><EOS> 
nrows=184903891-1
nchunk=12000000
val_size=1000000
frm=nrows-84903891
if debug:
frm=0
nchunk=100000
val_size=10000
to=frm+nchunk
sub=DO(frm,to,FILENO )