Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
train_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True)
<data_type_conversions>
is_valid = False
train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )<data_type_conversions>
start_time = time.time()
train_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id']
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
train=pd.read_csv(".. /input/train.csv", skiprows=range(1,129903891), nrows=61000000, usecols=train_columns, dtype=dtypes)
 test=pd.read_csv(".. /input/test_supplement.csv",usecols=test_columns, dtype=dtypes)
 print('loading of data is completed in [{}] seconds'.format(time.time() - start_time))
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('int16')
<data_type_conversions>
def datatimeFeatures(df):
df['datetime'] = pd.to_datetime(df['click_time'])
 df['dow'] = df['datetime'].dt.dayofweek
df['doy'] = df['datetime'].dt.dayofyear
df.drop(['click_time', 'datetime'], axis=1, inplace=True)
 return df
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_cumsum'] = cum['cumsum']
train_df['user_correct_cumcount'] = cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correct_cumsum'].fillna(0, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')
 train_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16' )<data_type_conversions>
y = train['is_attributed']
train.drop(['is_attributed'], axis=1, inplace=True)
 sub = pd.DataFrame()
test.drop('click_id', axis=1, inplace=True)
 gc.collect()
nrow_train = train.shape[0]
merge = pd.concat([train, test])
 del train, test
gc.collect()
ip_count = merge.groupby(['ip'])['channel'].count().reset_index()
ip_count.columns = ['ip', 'clicks_by_ip']
merge = pd.merge(merge, ip_count, on='ip', how='left', sort=False)
 merge['clicks_by_ip'] = merge['clicks_by_ip'].astype('uint16')
 merge.drop('ip', axis=1, inplace=True)
 train = merge[:nrow_train]
test = merge[nrow_train:]
del test, merge
gc.collect()
train = datatimeFeatures(train)
 gc.collect()
print('preprocessing is completed in [{}] seconds'.format(time.time() - start_time))
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 train_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()<data_type_conversions>
params = {'eta': 0.3,
'tree_method': "hist",
'grow_policy': "lossguide",
'max_leaves': 1400,
'max_depth': 0,
'subsample': 0.9,
'colsample_bytree': 0.7,
'colsample_bylevel':0.7,
'min_child_weight':0,
'alpha':4,
'objective': 'binary:logistic',
'scale_pos_weight':9,
'eval_metric': 'auc',
'nthread':8,
'random_state': 99,
'silent': True}
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_cumsum'] = cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['explanation_mean'].fillna(0, inplace=True)
 train_df['explanation_cumsum'].fillna(0, inplace=True)
 train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16' )<set_options>
if(is_valid == True):
x1, x2, y1, y2 = train_test_split(train, y, test_size=0.1, random_state=99)
 dtrain = xgb.DMatrix(x1, y1)
 dvalid = xgb.DMatrix(x2, y2)
 del x1, y2, x2, y2
gc.collect()
watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
model = xgb.train(params, dtrain, 200, watchlist, maximize=True, early_stopping_rounds = 25, verbose_eval=5)
 del dvalid
else:
dtrain = xgb.DMatrix(train, y)
 del train, y
gc.collect()
watchlist = [(dtrain, 'train')]
model = xgb.train(params, dtrain, 30, watchlist, maximize=True, verbose_eval=1)
 del dtrain
gc.collect()
print('XGBoost Training is finished in [{}] seconds'.format(time.time() - start_time))
del cum
gc.collect()<data_type_conversions>
test = pd.read_csv(".. /input/test.csv", usecols=test_columns, dtype=dtypes)
 test = pd.merge(test, ip_count, on='ip', how='left', sort=False)
 del ip_count
gc.collect()
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()<data_type_conversions>
sub['click_id'] = test['click_id'].astype('int' )
explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
<groupby>
test['clicks_by_ip'] = test['clicks_by_ip'].astype('uint16')
 test = datatimeFeatures(test)
 test.drop(['click_id', 'ip'], axis=1, inplace=True)
 dtest = xgb.DMatrix(test)
 del test
gc.collect()
<data_type_conversions><EOS> 
sub['is_attributed'] = model.predict(dtest, ntree_limit=model.best_ntree_limit)
 sub.to_csv('improved_xgb_sub_today.csv',float_format='%.8f',index=False)
 print('submission is done in [{}] seconds'.format(time.time() - start_time))