Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<string_transform>
pd.set_option('display.max_columns', 100 )
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result<categorify>
VALIDATE = False
VALID_SIZE = 0.90
VALIDATE_KFOLDS = True
NUMBER_KFOLDS = 5
SAMPLE = True
RANDOM_STATE = 2018
MAX_ROUNDS = 1000
EARLY_STOP = 50
OPT_ROUNDS = 650
skiprows = range(1,109903891)
 nrows = 75000000
SAMPLE_SIZE = 1
output_filename = 'submission.csv'
IS_LOCAL = False
if(IS_LOCAL):
PATH = '.. /input/talkingdata-adtracking-fraud-detection/'
else:
PATH = '.. /input/'
print(os.listdir(PATH))
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )<data_type_conversions>
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
train_cols = ['ip','app','device','os', 'channel', 'click_time', 'is_attributed']
if SAMPLE:
trainset = pd.read_csv(PATH+"train_sample.csv", dtype=dtypes, usecols=train_cols)
 else:
trainset = pd.read_csv(PATH+"train.csv", skiprows=skiprows, nrows=nrows,dtype=dtypes, usecols=train_cols)
 trainset = trainset.sample(frac=SAMPLE_SIZE)
 testset = pd.read_csv(PATH+"test.csv" )
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )<drop_column>
def missing_data(data):
total = data.isnull().sum().sort_values(ascending = False)
 percent =(data.isnull().sum() /data.isnull().count() *100 ).sort_values(ascending = False)
 return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'] )
questions_df.drop(columns=['tags'], inplace=True )<data_type_conversions>
missing_data(trainset )
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
<load_from_csv>
missing_data(testset )
<rename_columns>
trainset['year'] = pd.to_datetime(trainset.click_time ).dt.year
trainset['month'] = pd.to_datetime(trainset.click_time ).dt.month
trainset['day'] = pd.to_datetime(trainset.click_time ).dt.day
trainset['hour'] = pd.to_datetime(trainset.click_time ).dt.hour
trainset['min'] = pd.to_datetime(trainset.click_time ).dt.minute
trainset['sec'] = pd.to_datetime(trainset.click_time ).dt.second
trainset.head()
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<merge>
def show_max_clean(df,gp,agg_name,agg_type,show_max):
del gp
if show_max:
print(agg_name + " max value = ", df[agg_name].max())
 df[agg_name] = df[agg_name].astype(agg_type)
 gc.collect()
return(df)
 def perform_count(df, group_cols, agg_name, agg_type='uint32', show_max=False, show_agg=True):
if show_agg:
print("Aggregating by ", group_cols , '...')
 gp = df[group_cols][group_cols].groupby(group_cols ).size().rename(agg_name ).to_frame().reset_index()
df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_countuniq(df, group_cols, counted, agg_name, agg_type='uint32', show_max=False, show_agg=True):
if show_agg:
print("Counting unique ", counted, " by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_cumcount(df, group_cols, counted, agg_name, agg_type='uint32', show_max=False, show_agg=True):
if show_agg:
print("Cumulative count by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].cumcount()
df[agg_name]=gp.values
return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_mean(df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True):
if show_agg:
print("Calculating mean of ", counted, " by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))
def perform_var(df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True):
if show_agg:
print("Calculating variance of ", counted, " by ", group_cols , '...')
 gp = df[group_cols+[counted]].groupby(group_cols)[counted].var().reset_index().rename(columns={counted:agg_name})
 df = df.merge(gp, on=group_cols, how='left')
 return(show_max_clean(df,gp,agg_name,agg_type,show_max))
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
<drop_column>
testset['year'] = pd.to_datetime(testset.click_time ).dt.year
testset['month'] = pd.to_datetime(testset.click_time ).dt.month
testset['day'] = pd.to_datetime(testset.click_time ).dt.day
testset['hour'] = pd.to_datetime(testset.click_time ).dt.hour
testset['min'] = pd.to_datetime(testset.click_time ).dt.minute
testset['sec'] = pd.to_datetime(testset.click_time ).dt.second
testset.head()
del content_explation_agg<data_type_conversions>
testset = perform_countuniq(testset, ['ip'], 'channel', 'X0', 'uint8', show_max=True); gc.collect()
testset = perform_cumcount(testset, ['ip', 'device', 'os'], 'app', 'X1', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip', 'day'], 'hour', 'X2', 'uint8', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip'], 'app', 'X3', 'uint8', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip', 'app'], 'os', 'X4', 'uint8', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip'], 'device', 'X5', 'uint16', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['app'], 'channel', 'X6', show_max=True); gc.collect()
testset = perform_cumcount(testset, ['ip'], 'os', 'X7', show_max=True); gc.collect()
testset = perform_countuniq(testset, ['ip', 'device', 'os'], 'app', 'X8', show_max=True); gc.collect()
testset = perform_count(testset, ['ip', 'day', 'hour'], 'ip_tcount', show_max=True); gc.collect()
testset = perform_count(testset, ['ip', 'app'], 'ip_app_count', show_max=True); gc.collect()
testset = perform_count(testset, ['ip', 'app', 'os'], 'ip_app_os_count', 'uint16', show_max=True); gc.collect()
testset = perform_var(testset, ['ip', 'day', 'channel'], 'hour', 'ip_tchan_count', show_max=True); gc.collect()
testset = perform_var(testset, ['ip', 'app', 'os'], 'hour', 'ip_app_os_var', show_max=True); gc.collect()
testset = perform_var(testset, ['ip', 'app', 'channel'], 'day', 'ip_app_channel_var_day', show_max=True); gc.collect()
testset = perform_mean(testset, ['ip', 'app', 'channel'], 'hour', 'ip_app_channel_mean_hour', show_max=True); gc.collect()
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )<data_type_conversions>
start = datetime.now()
len_train = len(trainset)
 gc.collect()
most_freq_hours_in_test_data = [4, 5, 9, 10, 13, 14]
least_freq_hours_in_test_data = [6, 11, 15]
def prep_data(df):
df['hour'] = pd.to_datetime(df.click_time ).dt.hour.astype('uint8')
 df['day'] = pd.to_datetime(df.click_time ).dt.day.astype('uint8')
 df.drop(['click_time'], axis=1, inplace=True)
 gc.collect()
df['in_test_hh'] =(3
- 2*df['hour'].isin(most_freq_hours_in_test_data)
 - 1*df['hour'].isin(least_freq_hours_in_test_data)).astype('uint8')
 gp = df[['ip', 'day', 'in_test_hh', 'channel']].groupby(by=['ip', 'day', 'in_test_hh'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_day_test_hh'})
 df = df.merge(gp, on=['ip','day','in_test_hh'], how='left')
 df.drop(['in_test_hh'], axis=1, inplace=True)
 df['nip_day_test_hh'] = df['nip_day_test_hh'].astype('uint32')
 del gp
gc.collect()
gp = df[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_day_hh'})
 df = df.merge(gp, on=['ip','day','hour'], how='left')
 df['nip_day_hh'] = df['nip_day_hh'].astype('uint16')
 del gp
gc.collect()
gp = df[['ip', 'os', 'hour', 'channel']].groupby(by=['ip', 'os', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_os'})
 df = df.merge(gp, on=['ip','os','hour'], how='left')
 df['nip_hh_os'] = df['nip_hh_os'].astype('uint16')
 del gp
gc.collect()
gp = df[['ip', 'app', 'hour', 'channel']].groupby(by=['ip', 'app', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_app'})
 df = df.merge(gp, on=['ip','app','hour'], how='left')
 df['nip_hh_app'] = df['nip_hh_app'].astype('uint16')
 del gp
gc.collect()
gp = df[['ip', 'device', 'hour', 'channel']].groupby(by=['ip', 'device', 'hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_hh_dev'})
 df = df.merge(gp, on=['ip','device','hour'], how='left')
 df['nip_hh_dev'] = df['nip_hh_dev'].astype('uint32')
 del gp
gc.collect()
df.drop(['ip','day'], axis=1, inplace=True)
 gc.collect()
return df
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )<drop_column>
trainset = prep_data(trainset)
 gc.collect()
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric':'auc',
'learning_rate': 0.1,
'num_leaves': 9,
'max_depth': 5,
'min_child_samples': 100,
'max_bin': 100,
'subsample': 0.9,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'min_child_weight': 0,
'min_split_gain': 0,
'nthread': 8,
'verbose': 0,
'scale_pos_weight':99.7,
}
target = 'is_attributed'
predictors = ['app','device','os', 'channel', 'hour', 'nip_day_test_hh', 'nip_day_hh', 'nip_hh_os', 'nip_hh_app', 'nip_hh_dev']
categorical = ['app', 'device', 'os', 'channel', 'hour']
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()<categorify>
if VALIDATE:
train_df, val_df = train_test_split(trainset, test_size=VALID_SIZE, random_state=RANDOM_STATE, shuffle=True)
 dtrain = lgb.Dataset(train_df[predictors].values,
label=train_df[target].values,
feature_name=predictors,
categorical_feature=categorical)
 del train_df
gc.collect()
dvalid = lgb.Dataset(val_df[predictors].values,
label=val_df[target].values,
feature_name=predictors,
categorical_feature=categorical)
 del val_df
gc.collect()
evals_results = {}
model = lgb.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train','valid'],
evals_result=evals_results,
num_boost_round=MAX_ROUNDS,
early_stopping_rounds=EARLY_STOP,
verbose_eval=50,
feval=None)
 del dvalid
elif VALIDATE_KFOLDS:
kf = KFold(n_splits = NUMBER_KFOLDS, random_state = RANDOM_STATE, shuffle = True)
 for train_index, test_index in kf.split(trainset):
train_X, valid_X = trainset.iloc[train_index], trainset.iloc[test_index]
dtrain = lgb.Dataset(train_X[predictors].values, label=train_X[target].values,
feature_name=predictors, categorical_feature=categorical)
 dvalid = lgb.Dataset(valid_X[predictors].values, label=valid_X[target].values,
feature_name=predictors, categorical_feature=categorical)
 evals_results = {}
model = lgb.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train','valid'],
evals_result=evals_results,
num_boost_round=MAX_ROUNDS,
early_stopping_rounds=EARLY_STOP,
verbose_eval=50,
feval=None)
 else:
gc.collect()
dtrain = lgb.Dataset(train_df[predictors].values, label=train_df[target].values,
feature_name=predictors,
categorical_feature=categorical 
)
 del train_df
gc.collect()
evals_results = {}
model = lgb.train(params,
dtrain,
valid_sets=[dtrain],
valid_names=['train'],
evals_result=evals_results,
num_boost_round=OPT_ROUNDS,
verbose_eval=50,
feval=None)
 del dtrain
gc.collect()
<data_type_conversions><EOS> 
test_cols = ['ip','app','device','os', 'channel', 'click_time', 'click_id']
test_df = prep_data(testset)
 gc.collect()
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id']
sub['is_attributed'] = model.predict(test_df[predictors])
 sub.to_csv(output_filename, index=False, float_format='%.9f')