Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'<load_from_csv>
NROWS1 = 3000000
NROWS2 = 1000000
RG = 500000
SKIPROWS = range(1,RG)
 path = '.. /input/'
path_train = path + 'train.csv'
path_test = path + 'test.csv'
train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time']
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('Loading the training data...')
 train = pd.read_csv(path_train, skiprows=SKIPROWS, nrows=NROWS1, dtype=dtypes, header=0)
 print('End loading train data...')
 print('Loading the test data...')
 test = pd.read_csv(path_test, dtype=dtypes, header=0)
 print('End loading test data...' )
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()<load_from_csv>
print("Unique data train")
 Unique_data_train = pd.to_datetime(train.click_time ).dt.day.astype('uint8' ).value_counts().sort_index()
print(Unique_data_train)
 print("Unique data test")
 Unique_data_test = pd.to_datetime(test.click_time ).dt.day.astype('uint8' ).value_counts().sort_index()
print(Unique_data_test )
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
len_train = len(train)
 print('The initial size of the train set is', len_train)
 train = train.append(test)
 print('Binding the training and test set together...')
 del test
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
print("Train and test together")
 print(train.head(5))
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
train['hour'] = pd.to_datetime(train.click_time ).dt.hour.astype('uint8')
 train['day'] = pd.to_datetime(train.click_time ).dt.day.astype('uint8')
 train['wday'] = pd.to_datetime(train.click_time ).dt.dayofweek.astype('uint8')
 train['minute'] = pd.to_datetime(train.click_time ).dt.minute.astype('uint8')
 train['second'] = pd.to_datetime(train.click_time ).dt.second.astype('uint8')
 train["doy"] = pd.to_datetime(train.click_time ).dt.dayofyear.astype('uint8')
 train = train.drop(['click_time', 'attributed_time'], axis=1)
 print(train.dtypes )
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
print("Frequent hours")
 frequent_hour = train.hour.value_counts().sort_index()
print(frequent_hour)
 print("Frequent days")
 frequent_day = train.day.value_counts().sort_index()
print(frequent_day)
 print("Frequent doy")
 frequent_doy = train.doy.value_counts().sort_index()
print(frequent_doy)
 print("Frequent week days")
 frequent_wday = train.wday.value_counts().sort_index()
print(frequent_wday)
 print("Frequent minutes")
 frequent_minute = train.minute.value_counts().sort_index()
print(frequent_minute)
 print("Frequent seconds")
 frequent_second = train.second.value_counts().sort_index()
print(frequent_minute )
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )<data_type_conversions>
most_freq_hours_in_data = [4, 5, 9, 10, 13, 14]
middle1_freq_hours_in_data = [16, 17, 22]
least_freq_hours_in_data = [6, 11, 15]
train['in_hh'] =(4
- 3*train['hour'].isin(most_freq_hours_in_data)
 - 2*train['hour'].isin(middle1_freq_hours_in_data)
 - 1*train['hour'].isin(least_freq_hours_in_data)).astype('uint8')
 gp = train[['ip', 'day', 'in_hh', 'channel']].groupby(by=['ip', 'day', 'in_hh'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'nip_day_hh'})
 train = train.merge(gp, on=['ip','day','in_hh'], how='left')
 train.drop(['in_hh'], axis=1, inplace=True)
 train['nip_day_hh'] = train['nip_day_hh'].astype('uint32')
 del gp
gc.collect()
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )<filter>
print("Train new time parameters")
 print(train.dtypes )
train_lectures[train_lectures.user_id==5382]<filter>
train['app'] = train['app'].astype('uint16')
 train['channel'] = train['channel'].astype('uint16')
 train['device'] = train['device'].astype('uint16')
 train['ip'] = train['ip'].astype('uint32')
 train['os'] = train['os'].astype('uint16' )
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]<drop_column>
train_X = train[:len_train].drop(['click_id', 'is_attributed'], axis=1)
 train_y = train[:len_train]['is_attributed'].astype('uint8')
 test_X = train[len_train:].drop(['click_id', 'is_attributed'], axis=1)
 test_id = train[len_train:]['click_id'].astype('int')
 del train
del(train_lectures )<data_type_conversions>
predictors = ['app','device','os', 'channel', 'hour', 'day', 'doy', 'wday','minute','second',
'ip_app_channel_var_day',
'ip_app_device_var_day',
'ip_app_os_var_day',
'ip_day_hour_count_channel',
'ip_app_count_channel',
'ip_app_os_count_channel',
'ip_app_day_hour_count_channel',
'ip_app_channel_mean_hour',
'nip_day_hh']
categorical = ['app','device','os', 'channel', 'hour', 'day', 'doy', 'wday','minute','second']
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 train_df['user_lecture_cumsum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')
 user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'] )<data_type_conversions>
metrics = 'auc'
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric':metrics,
'learning_rate': 0.05,
'num_leaves': 7,
'max_depth': 4,
'min_child_samples': 100,
'max_bin': 100,
'subsample': 0.7,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'min_child_weight': 0,
'min_split_gain': 0,
'nthread': 8,
'verbose': 0,
'scale_pos_weight':99.7,
'metric':metrics
}
early_stopping_rounds = 100
num_boost_round = 10000
print("Preparing validation datasets")
 train_X, val_X = train_test_split(train_X, train_size=.95, shuffle=False)
 train_y, val_y = train_test_split(train_y, train_size=.95, shuffle=False)
 print("End preparing validation datasets")
 xgtrain = lgb.Dataset(train_X[predictors].values, label=train_y,feature_name=predictors,
categorical_feature=categorical)
 xgvalid = lgb.Dataset(val_X[predictors].values, label=val_y,feature_name=predictors,
categorical_feature=categorical)
 evals_results = {}
model_lgb = lgb.train(lgb_params,xgtrain,valid_sets=[xgtrain, xgvalid],
valid_names=['train','valid'],
evals_result=evals_results,
num_boost_round=num_boost_round,
early_stopping_rounds=early_stopping_rounds,
verbose_eval=10, feval=None)
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )<groupby>
sub = pd.DataFrame()
sub['click_id'] = test_id
print("Sub dimension " + str(sub.shape))
print("Test_X dimension " + str(test_X.shape))
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']<data_type_conversions>
print("Predicting...")
 sub['is_attributed'] = model_lgb.predict(test_X[predictors])
 print("Writing...")
 sub.to_csv('sub_Yatsenko_01.csv',index=False)
 print("Done..." )
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
train_df['lagtime'].fillna(0, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
<categorify>