Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<categorify>
%matplotlib inline
attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'])
 attempt_no_agg=attempt_no_agg.astype('int8')
<data_type_conversions>
train_col = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
df = pd.read_csv('.. /input/train.csv', nrows=30000000,
usecols=train_col, dtype=dtypes)
 df.head()
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_sum'] = train_df['content_id'].map(content_agg['sum'] ).astype('int32')
 train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 train_df.content_correctness=train_df.content_correctness.astype('float16')
 train_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )<load_from_csv>
df['hour'] = pd.to_datetime(df.click_time ).dt.hour.astype('uint8')
 df['minute'] = pd.to_datetime(df.click_time ).dt.minute.astype('uint8')
 df['day'] = pd.to_datetime(df.click_time ).dt.day.astype('uint8')
 df['dw'] = pd.to_datetime(df.click_time ).dt.dayofweek.astype('uint8')
 df_test = pd.read_csv('.. /input/test.csv')
 click_id = df_test['click_id']
df_test['hour'] = pd.to_datetime(df_test.click_time ).dt.hour.astype('uint8')
 df_test['minute'] = pd.to_datetime(df_test.click_time ).dt.minute.astype('uint8')
 df_test['day'] = pd.to_datetime(df_test.click_time ).dt.day.astype('uint8')
 df_test['dw'] = pd.to_datetime(df_test.click_time ).dt.dayofweek.astype('uint8' )
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1 )<rename_columns>
X = pd.concat(( df[['ip', 'app', 'device', 'hour', 'minute', 'os', 'channel', 'day', 'dw']],
df_test[['ip', 'app', 'device', 'hour', 'minute', 'os', 'channel', 'day', 'dw']]))
del df_test; gc.collect()
group = X[['ip','day','hour', 'minute','channel']].groupby(by=['ip','day','hour', 'minute'])['channel'].count().\
reset_index().rename(index=str, columns={'channel': 'ip_day_time'})
 X = X.merge(group, on=['ip','day','hour', 'minute'], how='left')
 del group; gc.collect()
group = X[['ip', 'app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])['channel'].count().\
reset_index().rename(index=str, columns={'channel': 'ip_app_os'})
 X = X.merge(group, on=['ip', 'app', 'os'], how='left')
 del group; gc.collect()
X[['app','device','os', 'channel', 'hour', 'minute', 'day', 'dw']].apply(LabelEncoder().fit_transform)
 len(X )
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )<data_type_conversions>
max_app = np.max(X.app)+1
max_device = np.max(X.device)+1
max_hour = np.max(X.hour)+1
max_minute = np.max(X.minute)+1
max_os = np.max(X.os)+1
max_channel = np.max(X.channel)+1
max_day = np.max(X.day)+1
max_dw = np.max(X.dw)+1
max_ip_day_time = np.max(X.ip_day_time)+1
max_ip_app_os = np.max(X.ip_app_os)+1
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16' )<categorify>
X_test = X[len(df):]
X = X[:len(df)]
y = df.is_attributed
X_train, X_valid, y_train, y_valid = train_test_split(X, y, random_state=42, train_size=0.95 )
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )<categorify>
print(np.sum(y_train)/ len(y_train))
print(np.sum(y_valid)/ len(y_valid))
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16' )<categorify>
del df; gc.collect()
def get_keras_data(data):
X = {
'app': np.array(data.app),
'device': np.array(data.device),
'hour': np.array(data.hour),
'minute': np.array(data.minute),
'os': np.array(data.os),
'channel': np.array(data.channel),
'day': np.array(data.day),
'dw': np.array(data.dw),
'ip_day_time': np.array(data.ip_day_time),
'ip_app_os': np.array(data.ip_app_os)
 }
return X
X_train = get_keras_data(X_train)
 X_valid = get_keras_data(X_valid)
 X_test = get_keras_data(X_test )
tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])
 questions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])
 questions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])
 questions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')
 questions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16' )<drop_column>
def get_model() :
app = Input(shape=[1], name='app')
 device = Input(shape=[1], name='device')
 hour = Input(shape=[1], name='hour')
 minute = Input(shape=[1], name='minute')
 os = Input(shape=[1], name='os')
 channel = Input(shape=[1], name='channel')
 day = Input(shape=[1], name='day')
 dw = Input(shape=[1], name='dw')
 ip_day_time = Input(shape=[1], name='ip_day_time')
 ip_app_os = Input(shape=[1], name='ip_app_os')
 emb_app = Embedding(max_app, 30 )(app)
 emb_device = Embedding(max_device, 30 )(device)
 emb_hour = Embedding(max_hour, 30 )(hour)
 emb_minute = Embedding(max_minute, 30 )(minute)
 emb_os = Embedding(max_os, 30 )(os)
 emb_channel = Embedding(max_channel, 30 )(channel)
 emb_day = Embedding(max_day, 30 )(day)
 emb_dw = Embedding(max_dw, 30 )(dw)
 emb_ip_day_time = Embedding(max_ip_day_time, 30 )(ip_day_time)
 emb_ip_app_os = Embedding(max_ip_app_os, 30 )(ip_app_os)
 main_l = concatenate([Flatten()(emb_app), Flatten()(emb_hour), Flatten()(emb_minute),
Flatten()(emb_os), Flatten()(emb_device),
Flatten()(emb_day), Flatten()(emb_dw), Flatten()(emb_channel),
Flatten()(emb_ip_day_time), Flatten()(emb_ip_app_os)])
 main_l = Dropout(0.1 )(Dense(1000, activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(1000, activation='relu' )(main_l))
output = Dense(1, activation='sigmoid' )(main_l)
 model = Model([app, device, hour, minute, os, channel, day, dw, ip_day_time, ip_app_os], output)
 model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer='adam')
 return model
model = get_model()
model.summary()
questions_df.drop(columns=['content_correctness'], inplace=True )<drop_column>
batch_size = 20000
epochs = 1
model.fit(X_train, np.array(y_train), epochs=epochs, batch_size=batch_size, verbose=1 )
del bundle_agg
del part_agg
del tags1_agg
gc.collect()<train_model>
pred = model.predict(X_valid )
train_df['user_correctness'].fillna(1, inplace=True)
 train_df['attempt_no'].fillna(1, inplace=True)
 train_df.fillna(0, inplace=True )<categorify>
del X; gc.collect()
MAX_SEQ = 160
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for user_id in group.index:
q, qa = group[user_id]
if len(q)>MAX_SEQ:
group[user_id] =(q[-MAX_SEQ:],qa[-MAX_SEQ:])
 pickle.dump(group, open("group.pkl", "wb"))
del group
gc.collect() 
<define_variables>
pred_test = model.predict(X_test )
<init_hyperparams><EOS> 
pred_test = pd.Series(pred_test.reshape(-1), name='is_attributed')
 sub = pd.concat([click_id, pred_test], axis=1)
 sub.to_csv('sub.csv', index=False )