Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<import_modules>
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
import lightgbm as lgb
import xgboost as xgb
import gc
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf
import pickle
import json
import typing
import math
from threading import Lock 
<import_modules>
path = '.. /input/'
path_train = path + 'train.csv'
path_test = path + 'test.csv'
unbalance_fact = 10
train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
test_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time' ]
dtypes = {
'ip' : 'uint64',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint64'
}
from modeling import RiiidAnswerModel
from model_config import config<categorify>
print("Loading Data")
 train = pd.read_csv(path_train, usecols=train_cols, dtype=dtypes, parse_dates=["click_time"])
 print("Loading is done" )
with open('.. /input/riiidmodel4thplace/data_map.pickle', 'rb')as f:
data_map = pickle.load(f)
 with open(
 '.. /input/riiidmodel4thplace/encoded_content_map_v2.json',
'r')as f:
encoded_content_map = json.load(f)
 with open(
 '.. /input/riiidmodel4thplace/encoded_content_id_map.json',
'r')as f:
encoded_content_id_map = json.load(f)
 encoded_content_table = pd.DataFrame.from_dict(
 encoded_content_id_map
).astype({
'content_id': 'int16',
'content_type_id':'int8',
'encoded_content_id':'int32'
})
 max_question_id = encoded_content_table[encoded_content_table.content_type_id == 0].encoded_content_id.max()<define_variables>
print(len(train))
train_pos = train[train["is_attributed"] == 1]
num_attr = len(train_pos)
 print("Number of attributed: {:d}".format(num_attr))
train_neg = train[train["is_attributed"] == 0].sample(num_attr)
 train = pd.concat([train_pos, train_neg])
 gc.collect()
SEQUENCE_LENGTH = 512
BATCH_SIZE = 32
data_keys = ['timestamp',
'encoded_content_id',
'user_answer',
'answered_correctly',
'question_elapsed_time',
'question_had_explanation',
'non_padding_mask',
'time_lag']
data_types = [
tf.int64,
tf.int32,
tf.int32,
tf.int32,
tf.float32,
tf.int32,
tf.float32,
tf.float32
]<categorify>
def prepare_dataframe(df):
print("Display frames:")
 display(df.head())
 display(df.dtypes)
 display(df.shape)
 print("Creating new time features:")
 df['hour'] = df["click_time"].dt.hour.astype('uint8')
 df['day'] = df["click_time"].dt.day.astype('uint8')
 df["minute"] = df["click_time"].dt.minute.astype('uint8')
 print("
n_chans = df[['ip','day','hour','channel']].groupby(by=['ip','day',
'hour'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_day_hour'})
 df = df.merge(n_chans, on=['ip','day','hour'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','app', 'channel']].groupby(by=['ip',
'app'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_count'})
 df = df.merge(n_chans, on=['ip','app'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app',
'os'])[['channel']].count().reset_index().rename(columns={'channel': 'ip_app_os_count'})
 df = df.merge(n_chans, on=['ip','app', 'os'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','channel']].groupby(by=['ip'])[['channel']].count().reset_index().rename(columns={'channel': 'count_by_ip'})
 print('Merging the channels data with the main data set...')
 df = df.merge(n_chans, on=['ip'], how='left')
 n_chans = df[['ip','hour','channel','os']].groupby(by=['ip','hour','channel'
])[['os']].count().reset_index().rename(columns={'os': 'ip_hour_channel'})
 df = df.merge(n_chans, on=['ip','hour','channel'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','hour','channel','os']].groupby(by=['ip','hour','os'
])[['channel']].count().reset_index().rename(columns={'channel': 'ip_hour_os'})
 df = df.merge(n_chans, on=['ip','hour','os'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','hour','channel','app']].groupby(by=['ip','hour','app'
])[['channel']].count().reset_index().rename(columns={'channel': 'ip_hour_app'})
 df = df.merge(n_chans, on=['ip','hour','app'], how='left')
 del n_chans
gc.collect()
n_chans = df[['ip','hour','channel','device']].groupby(by=['ip','hour','device'
])[['channel']].count().reset_index().rename(columns={'channel': 'ip_hour_device'})
 df = df.merge(n_chans, on=['ip','hour','device'], how='left')
 del n_chans
gc.collect()
print("Adjusting the data types of the new count features...")
 df.info()
df['ip_day_hour'] = df['ip_day_hour'].astype('uint8')
 df['ip_app_count'] = df['ip_app_count'].astype('uint8')
 df['ip_app_os_count'] = df['ip_app_os_count'].astype('uint8')
 df['count_by_ip'] = df['count_by_ip'].astype('uint16')
 df['ip_hour_channel'] = df['ip_hour_channel'].astype('uint16')
 df['ip_hour_os'] = df['ip_hour_os'].astype('uint16')
 df['ip_hour_app'] = df['ip_hour_app'].astype('uint16')
 df['ip_hour_device'] = df['ip_hour_device'].astype('uint16')
 return df
def build_test_ds_function(test_agg_df, data_map):
data = data_map['data']
index_map = data_map['index']
lock = Lock()
def process_row_i(i):
user_id = i.numpy()
row = test_agg_df.loc[user_id]
n = len(row['timestamp'])
 index = index_map.get(user_id, None)
 if index is not None:
is_question = data['encoded_content_id'][index] <= max_question_id
questions = data['timestamp'][index][is_question]
if len(questions)> 0:
last_question_timestamp = questions.max()
prior_question_cond = data['timestamp'][index] == last_question_timestamp
data['question_elapsed_time'][index][prior_question_cond] = row['prior_question_elapsed_time'][0]
data['question_had_explanation'][index][prior_question_cond] = row['prior_question_had_explanation'][0]
for k in data_keys:
data[k][index,:-n] = data[k][index, n:]
else:
lock.acquire()
index = data_map['next_index']
index_map[user_id] = index
data_map['next_index'] += 1
lock.release()
data['timestamp'][index, -n:] = row['timestamp']
data['encoded_content_id'][index,-n:] = row['encoded_content_id']
data['user_answer'][index, -n:] = 0
data['answered_correctly'][index, -n:] = 0
data['question_elapsed_time'][index, -n:] = 0
data['question_had_explanation'][index, -n:] = 0
data['non_padding_mask'][index,-n:] = 1
data['time_lag'][index, -n:] = row['timestamp'][0] - data['timestamp'][index, -(n+1)]
return [data[k][index] for k in data_keys]
def map_function(i):
x = tf.py_function(process_row_i, inp=[i], Tout=data_types)
 return {
k:v for k,v in zip(data_keys, x)
 }
return tf.data.Dataset.from_tensor_slices(test_agg_df.index ).map(
 map_function,
num_parallel_calls = tf.data.experimental.AUTOTUNE
).prefetch(
 tf.data.experimental.AUTOTUNE
).batch(BATCH_SIZE)
<categorify>
train = prepare_dataframe(train )
def update_map_with_user_answer(data_map, prev_test_agg_df):
data = data_map['data']
for user_id, row in prev_test_agg_df.iterrows() :
n = len(row['user_answer'])
 index = data_map['index'][user_id]
data['user_answer'][index][-n:] = row['user_answer']
data['answered_correctly'][index][-n:] = row['answered_correctly'] 
<predict_on_test>
train_df, test_df = train_test_split(train, test_size=0.2, stratify=train["is_attributed"])
 train_df, valid_df = train_test_split(train_df, test_size=0.2, stratify=train_df["is_attributed"] )
model = RiiidAnswerModel(
 encoded_content_map,
**config 
)
 @tf.function(
 input_signature =({
key : tf.TensorSpec(shape=(None, SEQUENCE_LENGTH), dtype=dt)
 for key, dt in zip(data_keys, data_types)
 },)
)
 def predict_batch(test_data):
logits = model(test_data, training = False)
 logits = tf.squeeze(logits, axis =-1)
 sigmoids = tf.sigmoid(logits)
 return sigmoids
sample_data = {k:data_map['data'][k][0:BATCH_SIZE,:] for k in data_map['data'].keys() }
predict_batch(sample_data)
 model.load_weights('.. /input/riiidmodel4thplace/weights.h5')
<set_options>
print(len(train_df))
print(len(valid_df))
print(len(test_df))
del train
gc.collect()
env = riiideducation.make_env()<feature_engineering>
target = 'is_attributed'
predictors = ['ip', 'device', 'app', 'os', 'channel', 'hour', "minute",
'ip_day_hour','count_by_ip','ip_app_count', 'ip_app_os_count',
"ip_hour_channel", "ip_hour_os", "ip_hour_app","ip_hour_device"]
gc.collect()
%%time
iter_test = env.iter_test()
prev_test_df = None
for test_df, sample_prediction_df in iter_test:
if prev_test_df is not None:

 prev_test_df['user_answer'] = eval(test_df.prior_group_responses.iloc[0])
 prev_test_df['answered_correctly'] = eval(test_df.prior_group_answers_correct.iloc[0])
 prev_test_agg_df = prev_test_df.groupby(
 'user_id',
sort = False,
as_index = True 
)[['user_answer', 'answered_correctly'
]].agg(list)
 update_map_with_user_answer(data_map, prev_test_agg_df)

test_df.fillna(0, inplace = True)
 test_agg_df = test_df.merge(
 encoded_content_table,
on = ['content_id', 'content_type_id'],
how = 'left'
).fillna(0 ).groupby('user_id',
sort = False,
as_index = True)[['timestamp',
'encoded_content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'row_id'
]].agg(list)
 probs = predict_and_update(test_agg_df, data_map)
 extracted_probs = []
row_id = []
for prob,(_, row)in zip(probs,
test_agg_df.iterrows()):

 n = len(row['row_id'])
 extracted_probs += prob[-n:].numpy().tolist()
row_id += row['row_id']
pred_df = pd.DataFrame.from_dict({
'row_id':row_id,
'answered_correctly': extracted_probs,
})
 pred_df = sample_prediction_df[['row_id']].merge(
 pred_df,
on = 'row_id',
how = 'left'
).fillna(0.5)
 env.predict(pred_df)
 prev_test_df = test_df 
<set_options>
print("Testing against test split")
 dtest = xgb.DMatrix(test_df[predictors])
 test_res = model.predict(dtest, ntree_limit=model.best_ntree_limit)
 test_score = roc_auc_score(test_df[target].values, test_res)
 print("Test score = {:f}".format(test_score))
def imports() :
for name, val in globals().items() :
if isinstance(val, types.ModuleType):
yield name, val
if hasattr(val, '__call__'):
yield name, val
np.seterr(divide='ignore', invalid='ignore')
 noglobal = lambda fn: types.FunctionType(fn.__code__, dict(imports()))
%load_ext Cython
warnings.filterwarnings('ignore', 'Mean of empty slice')
<categorify>
print("Loading test data for generating submission.")
 test = pd.read_csv(path_test, dtype=dtypes, parse_dates=["click_time"])
 test = prepare_dataframe(test)
 print("Preparing data for submission...")
 dtest = xgb.DMatrix(test[predictors])
 test['is_attributed'] = model.predict(dtest, ntree_limit=model.best_ntree_limit)
 print("Writing the submission data into a csv file...")
 test[["click_id","is_attributed"]].to_csv("submission_xgb_v2.csv",index=False)
 print("All Done..." )