Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<data_type_conversions>
import csv
import time
from csv import DictReader
from math import exp, log, sqrt
from numba import jit
import pandas as pd
from random import randint
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )<set_options>
data_path = ".. /input/"
train = data_path+'train.csv'
train_s = data_path+'train_sample.csv'
test = data_path+'test.csv'
submission = 'submission.csv'
del cum
gc.collect()<categorify>
alpha = 0.011
beta = 0.000000001
L1 = 0.0000001
L2 = 0.0001
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )<data_type_conversions>
D = 2 ** 26
interaction = False
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )<groupby>
epoch = 2
holdday = ''
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )<train_model>
class ftrl_proximal(object):

 def __init__(self, alpha, beta, L1, L2, D):
self.alpha = alpha
self.beta = beta
self.L1 = L1
self.L2 = L2
self.D = D
self.n = [0.] * D
self.z = [0.] * D
self.w = {}
def _indices(self, x):

 yield 0
for index in x:
yield index
def predict(self, x):

 alpha = self.alpha
beta = self.beta
L1 = self.L1
L2 = self.L2
n = self.n
z = self.z
w = {}
wTx = 0.
 for i in self._indices(x):
sign = -1.if z[i] < 0 else 1.
 if sign * z[i] <= L1:
w[i] = 0.
 else:
w[i] =(sign * L1 - z[i])/(( beta + sqrt(n[i])) / alpha + L2)
 wTx += w[i]
self.w = w
return 1./(1.+ exp(-max(min(wTx, 35.) , -35.)))
def update(self, x, p, y):

 alpha = self.alpha
n = self.n
z = self.z
w = self.w
g = p - y
f = 1 if y == 0 else randint(300, 400)
 for i in self._indices(x):
sigma =(sqrt(n[i] + g * g)- sqrt(n[i])) / alpha
z[i] += f*(g - sigma * w[i])
 n[i] += f*(g * g)
print('start questions data...' )<load_from_csv>
def temps(df):
date, time = df['click_time'].split(' ')
 hour = time.split(':')[0]
h=int(hour)
 df["Nuit"]=0
df["Matin"]=0
df["Apres-midi"]=0
df["Soir"]=0
if(h>=18):
df["Soir"]=1
if(h>=0 and h<8):
df["Nuit"]=1
if(h>=8 and h<12):
df["Matin"]=1
if(h>=12 and h<18):
df["Apres-midi"]=1
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)<groupby>
learner = ftrl_proximal(alpha, beta, L1, L2, D )
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )<data_type_conversions>
start_time = time.time()
for e in range(epoch):
for t, x, y, _ in data(train, D):
p = learner.predict(x)
 learner.update(x, p, y)
 if t%1000000 == 0:
print("ligne: %sM ; %sMin "%(int(t/1e+6), '%0.0f'%(( time.time() -start_time)/60)) )
<set_options><EOS> 
start_time = time.time()
with open(submission, 'w')as outfile:
outfile.write('click_id,is_attributed
')
 for t, x, y, click_id in data(test, D):
p = learner.predict(x)
 outfile.write('%s,%s
' %(click_id, str(p)))
 if t%1000000 == 0:
print("Test Rows Processed: %sM ; %ss "%(int(t/1e+6), '%0.0f'%(time.time() -start_time)))
 print('Commit & Run !' )