Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<merge>
train = pd.read_csv('.. /input/train.csv')
 train0 = train[ train['target']==0 ].copy()
train1 = train[ train['target']==1 ].copy()
train.sample(5 )
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )<groupby>
s = [0]*200
m = [0]*200
for i in range(200):
s[i] = np.std(train['var_'+str(i)])
 m[i] = np.mean(train['var_'+str(i)])
 def getp(i,x):
c = 3
a = len(train1[(train1['var_'+str(i)]>x-s[i]/c)&(train1['var_'+str(i)]<x+s[i]/c)])
 b = len(train0[(train0['var_'+str(i)]>x-s[i]/c)&(train0['var_'+str(i)]<x+s[i]/c)])
 if a+b<500: return 0.1
return a /(a+b)
 def smooth(x,st=1):
for j in range(st):
x2 = np.ones(len(x)) * 0.1
for i in range(len(x)-2):
x2[i+1] = 0.25*x[i]+0.5*x[i+1]+0.25*x[i+2]
x = x2.copy()
return x
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']<groupby>
warnings.filterwarnings("ignore")
 Picture = True
rmin=-5; rmax=5;
res=501
pr = 0.1 * np.ones(( 200,res))
pr2 = pr.copy()
xr = np.zeros(( 200,res))
xr2 = xr.copy()
ct2 = 0
for j in range(50):
if Picture: plt.figure(figsize=(15,8))
for v in range(4):
ct = 0
for i in np.linspace(rmin,rmax,res):
pr[v+4*j,ct] = getp(v+4*j,m[v+4*j]+i*s[v+4*j])
 xr[v+4*j,ct] = m[v+4*j]+i*s[v+4*j]
xr2[v+4*j,ct] = i
ct += 1
if Picture:
pr2[v+4*j,:] = smooth(pr[v+4*j,:],res//10)
 plt.subplot(2, 4, ct2%4+5)
 plt.plot(xr[v+4*j,:],pr2[v+4*j,:],'-')
 plt.title('P(t=1 | var_'+str(v+4*j)+')')
 xx = plt.xlim()
plt.subplot(2, 4, ct2%4+1)
 sns.distplot(train0['var_'+str(v+4*j)], label = 't=0')
 sns.distplot(train1['var_'+str(v+4*j)], label = 't=1')
 plt.title('var_'+str(v+4*j))
plt.legend()
plt.xlim(xx)
 plt.xlabel('')
 if(ct2%8==0): print('Showing vars',ct2,'to',ct2+7,'...')
 ct2 += 1
if Picture: plt.show()
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']<merge>
def getp2(i,x):
z =(x-m[i])/s[i]
ss =(rmax-rmin)/(res-1)
 if res%2==0: idx = min(( res+1)//2 + z//ss, res-1)
 else: idx = min(( res+1)//2 +(z-ss/2)//ss, res-1)
 idx = max(idx,0)
 return pr[i,int(idx)]
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<merge>
print('Calculating 200000 predictions and displaying a few examples...')
 pred = [0]*200000; ct = 0
for r in train.index:
p = 0.1
for i in range(200):
p *= 10*getp2(i,train.iloc[r,2+i])
 if ct%25000==0: print('train',r,'has target =',train.iloc[r,1],'and prediction =',p)
 pred[ct]=p; ct += 1
print('
print('Validation AUC =',roc_auc_score(train['target'], pred))
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )<feature_engineering>
test = pd.read_csv('.. /input/test.csv')
 print('Calculating 200000 predictions and displaying a few examples...')
 pred = [0]*200000; ct = 0
for r in test.index:
p = 0.1
for i in range(200):
p *= 10*getp2(i,test.iloc[r,1+i])
 if ct%25000==0: print('test',r,'has prediction =',p)
 pred[ct]=p
ct += 1
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = pred
sub.to_csv('submission.csv',index=False)
 print('
print('Finished.Wrote predictions to submission.csv' )