Kaggle competition: Don't Overfit! II
<SOS> metric: AUC Kaggle data source: dont-overfit!-ii<feature_engineering>
np.random.seed(300)
 data = pd.DataFrame(np.zeros(( 20000,300)))
 for i in range(300): data.iloc[:,i] = np.random.normal(0,1,20000)
 important = 35; noise = 3.5
a = np.random.normal(0,1,300)
 x = np.random.choice(np.arange(300),300-important,replace=False); a[x] = 0
data['target'] = data.values.dot(a)+ np.random.normal(0,noise,20000)
 data.sort_values('target',inplace=True)
 data.iloc[:7200,300] = 0.0
data.iloc[7200:,300] = 1.0
train = data.sample(250)
 public = data[ ~data.index.isin(train.index)].sample(1975)
 private = data[ ~data.index.isin(train.index)& ~data.index.isin(public.index)].sample(frac=1)
 train.reset_index(drop=True,inplace=True)
 public.reset_index(drop=True,inplace=True)
 private.reset_index(drop=True,inplace=True )
buro_counts_unstacked.columns = ['STATUS_0', 'STATUS_1','STATUS_2','STATUS_3','STATUS_4','STATUS_5','STATUS_C','STATUS_X']
buro_counts_unstacked['MONTHS_COUNT'] = buro_grouped_size
buro_counts_unstacked['MONTHS_MIN'] = buro_grouped_min
buro_counts_unstacked['MONTHS_MAX'] = buro_grouped_max<merge>
oof = np.zeros(len(train))
predsPU= np.zeros(len(public))
predsPR= np.zeros(len(private))
rskf = RepeatedStratifiedKFold(n_splits=25, n_repeats=5)
 for train_index, test_index in rskf.split(train.iloc[:,:-1], train['target']):
clf = LogisticRegression(solver='liblinear',penalty='l2',C=1.0,class_weight='balanced')
 clf.fit(train.loc[train_index].iloc[:,:-1],train.loc[train_index]['target'])
 oof[test_index] += clf.predict_proba(train.loc[test_index].iloc[:,:-1])[:,1]
predsPU += clf.predict_proba(public.iloc[:,:-1])[:,1]
predsPR += clf.predict_proba(private.iloc[:,:-1])[:,1]
aucTR = round(roc_auc_score(train['target'],oof),5)
 aucPU = round(roc_auc_score(public['target'],predsPU),5)
 aucPR = round(roc_auc_score(private['target'],predsPR),5)
 print('LR Model with L2-penalty: CV =',aucTR,'LB =',aucPU,'Private score =',aucPR )
buro = buro.join(buro_counts_unstacked, how ='left', on ='SK_ID_BUREAU' )<define_variables>
oof = np.zeros(len(train))
predsPU= np.zeros(len(public))
predsPR= np.zeros(len(private))
rskf = RepeatedStratifiedKFold(n_splits=25, n_repeats=5)
 for train_index, test_index in rskf.split(train.iloc[:,:-1], train['target']):
clf = LogisticRegression(solver='liblinear',penalty='l1',C=0.1,class_weight='balanced')
 clf.fit(train.loc[train_index].iloc[:,:-1],train.loc[train_index]['target'])
 oof[test_index] += clf.predict_proba(train.loc[test_index].iloc[:,:-1])[:,1]
predsPU += clf.predict_proba(public.iloc[:,:-1])[:,1]
predsPR += clf.predict_proba(private.iloc[:,:-1])[:,1]
aucTR = round(roc_auc_score(train['target'],oof),5)
 aucPU = round(roc_auc_score(public['target'],predsPU),5)
 aucPR = round(roc_auc_score(private['target'],predsPR),5)
 print('LR Model with L1-penalty: CV =',aucTR,'LB =',aucPU,'Private score =',aucPR )
prev_cat_features = [pcol for pcol in prev.columns if prev[pcol].dtype == 'object']<categorify>
clf = LogisticRegression(solver='liblinear',penalty='l1',C=0.1, class_weight='balanced')
 clf.fit(train.iloc[:,:-1],train['target'])
 u0 = clf.coef_[0]
u0 = u0 / np.sqrt(u0.dot(u0))
aucPU = round(roc_auc_score(public['target'],u0.dot(public.iloc[:,:-1].values.transpose())) ,5)
 aucPR = round(roc_auc_score(private['target'],u0.dot(private.iloc[:,:-1].values.transpose())) ,5)
 bestPU = aucPU; currentPR = aucPR; initial = u0.copy()
print('Our starting model has LB =',aucPU,'and Private score =',aucPR)
 df = pd.DataFrame({'var':np.arange(300),'CV':np.zeros(300),'diff':np.zeros(300)})
 for i in range(300):
df.loc[i,'CV'] = roc_auc_score(train['target'],train[i])
 df.loc[i,'diff'] = abs(df.loc[i,'CV']-0.5)
 df.sort_values('diff',inplace=True,ascending=False )
prev = pd.get_dummies(prev, columns=prev_cat_features )<groupby>
u0 = initial
angDeg = 5
angRad = 2*np.pi*angDeg/360
np.random.seed(42)
 for k in range(150):
u1 = np.random.normal(0,1,300)
 u1[ df.iloc[100:,0] ] = 0.0
u1 = u1 - u1.dot(u0)*u0
u1 = u1 / np.sqrt(u1.dot(u1))
u2 = u0*np.cos(angRad)+ u1*np.sin(angRad)
 aucPU = round(roc_auc_score(public['target'],u2.dot(public.iloc[:,:-1].values.transpose())) ,5)
 aucPR = round(roc_auc_score(private['target'],u2.dot(private.iloc[:,:-1].values.transpose())) ,5)
 if(aucPU>bestPU)|(k==0):
bestPU = aucPU
currentPR = aucPR
u0 = u2.copy()
print('Submission',k+1,': Best LB =',bestPU,'and Private score =',currentPR )
avg_prev = prev.groupby('SK_ID_CURR' ).mean()<groupby>
df = pd.DataFrame({'var':np.arange(300),'CV':np.zeros(300),'diff':np.zeros(300)})
 for i in range(300):
df.loc[i,'CV'] = roc_auc_score(train['target'],train[i])
 df.loc[i,'diff'] = abs(df.loc[i,'CV']-0.5)
 print('We need to LB probe',len(df.loc[ df['diff']>0.04 ,'CV']),'variables' )
cnt_prev = prev[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()<feature_engineering>
df.sort_values('diff',inplace=True,ascending=False)
 LBprobe = list(df.loc[ df['diff']>0.04, 'var'])
 df.sort_values('var',inplace=True)
 df['LB'] = 0.5; df['A'] = 0; ct=0
keep = []
for i in LBprobe:
ct += 1; found = True
df.loc[i,'LB'] = roc_auc_score(public['target'],public[i])
 if(df.loc[i,'LB']<0.47)|(df.loc[i,'LB']>0.53): keep.append(i)
 else: found = False
df.loc[keep,'A'] =(8/9)*df.loc[keep,'LB']+(1/9)*df.loc[keep,'CV']-0.5
predPU = df['A'].values.dot(public.iloc[:,:300].values.transpose())
 aucPU = round(roc_auc_score(public['target'],predPU),3)
 predPR = df['A'].values.dot(private.iloc[:,:300].values.transpose())
 aucPR = round(roc_auc_score(private['target'],predPR),3)
 if found: print('Submission',ct,': Best LB =',aucPU,'and Private score ='
,aucPR,'with',len(keep),'keep' )
avg_prev['nb_app'] = cnt_prev['SK_ID_PREV']<drop_column>
train = pd.read_csv('.. /input/train.csv')
 df = pd.DataFrame({'var':np.arange(300),'CV':np.zeros(300),'diff':np.zeros(300),'LB':0.5*np.ones(300)})
 for i in range(300):
df.loc[i,'CV'] = roc_auc_score(train['target'],train[str(i)])
 df.loc[i,'diff'] = abs(df.loc[i,'CV']-0.5)
 df.sort_values('diff',inplace=True,ascending=False)
 df.head()
del avg_prev['SK_ID_PREV']<categorify>
df.loc[ df['var']==33, 'LB' ] = 0.671
df.loc[ df['var']==65, 'LB' ] = 0.671
df.loc[ df['var']==217, 'LB' ] = 0.382
df.loc[ df['var']==117, 'LB' ] = 0.405
df.loc[ df['var']==91, 'LB' ] = 0.382
buro_cat_features = [bcol for bcol in buro.columns if buro[bcol].dtype == 'object']
buro_cat_features<categorify>
df.loc[ df['var']==295, 'LB' ] = 0.506
df.loc[ df['var']==24, 'LB' ] = 0.501
df.loc[ df['var']==199, 'LB' ] = 0.613
df.loc[ df['var']==80, 'LB' ] = 0.483
df.loc[ df['var']==73, 'LB' ] = 0.394
buro = pd.get_dummies(buro, columns=buro_cat_features )<groupby>
df.loc[ df['var']==194, 'LB' ] = 0.472
df.loc[ df['var']==189, 'LB' ] = 0.454
df.loc[ df['var']==16, 'LB' ] = 0.437
df.loc[ df['var']==183, 'LB' ] = 0.506
df.loc[ df['var']==82, 'LB' ] = 0.494
avg_buro = buro.groupby('SK_ID_CURR' ).mean()<groupby>
df.loc[ df['var']==258, 'LB' ] = 0.474
df.loc[ df['var']==63, 'LB' ] = 0.428
df.loc[ df['var']==298, 'LB' ] = 0.466
df.loc[ df['var']==201, 'LB' ] = 0.489
df.loc[ df['var']==165, 'LB' ] = 0.498
avg_buro['buro_count'] = buro[['SK_ID_BUREAU', 'SK_ID_CURR']].groupby('SK_ID_CURR' ).count() ['SK_ID_BUREAU']
del avg_buro['SK_ID_BUREAU']<categorify>
df.loc[ df['var']==133, 'LB' ] = 0.486
df.loc[ df['var']==209, 'LB' ] = 0.431
df.loc[ df['var']==164, 'LB' ] = 0.542
df.loc[ df['var']==129, 'LB' ] = 0.506
df.loc[ df['var']==134, 'LB' ] = 0.471
le = LabelEncoder()
POS_CASH['NAME_CONTRACT_STATUS'] = le.fit_transform(POS_CASH['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = POS_CASH[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
POS_CASH['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
POS_CASH['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
POS_CASH.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True )<define_variables>
df.loc[ df['var']==226, 'LB' ] = 0.492
df.loc[ df['var']==237, 'LB' ] = 0.474
df.loc[ df['var']==39, 'LB' ] = 0.488
df.loc[ df['var']==17, 'LB' ] = 0.508
df.loc[ df['var']==30, 'LB' ] = 0.494
credit_card_cat_features = [c_col for c_col in credit_card.columns if credit_card[c_col].dtype == 'object']
credit_card_cat_features<count_unique_values>
df.loc[ df['var']==114, 'LB' ] = 0.479
df.loc[ df['var']==272, 'LB' ] = 0.471
df.loc[ df['var']==108, 'LB' ] = 0.459
df.loc[ df['var']==220, 'LB' ] = 0.511
df.loc[ df['var']==150, 'LB' ] = 0.482
credit_card['NAME_CONTRACT_STATUS'] = le.fit_transform(credit_card['NAME_CONTRACT_STATUS'].astype(str))
nunique_status = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).nunique()
nunique_status2 = credit_card[['SK_ID_CURR', 'NAME_CONTRACT_STATUS']].groupby('SK_ID_CURR' ).max()
credit_card['NUNIQUE_STATUS'] = nunique_status['NAME_CONTRACT_STATUS']
credit_card['NUNIQUE_STATUS2'] = nunique_status2['NAME_CONTRACT_STATUS']
credit_card.drop(['SK_ID_PREV', 'NAME_CONTRACT_STATUS'], axis=1, inplace=True )<groupby>
df.loc[ df['var']==230, 'LB' ] = 0.482
df.loc[ df['var']==90, 'LB' ] = 0.486
df.loc[ df['var']==289, 'LB' ] = 0.482
df.loc[ df['var']==241, 'LB' ] = 0.494
df.loc[ df['var']==4, 'LB' ] = 0.516
avg_payments = payments.groupby('SK_ID_CURR' ).mean()
avg_payments2 = payments.groupby('SK_ID_CURR' ).max()
avg_payments3 = payments.groupby('SK_ID_CURR' ).min()
del avg_payments['SK_ID_PREV']<merge>
df.loc[ df['var']==43, 'LB' ] = 0.474
df.loc[ df['var']==239, 'LB' ] = 0.457
df.loc[ df['var']==127, 'LB' ] = 0.501
df.loc[ df['var']==45, 'LB' ] = 0.449
df.loc[ df['var']==151, 'LB' ] = 0.504
data = data.merge(right=avg_prev.reset_index() , how='left', on ='SK_ID_CURR' )<merge>
df.loc[ df['var']==244, 'LB' ] = 0.509
df.loc[ df['var']==26, 'LB' ] = 0.511
df.loc[ df['var']==105, 'LB' ] = 0.505
df.loc[ df['var']==176, 'LB' ] = 0.525
df.loc[ df['var']==101, 'LB' ] = 0.535
test = test.merge(right=avg_prev.reset_index() , how='left', on='SK_ID_CURR' )<merge>
df['A'] = 0
df['A'] =(8/9)*df['LB'] +(1/9)*df['CV'] - 0.500
keep_threshold = 0.04
df.loc[ abs(df['A'])<keep_threshold , 'A' ] = 0
df.sort_values('var',inplace=True)
 for i in range(300):
if df.loc[i,'LB'] != 0.500:
print('A_'+str(i)+' = ',round(df.loc[i,'A'],6))
data = data.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR')
 test = test.merge(right=avg_buro.reset_index() , how='left', on='SK_ID_CURR' )<merge>
test = pd.read_csv('.. /input/test.csv')
 pred = test.iloc[:,1:].values.dot(df['A'].values)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = pred
sub.to_csv('submission.csv',index=False )