Kaggle competition: Kannada MNIST
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_train.head()<feature_engineering>
%matplotlib inline
useful = np.zeros(( 256,512))
for i in range(512):
partial = df_train[ df_train['wheezy-copper-turtle-magic']==i ]
useful[:,i] = np.std(partial.iloc[:,1:-1], axis=0)
 useful = useful > 1.5
useful = np.sum(useful, axis=0 )<prepare_x_and_y>
train_df = pd.read_csv("/kaggle/input/Kannada-MNIST/train.csv" )
models = 512
RunLocally = True
if len(df_test)>512*300:
repeat = 1
models = 512 * repeat
RunLocally = False
all_preds = np.zeros(len(df_test))
all_y_pu = np.array([])
 all_y_pr = np.array([])
 all_preds_pu = np.array([])
 all_preds_pr = np.array([])
 for k in range(models):
if RunLocally:
obs = 512
X, y = make_classification(n_samples=1024, n_features=useful[k%512],
n_informative=useful[k%512], n_redundant=0, n_repeated=0,
n_classes=2, n_clusters_per_class=3, weights=None, flip_y=0.05,
class_sep=1.0, hypercube=True, shift=0.0, scale=1.0, shuffle=True,
random_state=None)
 else:
df_train2 = df_train[df_train['wheezy-copper-turtle-magic']==k%512]
df_test2 = df_test[df_test['wheezy-copper-turtle-magic']==k%512]
sel = VarianceThreshold(1.5 ).fit(df_train2.iloc[:,1:-1])
 df_train3 = sel.transform(df_train2.iloc[:,1:-1])
 df_test3 = sel.transform(df_test2.iloc[:,1:])
 obs = df_train3.shape[0]
X = np.concatenate(( df_train3,df_test3),axis=0)
 y = np.concatenate(( df_train2['target'].values,np.zeros(len(df_test2))))
train = X[:obs,:]
train_y = y[:obs]
test = X[obs:,:]
test_y = y[obs:]
comb = X
clf = QuadraticDiscriminantAnalysis(priors = [0.5,0.5])
 clf.fit(train,train_y)
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
clf = QuadraticDiscriminantAnalysis(priors = [0.5, 0.5])
 clf.fit(comb, np.concatenate(( train_y,test_pred)))
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
all_y = np.concatenate(( train_y,test_pred))
least = 0; ct = 1; thx=150
while least<thx:
if ct>=10: thx -= 10
else: thx = 150
clusters = np.zeros(( len(comb),6))
train4 = comb[ all_y==1, :]
clf = GaussianMixture(n_components=3 ).fit(train4)
 clusters[ all_y==1, 3:] = clf.predict_proba(train4)
 train4 = comb[ all_y==0, :]
clf = GaussianMixture(n_components=3 ).fit(train4)
 clusters[ all_y==0, :3] = clf.predict_proba(train4)
 for j in range(5): clusters[:,j+1] += clusters[:,j]
rand = np.random.uniform(0,1,clusters.shape[0])
 for j in range(6): clusters[:,j] = clusters[:,j]>rand
clusters2 = 6 - np.sum(clusters,axis=1)
 least = pd.Series(clusters2 ).value_counts().min() ; ct += 1
clf = QuadraticDiscriminantAnalysis(priors = [0.167, 0.167, 0.167, 0.167, 0.167, 0.167])
 clf.fit(comb,clusters2)
 pds = clf.predict_proba(test)
 test_pred = pds[:,3]+pds[:,4]+pds[:,5]
if RunLocally:
all_y_pu = np.append(all_y_pu, test_y[:256])
 all_y_pr = np.append(all_y_pr, test_y[256:])
 all_preds_pu = np.append(all_preds_pu, test_pred[:256])
 all_preds_pr = np.append(all_preds_pr, test_pred[256:])
 else:
all_preds[df_test2.index] += test_pred / repeat
if(( k+1)%64==0)|(k==0): print('modeled and predicted',k+1,'magic sub datasets')
 if RunLocally:
all_y_pu_pr = np.concatenate(( all_y_pu,all_y_pr))
all_preds_pu_pr = np.concatenate(( all_preds_pu,all_preds_pr))
auc1 = roc_auc_score(all_y_pu_pr, all_preds_pu_pr)
 auc2 = roc_auc_score(all_y_pu, all_preds_pu)
 auc3 = roc_auc_score(all_y_pr, all_preds_pr)
 print()
print('Validation AUC =',np.round(auc1,5))
print('Approx Public LB =',np.round(auc2,5))
print('Approx Private LB =',np.round(auc3,5))<save_to_csv>
test_df = pd.read_csv("/kaggle/input/Kannada-MNIST/test.csv" )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = all_preds
sub.to_csv('submission.csv',index=False)
 plt.hist(test_pred ,bins=100)
 plt.title('Model 512 test predictions')
 plt.show()<prepare_x_and_y>
train_df['label'].value_counts()
for clusters in range(4):
centers = np.array([])
 for k in range(512):
X, y = make_classification(n_samples=768, n_features=useful[k],
n_informative=useful[k], n_redundant=0, n_repeated=0,
n_classes=2, n_clusters_per_class=clusters+1, weights=None,
flip_y=0.05, class_sep=1.0, hypercube=True, shift=0.0, scale=1.0,
shuffle=True, random_state=None)
 centers = np.append(centers,np.mean(X[ np.argwhere(y==0 ).flatten() ,:],axis=0))
centers = np.append(centers,np.mean(X[ np.argwhere(y==1 ).flatten() ,:],axis=0))
plt.hist(centers,bins=100)
 plt.title('Variable means if clusters='+str(clusters+1))
plt.show() 
<prepare_x_and_y>
import keras
from keras import utils
from keras import models
from keras import backend as K
from keras import optimizers
from keras.layers.core import(Dense, Dropout, Activation, Flatten, Reshape)
 from keras.layers.convolutional import(Conv2D, MaxPooling2D )
centers = np.array([])
 for k in range(512):
df_train2 = df_train[df_train['wheezy-copper-turtle-magic']==k]
df_test2 = df_test[df_test['wheezy-copper-turtle-magic']==k]
sel = VarianceThreshold(1.5 ).fit(df_train2.iloc[:,1:-1])
 df_train3 = sel.transform(df_train2.iloc[:,1:-1])
 df_test3 = sel.transform(df_test2.iloc[:,1:])
 obs = df_train3.shape[0]
X = np.concatenate(( df_train3,df_test3),axis=0)
 y = np.concatenate(( df_train2['target'].values,np.zeros(len(df_test2))))
train = X[:obs,:]
train_y = y[:obs]
test = X[obs:,:]
test_y = y[obs:]
comb = X
clf = QuadraticDiscriminantAnalysis(priors = [0.5,0.5])
 clf.fit(train,train_y)
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
clf = QuadraticDiscriminantAnalysis(priors = [0.5, 0.5])
 clf.fit(comb, np.concatenate(( train_y,test_pred)))
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
y[obs:] = test_pred
centers = np.append(centers,np.mean(X[ np.argwhere(y==0 ).flatten() ,:],axis=0))
centers = np.append(centers,np.mean(X[ np.argwhere(y==1 ).flatten() ,:],axis=0))
plt.hist(centers,bins=100)
 plt.title('Real Data Variable Means(match clusters=3)')
 plt.show()<load_from_csv>
BATCH_SIZE = 64
NP_EPOCHS = 50
NP_CLASSES = 10
VALIDATION_SPLIT = 0.20
OPTIMIZER = optimizers.RMSprop()
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))<feature_engineering>
train = train_df.drop('label', axis=1)
 target = train_df['label']
test = test_df.drop('id', axis=1 )
test['target'] = preds
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))<load_from_csv>
train /= 255
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 oof_QDA = np.zeros(len(train))
preds_QDA = np.zeros(len(test))
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=25, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof_QDA[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds_QDA[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_QDA)
 print('QDA scores CV =',round(auc,8))
sub_QDA = pd.read_csv('.. /input/sample_submission.csv')
 sub_QDA['target'] = preds_QDA
sub_QDA.to_csv('submission_QDA.csv',index=False)
 oof_preds_QDA = train[['id', 'target']].copy()
oof_preds_QDA['target'] = oof_QDA
oof_preds_QDA.to_csv('oof_preds_QDA.csv', index = False )<feature_engineering>
test /= 255
test['target'] = preds_QDA
oof_QDA_PL = np.zeros(len(train))
preds_QDA_PL = np.zeros(len(test))
preds_QDA_PL_label = np.zeros(len(test))
n = 240
for k in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k].sort_values(by = 'target')
 idx2 = test2.index
test2p = pd.concat([test2[: n], test2[-n: ]], axis = 0)
 test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 preds_QDA_PL_label_slice = preds_QDA_PL_label[list(idx2[: n])+ list(idx2[-n: ])]
skf = StratifiedKFold(n_splits=25, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
test_index4 = test_index[ test_index>=len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_QDA_PL[idx1[test_index3]] = clf.predict_proba(train3p[test_index3])[:,1]
if(len(test_index4)> 0):
preds_QDA_PL_label_slice[test_index4 - len(train3)] = clf.predict_proba(train3p[test_index4])[:, 1]
preds_QDA_PL[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
preds_QDA_PL_label[list(idx2[: n])+ list(idx2[-n: ])] = preds_QDA_PL_label_slice
preds_QDA_PL[preds_QDA_PL_label != 0] = preds_QDA_PL_label[preds_QDA_PL_label != 0]
auc = roc_auc_score(train['target'],oof_QDA_PL)
 print('Pseudo Labeled QDA scores CV =',round(auc,8))
sub_QDA_PL = pd.read_csv('.. /input/sample_submission.csv')
 sub_QDA_PL['target'] = preds_QDA_PL
sub_QDA_PL.to_csv('submission_QDA_PL.csv',index=False)
 oof_preds_QDA_PL = train[['id', 'target']].copy()
oof_preds_QDA_PL['target'] = oof_QDA_PL
oof_preds_QDA_PL.to_csv('oof_preds_QDA_PL.csv', index = False )<data_type_conversions>
target = utils.to_categorical(target, NP_CLASSES )
def get_mean_cov(x,y):
model = ShrunkCovariance()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps
oof_GMM = np.zeros(len(train))
preds_GMM = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=25, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gm = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3[train_index,:],test3],axis = 0))
oof_GMM[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:,0]
preds_GMM[idx2] += gm.predict_proba(test3)[:,0] / skf.n_splits
auc = roc_auc_score(train['target'],oof_GMM)
 print('GMM scores CV =',round(auc,8))
sub_GMM = pd.read_csv('.. /input/sample_submission.csv')
 sub_GMM['target'] = preds_GMM
sub_GMM.to_csv('submission_GMM.csv', index = False)
 oof_preds_GMM = train[['id', 'target']].copy()
oof_preds_GMM['target'] = oof_GMM
oof_preds_GMM.to_csv('oof_preds_GMM.csv', index = False )<feature_engineering>
NP_OUTPUT_FUNC = "softmax"
NP_DESCRIMINATOR_FUNC = "sigmoid"
NP_DROPOUT_FIRST = 0.25
NP_DROPOUT_SECOND = 0.20
test['target'] = preds_QDA
oof_NuSVC = np.zeros(len(train))
preds_NuSVC = np.zeros(len(test))
preds_NuSVC_label = np.zeros(len(test))
n = 220
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i].sort_values(by = 'target')
 idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pca = PCA(svd_solver='full',n_components='mle')
 scaler = StandardScaler()
data2 = scaler.fit_transform(pca.fit_transform(data[cols]))
train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
test2p = pd.concat([test2[: n], test2[-n: ]], axis = 0)
 test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2, test2p], axis = 0)
 train2p.reset_index(drop=True,inplace=True)
 test5 = scaler.transform(pca.transform(test2p[cols]))
train3p = np.concatenate([train3, test5])
 preds_NuSVC_label_slice = preds_NuSVC_label[list(idx2[: n])+ list(idx2[-n: ])]
skf = StratifiedKFold(n_splits=25, random_state=42, shuffle = True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[test_index < len(train3)]
test_index4 = test_index[test_index >= len(train3)]
clf = NuSVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=4, nu=0.75, coef0=0.053)
 clf.fit(train3p[train_index],train2p.loc[train_index]['target'])
 oof_NuSVC[idx1[test_index3]] = clf.predict_proba(train3p[test_index3])[:,1]
if(len(test_index4)> 0):
preds_NuSVC_label_slice[test_index4 - len(train3)] = clf.predict_proba(train3p[test_index4])[:, 1]
preds_NuSVC[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
preds_NuSVC_label[list(idx2[: n])+ list(idx2[-n: ])] = preds_NuSVC_label_slice
preds_NuSVC[preds_NuSVC_label != 0] = preds_NuSVC_label[preds_NuSVC_label != 0]
auc = roc_auc_score(train['target'],oof_NuSVC)
 print('Pseudo Labeled NuSVC scores CV =',round(auc,8))
sub_NuSVC = pd.read_csv('.. /input/sample_submission.csv')
 sub_NuSVC['target'] = preds_NuSVC
sub_NuSVC.to_csv('submission_NuSVC.csv', index = False)
 oof_preds_NuSVC = train[['id', 'target']].copy()
oof_preds_NuSVC['target'] = oof_NuSVC
oof_preds_NuSVC.to_csv('oof_preds_NuSVC.csv', index = False )<feature_engineering>
model = Sequential()
model.add(Conv2D(32,(5, 5), padding='same', input_shape=(28, 28, 1)))
 model.add(LeakyReLU(alpha=0.02))
model.add(Conv2D(32,(5, 5)))
 model.add(LeakyReLU(alpha=0.02))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(NP_DROPOUT_FIRST))
model.add(Conv2D(64,(3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.02))
model.add(Conv2D(64,(3, 3)))
 model.add(LeakyReLU(alpha=0.02))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(NP_DROPOUT_FIRST))
model.add(Flatten())
 model.add(Dense(256))
model.add(LeakyReLU(alpha=0.02))
model.add(Dropout(NP_DROPOUT_SECOND))
model.add(Dense(256))
model.add(LeakyReLU(alpha=0.02))
model.add(Dropout(NP_DROPOUT_SECOND))
model.add(Dense(NP_CLASSES))
model.add(Activation(NP_OUTPUT_FUNC))
test["target"] = preds_QDA
oof_LS = np.zeros(len(train))
preds_LS = np.zeros(len(test))
preds_LS_label = np.zeros(len(test))
n = 230
for k in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k].sort_values(by = 'target')
 idx2 = test2.index
test2p = pd.concat([test2[: n], test2[-n: ]], axis = 0)
 test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 preds_LS_label_slice = preds_LS_label[list(idx2[: n])+ list(idx2[-n: ])]
skf = StratifiedKFold(n_splits = 25, random_state = 42, shuffle = True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
test_index4 = test_index[ test_index>=len(train3)]
clf = LabelSpreading(gamma = 0.0125, kernel = 'rbf', max_iter = 10,alpha = 0.4,tol = 0.001)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_LS[idx1[test_index3]] = clf.predict_proba(train3p[test_index3])[:,1]
if(len(test_index4)> 0):
preds_LS_label_slice[test_index4 - len(train3)] = clf.predict_proba(train3p[test_index4])[:, 1]
preds_LS[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
preds_LS_label[list(idx2[: n])+ list(idx2[-n: ])] = preds_LS_label_slice
preds_LS[preds_LS_label != 0] = preds_LS_label[preds_LS_label != 0]
auc = roc_auc_score(train['target'],oof_LS)
 print('Pseudo Labeled LS scores CV =',round(auc,8))
sub_LS = pd.read_csv('.. /input/sample_submission.csv')
 sub_LS['target'] = preds_LS
sub_LS.to_csv('submission_LS.csv', index = False)
 oof_preds_LS = train[['id', 'target']].copy()
oof_preds_LS['target'] = oof_LS
oof_preds_LS.to_csv('oof_preds_LS.csv', index = False )<train_model>
optimizer = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, decay=0.0)
 optimizer
oof_preds = pd.concat([oof_preds_QDA_PL['target'], oof_preds_NuSVC['target'], oof_preds_GMM['target'], oof_preds_LS['target']], axis = 1)
 sub_preds = pd.concat([sub_QDA_PL['target'], sub_NuSVC['target'], sub_GMM['target'], sub_LS['target']], axis = 1)
 oof_stacking = np.zeros(len(train))
preds_stacking = np.zeros(len(test))
skf = StratifiedKFold(n_splits=25, random_state=42, shuffle = True)
 for train_index, test_index in skf.split(oof_preds, train['target']):
lrr = linear_model.LogisticRegression()
lrr.fit(oof_preds.loc[train_index], train.loc[train_index, 'target'])
 oof_stacking[test_index] = lrr.predict_proba(oof_preds.loc[test_index,:])[:,1]
preds_stacking += lrr.predict_proba(sub_preds)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_stacking)
 print('Stacking scores CV =',round(auc,8))
sub_stacking = pd.read_csv('.. /input/sample_submission.csv')
 sub_stacking['target'] = preds_stacking
sub_stacking.to_csv('submission_stacking.csv', index = False)
 oof_preds_stacking = train[['id', 'target']].copy()
oof_preds_stacking['target'] = oof_stacking
oof_preds_stacking.to_csv('oof_preds_stacking.csv', index = False )<import_modules>
model.compile(
 optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["categorical_accuracy"]
)
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.metrics import roc_auc_score
from tqdm import tqdm_notebook as tqdm
from sklearn.covariance import LedoitWolf, OAS
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import RobustScaler
from sklearn.cluster import KMeans, Birch
from sklearn.pipeline import Pipeline
from scipy.stats import multivariate_normal
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from scipy.stats import rankdata
from scipy.optimize import minimize
from sklearn.neighbors import KNeighborsClassifier
import matplotlib.pyplot as plt<load_from_csv>
X_train, X_val, y_train, y_val = model_selection.train_test_split(train, target, test_size=0.20, random_state=123456789 )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
<define_variables>
generator = image.ImageDataGenerator(
 featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
vertical_flip=False,
horizontal_flip=False )
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
oof2 = np.zeros(len(train))
preds2 = np.zeros(len(test))
oof3 = np.zeros(len(train))
preds3 = np.zeros(len(test))
ooff = np.zeros(len(train))
predsf = np.zeros(len(test))
K2=[]
K3=[]
K4=[]
Seed = 777
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ,('scaler', RobustScaler(quantile_range=(35, 65)))])
 sel = pipe.fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 NK = 2
skf = StratifiedKFold(n_splits=10, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
x = train3[train_index]
y = train2.loc[train_index,'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = GaussianMixture(n_components=NK, init_params='random',covariance_type='full', n_init=1,random_state=Seed, reg_covar=0.075 ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = GaussianMixture(n_components=NK, init_params='random', covariance_type='full',
n_init=1,random_state=Seed, reg_covar=0.075 ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
ms = np.stack([m11,m12,m21,m22])
 ps = np.stack([p11,p12,p21,p22])
 gm2 = GaussianMixture(n_components=4,random_state=42, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=1000)
 gm2.fit(np.concatenate([train3[train_index],test3],axis = 0))
oof[idx1[oof_test_index]] = gm2.predict_proba(train3[oof_test_index,:])[:,0:NK].sum(axis=1)
 preds[idx2] += gm2.predict_proba(test3)[:,0:NK].sum(axis=1)/ skf.n_splits
sc2 = roc_auc_score(train2['target'], oof[idx1])
 NK = 3
for train_index, test_index in skf.split(train3, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
x = train3[train_index]
y = train2.loc[train_index,'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = GaussianMixture(n_components=NK, init_params='random',covariance_type='full',
n_init=1,random_state=Seed, reg_covar=0.075 ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
x13 = x1[cc1==2]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
model13 = OAS(assume_centered =False ).fit(x13)
 p13 = model13.precision_
m13 = model13.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = GaussianMixture(n_components=NK, init_params='random', covariance_type='full',
n_init=1,random_state=Seed, reg_covar=0.075 ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
x23 = x2[cc2==2]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
model23 = OAS(assume_centered =False ).fit(x23)
 p23 = model23.precision_
m23 = model23.location_
ms = np.stack([m11,m12,m13,m21,m22,m23])
 ps = np.stack([p11,p12,p13,p21,p22,p23])
 gm3 = GaussianMixture(n_components=6,random_state=42, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=1000)
 gm3.fit(np.concatenate([train3[train_index],test3],axis = 0))
oof2[idx1[oof_test_index]] = gm3.predict_proba(train3[oof_test_index,:])[:,0:NK].sum(axis=1)
 preds2[idx2] += gm3.predict_proba(test3)[:,0:NK].sum(axis=1)/ skf.n_splits
sc3 = roc_auc_score(train2['target'], oof2[idx1])
 NK = 4
for train_index, test_index in skf.split(train3, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
x = train3[train_index]
y = train2.loc[train_index,'target'].values
x1 = x[(y==1 ).astype(bool)]
cc1 = GaussianMixture(n_components=NK, init_params='random', covariance_type='full',
n_init=1,random_state=Seed, reg_covar=0.075 ).fit_predict(x1)
 x11 = x1[cc1==0]
x12 = x1[cc1==1]
x13 = x1[cc1==2]
x14 = x1[cc1==3]
model11 = OAS(assume_centered =False ).fit(x11)
 p11 = model11.precision_
m11 = model11.location_
model12 = OAS(assume_centered =False ).fit(x12)
 p12 = model12.precision_
m12 = model12.location_
model13 = OAS(assume_centered =False ).fit(x11)
 p13 = model13.precision_
m13 = model13.location_
model14 = OAS(assume_centered =False ).fit(x14)
 p14 = model14.precision_
m14 = model14.location_
x2 = x[(y==0 ).astype(bool)]
cc2 = GaussianMixture(n_components=NK, init_params='random', covariance_type='full',
n_init=1,random_state=Seed, reg_covar=0.075 ).fit_predict(x2)
 x21 = x2[cc2==0]
x22 = x2[cc2==1]
x23 = x2[cc2==2]
x24 = x2[cc2==3]
model21 = OAS(assume_centered =False ).fit(x21)
 p21 = model21.precision_
m21 = model21.location_
model22 = OAS(assume_centered =False ).fit(x22)
 p22 = model22.precision_
m22 = model22.location_
model23 = OAS(assume_centered =False ).fit(x21)
 p23 = model23.precision_
m23 = model23.location_
model24 = OAS(assume_centered =False ).fit(x24)
 p24 = model24.precision_
m24 = model24.location_
ms = np.stack([m11,m12,m13,m14,m21,m22,m23,m24])
 ps = np.stack([p11,p12,p13,p14,p21,p22,p23,p24])
 gm4 = GaussianMixture(n_components=8,random_state=42, covariance_type='full', means_init=ms, precisions_init=ps, tol=0.000001, max_iter=1000)
 gm4.fit(np.concatenate([train3[train_index],test3],axis = 0))
oof3[idx1[oof_test_index]] = gm4.predict_proba(train3[oof_test_index,:])[:,0:NK].sum(axis=1)
 preds3[idx2] += gm4.predict_proba(test3)[:,0:NK].sum(axis=1)/ skf.n_splits
sc4 = roc_auc_score(train2['target'], oof3[idx1])
 scmax = max(sc2, sc3, sc4)
 if scmax == sc2:
ooff[idx1]=oof[idx1]
predsf[idx2]=preds[idx2]
elif scmax == sc3:
ooff[idx1]=oof2[idx1]
predsf[idx2]=preds2[idx2]
elif scmax == sc4:
ooff[idx1]=oof2[idx1]
predsf[idx2]=preds3[idx2]
K2.append(sc2)
 K3.append(sc3)
 K4.append(sc4)
 print(sc2,sc3,sc4)
<compute_test_metric>
num_train_sequences = len(X_train)
 num_val_sequences = len(X_val)
 print("
print("
print(roc_auc_score(train['target'], ooff))<save_to_csv>
if(num_train_sequences % BATCH_SIZE)== 0:
steps_per_epoch = int(num_train_sequences / BATCH_SIZE)
 else:
steps_per_epoch = int(num_train_sequences / BATCH_SIZE)+ 1
if(num_val_sequences % BATCH_SIZE)== 0:
validation_steps = int(num_val_sequences / BATCH_SIZE)
 else:
validation_steps = int(num_val_sequences / BATCH_SIZE)+ 1
print("
print("
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = predsf
sub.to_csv('submission.csv',index=False )<set_options>
LR = ReduceLROnPlateau(monitor="val_loss", factor=0.2, patience=2, min_lr=0.000001, verbose=1, cooldown=1 )
warnings.filterwarnings('ignore')
 warnings.filterwarnings('ignore')
<load_from_csv>
def exponential_decay_fn(epoch):
return 0.01 * 0.1**(epoch / 20 )
%%time
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )<feature_engineering>
def exponential_decay(lr, s):
def exponential_decay_fn(epoch):
return lr * 0.1**(epoch / s)
 return exponential_decay_fn
exponential_decay_fn = exponential_decay(lr=0.001, s=20 )
train_columns = [c for c in train_df.columns if c not in ['id','target','wheezy-copper-turtle-magic']]
magic_variance_over2 = {}
for magic in sorted(train_df['wheezy-copper-turtle-magic'].unique()):
temp = train_df.loc[train_df['wheezy-copper-turtle-magic']==magic]
std = temp[train_columns].std()
magic_variance_over2[magic] = list(std.index.values[np.where(std >2)] )<import_modules>
lr_scheduler = keras.callbacks.LearningRateScheduler(exponential_decay_fn )
from sklearn.neighbors import KNeighborsClassifier, RadiusNeighborsClassifier
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import Lasso, LassoLars<define_variables>
callbacks = [
checkpoint,
LR,
lr_scheduler]
random_state = 42
debug = True
debug = False<init_hyperparams>
VERBOSE = 1
history = model.fit_generator(generator.flow(X_train, y_train, batch_size=BATCH_SIZE),
validation_data=generator.flow(X_val, y_val, batch_size=BATCH_SIZE),
epochs=NP_EPOCHS,
verbose=VERBOSE,
steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps,
class_weight=None,
initial_epoch=0,
callbacks=callbacks )
svnu_params = {'probability':True, 'kernel':'poly','degree':4,'gamma':'auto','nu':0.4,'coef0':0.08, 'random_state':4}
svnu2_params = {'probability':True, 'kernel':'poly','degree':2,'gamma':'auto','nu':0.4,'coef0':0.08, 'random_state':4}
svc_params = {'probability':True,'kernel':'poly','degree':4,'gamma':'auto', 'random_state':4}
lr_params = {'solver':'liblinear','penalty':'l1','C':0.05,'n_jobs':-1, 'random_state':42}
mlp16_params = {'activation':'relu','solver':'lbfgs','tol':1e-06, 'hidden_layer_sizes':(16,), 'random_state':42}
mlp128_params = {'activation':'relu','solver':'lbfgs','tol':1e-06, 'hidden_layer_sizes':(128,), 'random_state':42}
gnb_params = {}<define_variables>
values = {}
models = os.listdir(model_name)
 for model in models:
converted = model.replace(".h5", "")
 accuracy = float(converted.split("-")[-1])
 values.update({accuracy: model})
 key = max(values, key = values.get)
 best = values.get(key)
 print("Best model found: {}".format(best))
def get_oofs(random_state):
oof_nusvc = np.zeros(len(train_df))
preds_nusvc = np.zeros(len(test_df))
oof_nusvc2 = np.zeros(len(train_df))
preds_nusvc2 = np.zeros(len(test_df))
oof_qda = np.zeros(len(train_df))
preds_qda = np.zeros(len(test_df))
oof_svc = np.zeros(len(train_df))
preds_svc = np.zeros(len(test_df))
oof_knn = np.zeros(len(train_df))
preds_knn = np.zeros(len(test_df))
oof_lr = np.zeros(len(train_df))
preds_lr = np.zeros(len(test_df))
oof_gnb = np.zeros(len(train_df))
preds_gnb = np.zeros(len(test_df))
cols = [c for c in train_df.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
for i in tqdm_notebook(range(512)) :
train = train_df[train_df['wheezy-copper-turtle-magic'] == i]
test = test_df[test_df['wheezy-copper-turtle-magic'] == i]
train_idx_origin = train.index
test_idx_origin = test.index
cols = magic_variance_over2[i]
X_train = train.reset_index(drop=True)[cols].values
y_train = train.reset_index(drop=True ).target
X_test = test[cols].values
data = np.vstack([X_train, X_test])
 data = KernelPCA(n_components=len(cols), kernel='cosine', random_state=random_state ).fit_transform(data)

gmm = GMM(n_components=5, random_state=random_state, max_iter=1000 ).fit(data)
 gmm_pred = gmm.predict_proba(data)
 gmm_score = gmm.score_samples(data)
 gmm_label = gmm.predict(data)
 hist = hist_model() ; hist.fit(data)
 hist_pred = hist.predict(data ).reshape(-1, 1)
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, hist_pred, gmm_score.reshape(-1, 1)])
 data = np.hstack([data, gmm_score.reshape(-1, 1)])
 data = np.hstack([data, gmm_score.reshape(-1, 1)])
 data = StandardScaler().fit_transform(data)
 X_train = data[:X_train.shape[0]]
X_test = data[X_train.shape[0]:]
fold = StratifiedKFold(n_splits=5, random_state=random_state)
 for tr_idx, val_idx in fold.split(X_train, gmm_label[:X_train.shape[0]]):
clf = svm.NuSVC(**svnu_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_nusvc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_nusvc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = svm.NuSVC(**svnu2_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_nusvc2[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_nusvc2[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = QuadraticDiscriminantAnalysis(reg_param=0.111)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_qda[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_qda[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = svm.SVC(**svc_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_svc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_svc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = KNeighborsClassifier(n_neighbors=16)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_knn[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_knn[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = linear_model.LogisticRegression(**lr_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_lr[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_lr[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
oof_train = pd.DataFrame()
oof_train['nusvc'] = oof_nusvc
oof_train['nusvc2'] = oof_nusvc2
oof_train['qda'] = oof_qda
oof_train['svc'] = oof_svc
oof_train['knn'] = oof_knn
oof_train['lr'] = oof_lr
oof_test = pd.DataFrame()
oof_test['nusvc'] = preds_nusvc
oof_test['nusvc2'] = preds_nusvc2
oof_test['qda'] = preds_qda
oof_test['svc'] = preds_svc
oof_test['knn'] = preds_knn
oof_test['lr'] = preds_lr
print('nusvc', roc_auc_score(train_df['target'], oof_nusvc))
print('nusvc2', roc_auc_score(train_df['target'], oof_nusvc2))
print('qda', roc_auc_score(train_df['target'], oof_qda))
print('svc', roc_auc_score(train_df['target'], oof_svc))
print('knn', roc_auc_score(train_df['target'], oof_knn))
print('lr', roc_auc_score(train_df['target'], oof_lr))
return oof_train, oof_test<define_variables>
model_path = model_name + best
print("Absolute path found: {}".format(model_path))
def get_oofs_2(random_state):
oof_nusvc = np.zeros(len(train_df))
preds_nusvc = np.zeros(len(test_df))
oof_nusvc2 = np.zeros(len(train_df))
preds_nusvc2 = np.zeros(len(test_df))
oof_qda = np.zeros(len(train_df))
preds_qda = np.zeros(len(test_df))
oof_svc = np.zeros(len(train_df))
preds_svc = np.zeros(len(test_df))
oof_knn = np.zeros(len(train_df))
preds_knn = np.zeros(len(test_df))
oof_lr = np.zeros(len(train_df))
preds_lr = np.zeros(len(test_df))
oof_gnb = np.zeros(len(train_df))
preds_gnb = np.zeros(len(test_df))
cols = [c for c in train_df.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
for i in tqdm_notebook(range(512)) :
train = train_df[train_df['wheezy-copper-turtle-magic'] == i]
test = test_df[test_df['wheezy-copper-turtle-magic'] == i]
train_idx_origin = train.index
test_idx_origin = test.index
cols = magic_variance_over2[i]
X_train = train.reset_index(drop=True)[cols].values
y_train = train.reset_index(drop=True ).target
X_test = test[cols].values
data = np.vstack([X_train, X_test])
 data = KernelPCA(n_components=len(cols), kernel='cosine', random_state=random_state ).fit_transform(data)

gmm = GMM(n_components=5, random_state=random_state, max_iter=1000, init_params='random' ).fit(data)
 gmm_pred = gmm.predict_proba(data)
 gmm_score = gmm.score_samples(data)
 gmm_label = gmm.predict(data)
 hist = hist_model() ; hist.fit(data)
 hist_pred = hist.predict(data ).reshape(-1, 1)
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, gmm_pred])
 data = np.hstack([data, hist_pred, gmm_score.reshape(-1, 1)])
 data = np.hstack([data, gmm_score.reshape(-1, 1)])
 data = np.hstack([data, gmm_score.reshape(-1, 1)])
 data = StandardScaler().fit_transform(data)
 X_train = data[:X_train.shape[0]]
X_test = data[X_train.shape[0]:]
fold = StratifiedKFold(n_splits=5, random_state=random_state)
 for tr_idx, val_idx in fold.split(X_train, gmm_label[:X_train.shape[0]]):
clf = svm.NuSVC(**svnu_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_nusvc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_nusvc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = svm.NuSVC(**svnu2_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_nusvc2[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_nusvc2[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = QuadraticDiscriminantAnalysis(reg_param=0.111)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_qda[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_qda[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = svm.SVC(**svc_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_svc[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_svc[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = KNeighborsClassifier(n_neighbors=16)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_knn[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_knn[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
clf = linear_model.LogisticRegression(**lr_params)
 clf.fit(X_train[tr_idx], y_train[tr_idx])
 oof_lr[train_idx_origin[val_idx]] = clf.predict_proba(X_train[val_idx])[:,1]
preds_lr[test_idx_origin] += clf.predict_proba(X_test)[:,1] / fold.n_splits
oof_train = pd.DataFrame()
oof_train['nusvc'] = oof_nusvc
oof_train['nusvc2'] = oof_nusvc2
oof_train['qda'] = oof_qda
oof_train['svc'] = oof_svc
oof_train['knn'] = oof_knn
oof_train['lr'] = oof_lr
oof_test = pd.DataFrame()
oof_test['nusvc'] = preds_nusvc
oof_test['nusvc2'] = preds_nusvc2
oof_test['qda'] = preds_qda
oof_test['svc'] = preds_svc
oof_test['knn'] = preds_knn
oof_test['lr'] = preds_lr
print('nusvc', roc_auc_score(train_df['target'], oof_nusvc))
print('nusvc2', roc_auc_score(train_df['target'], oof_nusvc2))
print('qda', roc_auc_score(train_df['target'], oof_qda))
print('svc', roc_auc_score(train_df['target'], oof_svc))
print('knn', roc_auc_score(train_df['target'], oof_knn))
print('lr', roc_auc_score(train_df['target'], oof_lr))
print('gnb', roc_auc_score(train_df['target'], oof_gnb))
return oof_train, oof_test<feature_engineering>
model = load_model(model_path )
oof_train_1, oof_test_1 = get_oofs(1)
 oof_train_2, oof_test_2 = get_oofs(2)
 oof_train_3, oof_test_3 = get_oofs_2(1)
 oof_train_4, oof_test_4 = get_oofs_2(2 )<compute_test_metric>
predictions = model.predict(test )
x_train_second_layer = oof_train_1 + oof_train_2 + oof_train_3 + oof_train_4
x_test_second_layer = oof_test_1 + oof_test_2 + oof_test_3 + oof_test_4
print('Ensemble', roc_auc_score(train_df['target'], x_train_second_layer.mean(1)) )<save_to_csv>
conversion = np.argmax(predictions, axis=1 )
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit["target"] = x_test_second_layer.mean(1)
 submit.to_csv("submission0.csv", index=False )<categorify>
print("No.f Id's available in the test dataset: {}".format(len(test_df['id'])))
 print("No.f predictions done by the model on test data: {}".format(len(result)) )
def time_decorator(func):
@wraps(func)
 def wrapper(*args, **kwargs):
print("
StartTime: ", datetime.now() + timedelta(hours=9))
start_time = time.time()
df = func(*args, **kwargs)
 print("EndTime: ", datetime.now() + timedelta(hours=9))
print("TotalTime: ", time.time() - start_time)
 return df
return wrapper
class SklearnWrapper(object):
def __init__(self, clf, params=None, **kwargs):

 if 'seed' in kwargs:
params['random_state'] = kwargs.get('seed', 0)
 self.clf = clf(**params)
 self.is_classification_problem = True
def train(self, x_train, y_train, x_cross=None, y_cross=None):
if len(np.unique(y_train)) > 30:
self.is_classification_problem = False
self.clf.fit(x_train, y_train)
 def predict(self, x):
if self.is_classification_problem is True:
return self.clf.predict_proba(x)[:,1]
else:
return self.clf.predict(x)
 class LgbmWrapper(object):
def __init__(self, params=None, **kwargs):
self.param = params
if 'seed' in kwargs:
self.param['seed'] = kwargs.get('seed', 0)
 self.num_rounds = kwargs.get('num_rounds', 1000)
 self.early_stopping = kwargs.get('ealry_stopping', 100)
 self.eval_function = kwargs.get('eval_function', None)
 self.verbose_eval = kwargs.get('verbose_eval', 100)
 self.best_round = 0
self.feature_importance = pd.DataFrame()
def train(self, x_train, y_train, x_cross=None, y_cross=None):

 if isinstance(y_train, pd.DataFrame)is True:
y_train = y_train[y_train.columns[0]]
if y_cross is not None:
y_cross = y_cross[y_cross.columns[0]]
if x_cross is None:
dtrain = lgb.Dataset(x_train, label=y_train, silent= True)
 train_round = self.best_round
if self.best_round == 0:
train_round = self.num_rounds
self.clf = lgb.train(self.param, train_set=dtrain, num_boost_round=train_round)
 del dtrain
else:
dtrain = lgb.Dataset(x_train, label=y_train, silent=True)
 dvalid = lgb.Dataset(x_cross, label=y_cross, silent=True)
 self.clf = lgb.train(self.param, train_set=dtrain, num_boost_round=self.num_rounds, valid_sets=[dtrain, dvalid],
feval=self.eval_function, early_stopping_rounds=self.early_stopping,
verbose_eval=self.verbose_eval)
 try:
self.feature_importance = pd.DataFrame()
self.feature_importance["Feature"] = x_train.columns
self.feature_importance["Importance"] = self.clf.feature_importance()
except:
pass
self.best_round = max(self.best_round, self.clf.best_iteration)
 del dtrain, dvalid
gc.collect()
def get_importance_df(self):
return self.feature_importance
def predict(self, x):
return self.clf.predict(x, num_iteration=self.clf.best_iteration)
 def plot_importance(self):
lgb.plot_importance(self.clf, max_num_features=50, height=0.7, figsize=(10,30))
plt.show()
def get_params(self):
return self.param<split>
submission = pd.concat([test_df['id'], result],axis = 1 )
def get_oof(clf, x_train, y_train, x_test, eval_func, **kwargs):
nfolds = kwargs.get('NFOLDS', 5)
 kfold_shuffle = kwargs.get('kfold_shuffle', True)
 kfold_random_state = kwargs.get('kfold_random_state', 0)
 stratified_kfold_ytrain = kwargs.get('stratifed_kfold_y_value', None)
 inner_predict = kwargs.get('inner_predict', True)
 export_feature_importance = kwargs.get('export_feature_importance', True)
 ntrain = x_train.shape[0]
ntest = x_test.shape[0]
kf_split = None
if stratified_kfold_ytrain is None:
kf = KFold(n_splits=nfolds, shuffle=kfold_shuffle, random_state=kfold_random_state)
 kf_split = kf.split(x_train)
 else:
kf = StratifiedKFold(n_splits=nfolds, shuffle=kfold_shuffle, random_state=kfold_random_state)
 kf_split = kf.split(x_train, stratified_kfold_ytrain)
 oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
cv_sum = 0

 feature_importance_df = pd.DataFrame()
for i,(train_index, cross_index)in enumerate(kf_split):
x_tr, x_cr = None, None
y_tr, y_cr = None, None
if isinstance(x_train, pd.DataFrame):
x_tr, x_cr = x_train.iloc[train_index], x_train.iloc[cross_index]
y_tr, y_cr = y_train.iloc[train_index], y_train.iloc[cross_index]
else:
x_tr, x_cr = x_train[train_index], x_train[cross_index]
y_tr, y_cr = y_train[train_index], y_train[cross_index]
clf.train(x_tr, y_tr, x_cr, y_cr)
 if isinstance(clf, LgbmWrapper)is True:
feature_importance_df = pd.concat([feature_importance_df, clf.get_importance_df() ], axis=0)
 oof_train[cross_index] = clf.predict(x_cr)
 if inner_predict is True:
oof_test += clf.predict(x_test)
 cv_score = eval_func(y_cr, oof_train[cross_index])
 cv_sum = cv_sum + cv_score
del x_tr, x_cr, y_tr, y_cr
gc.collect()
score = cv_sum / nfolds
if export_feature_importance is True:
print("Export Feature Importance")
 filename = '{}_cv{:.6f}'.format(datetime.now().strftime('%Y%m%d_%H%M%S'), score)
 if os.path.isdir("importance/")is True:
feature_importance_df.to_csv('importance/importance_{}.csv'.format(filename),index=False)
 else:
feature_importance_df.to_csv('importance_{}.csv'.format(filename),index=False)
 if inner_predict is True:
oof_test = oof_test/nfolds
else:
clf.train(x_train, y_train)
 oof_test = clf.predict(x_test)
 return oof_train, oof_test, score<create_dataframe>
submission.to_csv("submission.csv", index=False )