Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<split>
sns.set()
%matplotlib inline
def train_qda(Y_pseudo, low, high, params=params_qda):
preds, oof, auc_all = initialize_cv()
print(f"Computing centroids and covariances for the four clusters(two per class ).")
 for i in tqdm(range(magic_min, magic_max+1)) :
X, X_test, Y, idx_train, idx_test = get_data(i=i, data=data)
 auc_folds=np.array([])
 folds = StratifiedKFold(n_splits=NFOLDS, random_state=RS)
 for train_index, val_index in folds.split(X, Y):
X_train, Y_train = X[train_index, :], Y[train_index]
X_val, Y_val = X[val_index, :], Y[val_index]
clf = QuadraticDiscriminantAnalysis(**params)
 if Y_pseudo is None:
X_l, Y_l = get_labels(X_train, Y_train)
 else:
X_aug, Y_aug = pseudolabeling_qda(X_train, X_test, Y_train, Y_pseudo, idx_test, low, high)
 X_l, Y_l = get_labels(X_aug, Y_aug.ravel())
 clf.fit(X_l, Y_l.ravel())
 oof[idx_train[val_index]] = np.sum(clf.predict_proba(X_val)[:, 2:], axis=1)
 preds[idx_test] += np.sum(clf.predict_proba(X_test)[:,2: ], axis=1)/NFOLDS
auc = roc_auc_score(Y_val, oof[idx_train[val_index]])
 auc_folds = np.append(auc_folds, auc)
 auc_all = np.append(auc_all, np.mean(auc_folds))
report_results(oof, auc_all, clf_name='QDA')
 return preds, oof, auc_all<save_to_csv>
config = ConfigProto()
config.gpu_options.allow_growth = True
session = InteractiveSession(config=config )
Y_pseudo=preds_gmm
for rp, low, high in zip(rp_values, low_vals, high_vals):
parmas_qda = {'reg_param': rp}
Y_pseudo, oof_qda, auc_qda = train_qda(Y_pseudo=Y_pseudo, low=low, high=high, params=params_qda)
 preds_qda = Y_pseudo
sub['target'] = preds_qda
sub.to_csv('submission_qda.csv',index=False )<define_variables>
train = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 test = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 dig = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv")
 sample = pd.read_csv(".. /input/Kannada-MNIST/sample_submission.csv" )
preds_highest = preds_gmm
oof_highest = oof_gmm
mask =(auc_qda > auc_gmm)
 print(f"The number of models where QDA's predictions are better is {sum(mask)}." )<compute_test_metric>
x_train = train.drop(columns=['label'])
 y_train = to_categorical(train.label)
 x_test = test.drop(columns="id")
for i in tqdm(range(magic_min, magic_max+1)) :
if mask[i]:
_, _, _, idx_train, idx_test = get_data(i=i, data=data)
 oof_highest[idx_train] = oof_qda[idx_train]
preds_highest[idx_test] = preds_qda[idx_test]
auc = roc_auc_score(train['target'].values, oof_highest)
 print(f"The 'highest' ROC AUC score is {auc}." )<save_to_csv>
x_train[train.label == 0].iloc[0].to_numpy().astype(np.uint8 ).reshape(28, 28 )
sub['target'] = preds_highest
sub.to_csv('submission_highest.csv',index=False )<compute_train_metric>
fig, ax = plt.subplots(ncols=10, figsize=(15,15))
for i in range(10):
kannada = x_train[train.label == i]
ax[i].set_title(i)
 ax[i].axis('off')
 ax[i].imshow(kannada.iloc[0, :].to_numpy().astype(np.uint8 ).reshape(28, 28))
oof_all = pd.DataFrame()
preds_all = pd.DataFrame()
oof_all['gmm'] = rankdata(oof_gmm)/len(oof_gmm)
 oof_all['qda'] = rankdata(oof_qda)/len(oof_qda)
 preds_all['gmm'] = rankdata(preds_gmm)/len(preds_gmm)
 preds_all['qda'] = rankdata(preds_qda)/len(preds_qda)
 lr = LogisticRegression()
lr.fit(oof_all.values, train['target'].values)
 preds_lr = lr.predict_proba(preds_all.values)[:,1]
preds_train = lr.predict_proba(oof_all)[:,1]
auc = roc_auc_score(train['target'].values, preds_train)
 print(f"The final ROC AUC score is {auc}." )<save_to_csv>
x_train = x_train /255
x_test = x_test / 255
sub['target'] = preds_lr
sub.to_csv('submission_lr.csv',index=False )<save_to_csv>
x_train = x_train.values.reshape(-1,28,28,1)
 x_test = x_test.values.reshape(-1,28,28,1 )
w = 0.02
mask =(preds_gmm <(0.5 + w)) &(preds_gmm >(0.5 - w))
preds = rankdata(preds_gmm)/len(preds_gmm)
 preds[mask] = preds_lr[mask]
sub['target'] = preds
sub.to_csv('submission_picking.csv',index=False )<set_options>
X_train, X_test, Y_train, Y_test = train_test_split(x_train, y_train, test_size=0.2, random_state=42 )
warnings.filterwarnings("ignore")
<load_from_csv>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1 )
train = pd.read_csv('.. /input/train.csv')
 train_t = train.copy()
test = pd.read_csv('.. /input/test.csv' )<define_variables>
datagen.fit(X_train )
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
reg_best = [0.5,
0.2,
0.3,
0.2,
0.5,
0.1,
0.1,
0.2,
0.3,
0.5,
0.2,
0.4,
0.1,
0.3,
0.1,
0.4,
0.3,
0.2,
0.2,
0.5,
0.1,
0.4,
0.4,
0.1,
0.5,
0.4,
0.1,
0.4,
0.4,
0.1,
0.1,
0.3,
0.4,
0.1,
0.5,
0.2,
0.3,
0.1,
0.1,
0.5,
0.5,
0.5,
0.3,
0.5,
0.4,
0.1,
0.1,
0.1,
0.5,
0.5,
0.5,
0.1,
0.3,
0.1,
0.1,
0.4,
0.2,
0.3,
0.1,
0.1,
0.5,
0.2,
0.4,
0.1,
0.1,
0.1,
0.1,
0.4,
0.2,
0.1,
0.1,
0.5,
0.4,
0.1,
0.3,
0.2,
0.4,
0.1,
0.3,
0.5,
0.1,
0.5,
0.1,
0.5,
0.1,
0.1,
0.4,
0.5,
0.4,
0.2,
0.1,
0.1,
0.4,
0.5,
0.2,
0.5,
0.5,
0.4,
0.1,
0.5,
0.5,
0.3,
0.5,
0.2,
0.4,
0.4,
0.1,
0.4,
0.4,
0.1,
0.1,
0.5,
0.5,
0.5,
0.1,
0.2,
0.4,
0.1,
0.4,
0.5,
0.5,
0.5,
0.2,
0.2,
0.2,
0.5,
0.1,
0.1,
0.3,
0.5,
0.3,
0.1,
0.4,
0.1,
0.3,
0.1,
0.2,
0.5,
0.5,
0.1,
0.1,
0.1,
0.4,
0.1,
0.5,
0.5,
0.5,
0.1,
0.5,
0.5,
0.1,
0.5,
0.5,
0.2,
0.4,
0.2,
0.1,
0.5,
0.3,
0.5,
0.2,
0.4,
0.4,
0.5,
0.2,
0.3,
0.1,
0.1,
0.5,
0.1,
0.5,
0.5,
0.5,
0.5,
0.1,
0.5,
0.4,
0.1,
0.4,
0.3,
0.4,
0.4,
0.3,
0.1,
0.4,
0.4,
0.2,
0.5,
0.4,
0.4,
0.2,
0.1,
0.2,
0.5,
0.5,
0.1,
0.5,
0.3,
0.4,
0.5,
0.1,
0.5,
0.5,
0.5,
0.1,
0.1,
0.3,
0.2,
0.5,
0.1,
0.5,
0.5,
0.4,
0.1,
0.5,
0.1,
0.5,
0.1,
0.3,
0.3,
0.1,
0.1,
0.1,
0.4,
0.3,
0.1,
0.1,
0.4,
0.3,
0.3,
0.4,
0.5,
0.2,
0.1,
0.5,
0.5,
0.4,
0.4,
0.3,
0.1,
0.1,
0.5,
0.1,
0.1,
0.1,
0.1,
0.3,
0.3,
0.2,
0.1,
0.5,
0.4,
0.3,
0.1,
0.3,
0.1,
0.2,
0.4,
0.5,
0.3,
0.1,
0.1,
0.3,
0.3,
0.4,
0.4,
0.2,
0.5,
0.1,
0.5,
0.3,
0.1,
0.2,
0.5,
0.1,
0.1,
0.5,
0.4,
0.1,
0.5,
0.5,
0.5,
0.3,
0.2,
0.4,
0.5,
0.4,
0.3,
0.1,
0.4,
0.3,
0.2,
0.2,
0.1,
0.4,
0.4,
0.1,
0.2,
0.1,
0.5,
0.3,
0.2,
0.1,
0.2,
0.3,
0.2,
0.5,
0.4,
0.5,
0.5,
0.1,
0.1,
0.4,
0.3,
0.3,
0.4,
0.3,
0.2,
0.5,
0.4,
0.1,
0.1,
0.4,
0.1,
0.1,
0.5,
0.4,
0.1,
0.4,
0.5,
0.3,
0.2,
0.5,
0.4,
0.4,
0.5,
0.1,
0.1,
0.5,
0.5,
0.5,
0.1,
0.5,
0.1,
0.5,
0.2,
0.1,
0.1,
0.1,
0.5,
0.5,
0.4,
0.5,
0.1,
0.3,
0.5,
0.5,
0.3,
0.5,
0.1,
0.3,
0.1,
0.4,
0.3,
0.5,
0.5,
0.5,
0.4,
0.2,
0.5,
0.5,
0.5,
0.5,
0.1,
0.1,
0.1,
0.5,
0.4,
0.3,
0.1,
0.5,
0.5,
0.2,
0.3,
0.5,
0.5,
0.1,
0.1,
0.1,
0.5,
0.3,
0.1,
0.4,
0.1,
0.1,
0.5,
0.5,
0.4,
0.1,
0.5,
0.4,
0.2,
0.5,
0.1,
0.4,
0.1,
0.1,
0.1,
0.4,
0.2,
0.1,
0.2,
0.2,
0.5,
0.4,
0.1,
0.1,
0.1,
0.4,
0.5,
0.4,
0.1,
0.1,
0.1,
0.1,
0.1,
0.1,
0.3,
0.5,
0.3,
0.5,
0.5,
0.5,
0.5,
0.5,
0.3,
0.5,
0.5,
0.1,
0.5,
0.1,
0.1,
0.1,
0.1,
0.2,
0.1,
0.5,
0.5,
0.1,
0.1,
0.4,
0.3,
0.1,
0.2,
0.1,
0.1,
0.1,
0.3,
0.5,
0.2,
0.1,
0.3,
0.2,
0.4,
0.4,
0.2,
0.1,
0.3,
0.1,
0.1,
0.4,
0.1,
0.2,
0.4,
0.5,
0.3,
0.1,
0.1,
0.5,
0.5,
0.5,
0.5,
0.5,
0.1,
0.1,
0.1,
0.1,
0.4,
0.1,
0.4,
0.2,
0.1,
0.1,
0.4,
0.1,
0.5,
0.2,
0.1,
0.1,
0.3,
0.5,
0.1,
0.5,
0.5,
0.1,
0.1,
0.2,
0.1,
0.1,
0.1,
0.2,
0.3]<create_dataframe>
EPOCHS=50
BATCH_SIZE=64
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(threshold=2 ).fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_best[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
<compute_test_metric>
callbacks = [ReduceLROnPlateau(monitor='val_accuracy',
patience=5,
verbose=1,
factor=0.5,
min_lr=0.00001)]
auc = roc_auc_score(train_t['target'], oof)
 print(f'AUC: {auc:.5}' )<feature_engineering>
model = Sequential()
model.add(Conv2D(32,kernel_size=3,activation='relu',input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=3,activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64,kernel_size=5,strides=2,padding='same',activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer=Adam(lr=0.001), loss="categorical_crossentropy", metrics=["accuracy"] )
train.loc[oof > 0.99, 'target'] = 1
train.loc[oof < 0.01, 'target'] = 0<concatenate>
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=BATCH_SIZE),
epochs = EPOCHS,
validation_data =(X_test,Y_test),
verbose = 1,
steps_per_epoch=X_train.shape[0] // BATCH_SIZE,
callbacks=callbacks )
oof_ls = np.zeros(len(train))
pred_te_ls = np.zeros(len(test))
for k in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2['target']=-1
train2p = pd.concat([train2,test2],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train4p = sel.transform(train2p[cols])
 train4 = sel.transform(train2[cols])
 test4 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train4p, train2p['target']):
test_index3 = test_index[ test_index<len(train4)]
clf = LabelSpreading(gamma=0.01,kernel='rbf', max_iter=10,n_jobs=-1)
 clf.fit(train4p[train_index,:],train2p.loc[train_index]['target'])
 oof_ls[idx1[test_index3]] = clf.predict_proba(train4[test_index3,:])[:,1]
pred_te_ls[test2.index] += clf.predict_proba(test4)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_ls)
 print('CV for LabelSpreading =',round(auc,5))<concatenate>
%%time
pred = model.predict(X_test)
 pred = np.argmax(pred,axis=1)
 target = np.argmax(Y_test,axis=1 )
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 data2 = VarianceThreshold(threshold=2 ).fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
train4 = np.hstack([train3,np.array([oof_ls[idx1]] ).T])
 test4 = np.hstack([test3,np.array([pred_te_ls[idx2]] ).T])
 skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train4, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_best[i])
 clf.fit(train4[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train4[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test4)[:,1] / skf.n_splits
auc = roc_auc_score(train_t['target'], oof)
 print(f'AUC: {auc:.5}' )<save_to_csv>
cm = confusion_matrix(target, pred)
 print(cm )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False )<load_from_csv>
accuracy_score(target, pred )
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape)
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0
oof2 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=time.time)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof2[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof2)
 print(f'AUC: {auc:.5}')
 sub1 = pd.read_csv('.. /input/sample_submission.csv')
 sub1['target'] = preds 
<load_from_csv>
print(classification_report(target, pred))
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape)
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 for itr in range(4):
test['target'] = preds
test.loc[test['target'] > 0.94, 'target'] = 1
test.loc[test['target'] < 0.06, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.98, 'target'] = 1
new_train.loc[oof < 0.02, 'target'] = 0
oof2 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof2[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof2)
 print(f'AUC: {auc:.5}')
 sub2 = pd.read_csv('.. /input/sample_submission.csv')
 sub2['target'] = preds 
<load_from_csv>
submission_pred = model.predict(x_test )
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape)
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
params = [{'reg_param': [0.1, 0.2, 0.3, 0.4, 0.5]}]
reg_params = np.zeros(512)
 for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42)
 for train_index, test_index in skf.split(train2, train2['target']):
qda = QuadraticDiscriminantAnalysis()
clf = GridSearchCV(qda, params, cv=4)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 reg_params[i] = clf.best_params_['reg_param']
oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 for itr in range(10):
test['target'] = preds
test.loc[test['target'] > 0.955, 'target'] = 1
test.loc[test['target'] < 0.045, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof > 0.995, 'target'] = 1
new_train.loc[oof < 0.005, 'target'] = 0
oof2 = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = new_train[new_train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train[train['wheezy-copper-turtle-magic']==i].index
idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) ,('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=time.time)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
clf = QuadraticDiscriminantAnalysis(reg_params[i])
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 if len(oof_test_index)> 0:
oof2[idx1[oof_test_index]] = clf.predict_proba(train3[oof_test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof2)
 print(f'AUC: {auc:.5}')
 sub3 = pd.read_csv('.. /input/sample_submission.csv')
 sub3['target'] = preds 
<load_from_csv>
submission_pred = np.argmax(submission_pred, axis=1 )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.head()<feature_engineering>
df_submission = pd.DataFrame({'id': np.arange(0,submission_pred.shape[0]),
'label':submission_pred} )
<save_to_csv><EOS> 
df_submission.to_csv('submission.csv',index=False )