Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<load_from_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
test= pd.read_csv(".. /input/test.csv")
 print(test.shape)
 test.head()<prepare_x_and_y>
def get_mv(x,y):
ones =(y==1 ).astype(bool)
 x2 = x[ones]
cov1 = np.cov(x2.T)
 m1 = np.mean(x2, axis = 0)
 zeros =(y==0 ).astype(bool)
 x2b = x[zeros]
cov2 = np.cov(x2b.T)
 m2 = np.mean(x2b, axis = 0)
 mv1 = multivariate_normal(mean=m1, cov=cov1)
 mv2 = multivariate_normal(mean=m2, cov=cov2)
 return mv1, mv2
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_train = X_train / 255.0
X_test = test / 255.0
X_train = X_train.values.reshape(-1,28,28,1)
 X_test = X_test.values.reshape(-1,28,28,1)
 Y_train = to_categorical(Y_train, num_classes = 10 )<create_dataframe>
def calc_prob(x,mv1,mv2):
y_pred2 = np.zeros(( len(x),))
for i in range(len(x)) :
a = mv1.pdf(x[i])
 b = mv2.pdf(x[i])
 y_pred2[i] = a/(a+b)
 return y_pred2
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )<choose_model_class>
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
mv1, mv2 = get_mv(train3[train_index,:],train2.loc[train_index]['target'].values)
 oof[idx1[test_index]] = calc_prob(train3[test_index,:],mv1,mv2)
 preds[idx2] += calc_prob(test3,mv1,mv2)/ skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))
nets = 10
model = [0] *nets
for j in range(nets):
model[j] = Sequential()
model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Dropout(0.4))
model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))
model[j].add(BatchNormalization())
 model[j].add(Flatten())
 model[j].add(Dropout(0.4))
model[j].add(Dense(10, activation='softmax'))
model[j].compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )<train_model>
test['target'] = preds
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for k in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
mv1, mv2 = get_mv(train3p[train_index,:],train2p.loc[train_index]['target'].values)
 oof[idx1[test_index3]] = calc_prob(train3[test_index3,:],mv1,mv2)
 preds[test2.index] += calc_prob(test3,mv1,mv2)/ skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * nets
epochs = 45
for j in range(nets):
X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=0)
 print("CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc'])) )<save_to_csv>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()