Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<train_model>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
learn.fit(1, 1e-3 )<predict_on_test>
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps
preds, targets = learn.get_preds()
predictions = np.argmax(preds, axis = 1)
 pd.crosstab(predictions, targets )<predict_on_test>
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gm = GaussianMixture(n_components=2, init_params='random', covariance_type='full', tol=0.001,reg_covar=0.001, max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gm.fit(np.concatenate([train3,test3],axis = 0))
oof[idx1[test_index]] = gm.predict_proba(train3[test_index,:])[:,0]
preds[idx2] += gm.predict_proba(test3)[:,0] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))
predictions, *_ = learn.get_preds(DatasetType.Test)
 labels = np.argmax(predictions, 1 )<save_to_csv>
cat_dict = dict()
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 cat_dict[i] = train3.shape[1]
sub_df = pd.DataFrame({'PassengerId': test_df['PassengerId'], 'Survived': labels})
 sub_df.to_csv('submission.csv', index=False )<set_options>
test['target'] = preds
oof_qda = np.zeros(len(train))
preds_qda = np.zeros(len(test))
oof_knn = np.zeros(len(train))
preds_knn = np.zeros(len(test))
oof_svnu = np.zeros(len(train))
preds_svnu = np.zeros(len(test))
oof_svc = np.zeros(len(train))
preds_svc = np.zeros(len(test))
oof_rf = np.zeros(len(train))
preds_rf = np.zeros(len(test))
oof_mlp = np.zeros(len(train))
preds_mlp = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 pca = PCA(n_components=cat_dict[k], random_state= 1234)
 pca.fit(train2p[cols])
 train3p = pca.transform(train2p[cols])
 train3 = pca.transform(train2[cols])
 test3 = pca.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_qda[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_qda[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = neighbors.KNeighborsClassifier(n_neighbors=17, p=2.9)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_knn[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_knn[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = NuSVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=4, nu=0.59, coef0=0.053)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_svnu[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_svnu[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = svm.SVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=42)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_svc[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_svc[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = RandomForestClassifier(n_estimators=100,random_state=1)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_rf[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_rf[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = neural_network.MLPClassifier(random_state=3, activation='relu', solver='lbfgs', tol=1e-06, hidden_layer_sizes=(250,))
clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_mlp[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_mlp[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
if k%32==0: print(k)
 auc = roc_auc_score(train['target'],oof_qda)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))
auc = roc_auc_score(train['target'],oof_knn)
 print('Pseudo Labeled KNN scores CV =',round(auc,5))
auc = roc_auc_score(train['target'],oof_svnu)
 print('Pseudo Labeled SVNU scores CV =',round(auc,5))
auc = roc_auc_score(train['target'],oof_svc)
 print('Pseudo Labeled SVC scores CV =',round(auc,5))
auc = roc_auc_score(train['target'],oof_rf)
 print('Pseudo Labeled RF scores CV =',round(auc,5))
auc = roc_auc_score(train['target'],oof_mlp)
 print('Pseudo Labeled MLP scores CV =',round(auc,5))
warnings.filterwarnings('ignore' )<load_from_csv>
test['target'] = preds
oof_qda2 = np.zeros(len(train))
preds_qda2 = np.zeros(len(test))
oof_knn2 = np.zeros(len(train))
preds_knn2 = np.zeros(len(test))
oof_svnu2 = np.zeros(len(train))
preds_svnu2 = np.zeros(len(test))
oof_svc2 = np.zeros(len(train))
preds_svc2 = np.zeros(len(test))
oof_rf2 = np.zeros(len(train))
preds_rf2 = np.zeros(len(test))
oof_mlp2 = np.zeros(len(train))
preds_mlp2 = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_qda2[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_qda2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = neighbors.KNeighborsClassifier(n_neighbors=17, p=2.9)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_knn2[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_knn2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = NuSVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=4, nu=0.59, coef0=0.053)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_svnu2[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_svnu2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = svm.SVC(probability=True, kernel='poly', degree=4, gamma='auto', random_state=42)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_svc2[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_svc2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = RandomForestClassifier(n_estimators=100,random_state=1)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_rf2[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_rf2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
clf = neural_network.MLPClassifier(random_state=3, activation='relu', solver='lbfgs', tol=1e-06, hidden_layer_sizes=(250,))
clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_mlp2[idx1[test_index3]] = clf.predict_proba(train3[test_index3,:])[:,1]
preds_mlp2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
if k%32==0: print(k)
 auc = roc_auc_score(train['target'],oof_qda2)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))
print('----------------')
 print('knn', roc_auc_score(train['target'], oof_knn2))
print('svc', roc_auc_score(train['target'], oof_svc2))
print('svnu', roc_auc_score(train['target'], oof_svnu2))
print('rf', roc_auc_score(train['target'], oof_rf2))
print('mlp', roc_auc_score(train['target'], oof_mlp2))
class ReadCsvData(object):
def __init__(self):
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 self._train = df_train.drop(['label'], axis=1 ).values
self._labels = df_train['label'].values
self._test = df_test.values
def get_data(self):
self._train = self._train.astype(np.float32)
 self._train = np.multiply(self._train, 1.0 / 255.0)
 self._test = self._test.astype(np.float32)
 self._test = np.multiply(self._test, 1.0 / 255.0)
 self._labels = np.identity(10)[self._labels]
return self._train, self._labels,self._test<define_variables>
class DataSet(object):
def __init__(self,
images,
labels):
assert images.shape[0] == labels.shape[0],('images.shape: %s labels.shape: %s' %(images.shape, labels.shape))
self._num_examples = images.shape[0]
self._images = images
self._labels = labels
self._epochs_completed = 0
self._index_in_epoch = 0
def next_batch(self, batch_size):

 start = self._index_in_epoch
self._index_in_epoch += batch_size
if self._index_in_epoch > self._num_examples:
self._epochs_completed += 1
perm = np.arange(self._num_examples)
 np.random.shuffle(perm)
 self._images = self._images[perm]
self._labels = self._labels[perm]
start = 0
self._index_in_epoch = batch_size
assert batch_size <= self._num_examples
end = self._index_in_epoch
return self._images[start:end], self._labels[start:end]<prepare_x_and_y>
auc = roc_auc_score(train['target'],oof_qda2*0.6+oof_svnu2*0.25 + oof_svc2*0.05 +oof_rf2*0.1)
 print('Pseudo Labeled BLEND scores CV =',round(auc,5))
sess = tf.InteractiveSession()
x = tf.placeholder(tf.float32, shape=[None, 784])
 y_ = tf.placeholder(tf.float32, shape=[None, 10])
 W = tf.Variable(tf.zeros([784,10]))
b = tf.Variable(tf.zeros([10]))
W_conv1 = tf.Variable(tf.truncated_normal([5, 5, 1, 50], stddev=0.1))
b_conv1 = tf.Variable(tf.constant(0.1, shape=[50]))<define_search_space>
auc = roc_auc_score(train['target'],oof_qda2*0.5+oof_svnu2*0.3 + oof_svc2*0.05 + oof_knn2*0.025 + oof_rf2*0.1 + oof_mlp2*0.025)
 print('Pseudo Labeled BLEND2 scores CV =',round(auc,5))
x_image = tf.reshape(x, [-1,28,28,1])
 h_conv1 = tf.nn.relu(tf.nn.conv2d(x_image, W_conv1, strides=[1, 1, 1, 1], padding='SAME')+ b_conv1)
 h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME' )<init_hyperparams>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_qda2
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()
W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 50, 100], stddev=0.1))
b_conv2 = tf.Variable(tf.constant(0.1, shape=[100]))
h_conv2 = tf.nn.relu(tf.nn.conv2d(h_pool1, W_conv2, strides=[1, 1, 1, 1], padding='SAME')+ b_conv2)
 h_pool2 = tf.nn.max_pool(h_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME' )<concatenate>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_qda2*0.6+preds_svnu2*0.25 + preds_svc2*0.05 +preds_rf2*0.1
sub.to_csv('submission_blend.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Blend Test.csv predictions')
 plt.show()
<categorify><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_qda2*0.5+preds_svnu2*0.3 + preds_svc2*0.05 + preds_knn2*0.025 + preds_rf2*0.1 + preds_mlp2*0.025
sub.to_csv('submission_blend2.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Blend2 Test.csv predictions')
 plt.show()