Kaggle competition: Instant Gratification
gnb = GaussianNB()<import_modules>
%matplotlib inline
from keras.models import Sequential<import_modules>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.head()
from keras.layers import Dense,BatchNormalization,Dropout<import_modules>
def data_augmentation(X, y):
mean_train2_0 = X[y==0].mean()
mean_train2_1 = X[y==1].mean()
train2_0 = 2*mean_train2_0.reshape(1,-1)- X[y==0]
train2_1 = 2*mean_train2_1.reshape(1,-1)- X[y==1]
tmp_train2_0 = np.vstack([X[y==0], train2_0])
 tmp_train2_1 = np.vstack([X[y==1], train2_1])
 train2 = np.vstack([tmp_train2_0, tmp_train2_1])
 y2 = np.array([0]*len(tmp_train2_0)+ [1]*len(tmp_train2_1))
return np.vstack([X, train2]), np.concatenate([y, y2])
from keras import callbacks<choose_model_class>
def get_kmeans_clusters(x, y, k_pos=1, k_neg=1):
x_zeros = x[y==0]
x_ones = x[y==1]
model_0 = KMeans(n_clusters=k_neg)
 model_1 = KMeans(n_clusters=k_pos)
 model_0.fit(x_zeros)
 model_1.fit(x_ones)
 model_0_clus = [x_zeros[model_0.labels_==k] for k in range(model_0.n_clusters)]
model_1_clus = [x_ones[model_1.labels_==k] for k in range(model_1.n_clusters)]
return model_1_clus + model_0_clus
def fit_multicluster_gmm(x, y, xt, k_pos, k_neg, max_iter=100):

 clusters = get_kmeans_clusters(x, y, k_pos=k_pos, k_neg=k_neg)
 for i in range(len(clusters)) :
x_cluster = clusters[i]
model = ShrunkCovariance()
model.fit(x_cluster)
 if(i==0):
ps = np.expand_dims(model.precision_, axis=0)
 ms = np.expand_dims(model.location_, axis=0)
 else:
ps = np.concatenate([ps, np.expand_dims(model.precision_, axis=0)], axis=0)
 ms = np.concatenate([ms, np.expand_dims(model.location_, axis=0)], axis=0)
 gm = mixture.GaussianMixture(n_components=k_pos+k_neg,
init_params='random',
covariance_type='full',
tol=0.001,
reg_covar=0.001,
max_iter=100,
n_init=5,
means_init=ms,
precisions_init=ps)
 gm.fit(np.vstack(( x.astype(np.float), xt.astype(np.float))))
preds = gm.predict_proba(x.astype(np.float)) [:,0]
score = roc_auc_score(y, preds)
 return score, gm, k_pos, k_neg
def get_mean_cov(x,y, model=GraphicalLasso() , max_iter=100):
try:
model.set_params(**{'max_iter':200})
 except:
pass
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps
model = Sequential()<choose_model_class>
def extract_wheezy_copper_turtle_magic(train, i):
train2 = train[train['wheezy-copper-turtle-magic']==i].copy()
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
target = train2['target'].astype(np.int ).values
train2.reset_index(drop=True, inplace=True)
 return train2.drop(['id', 'target'], axis=1), test2.drop(['id'], axis=1), idx1, idx2, target
model.add(Dense(32,activation = "relu",input_shape =(66,)))
 model.add(Dense(64,activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.2))
model.add(Dense(1,activation = 'sigmoid'))<train_model>
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
test_preds = np.zeros(len(test))
oof_gmm = np.zeros(len(train))
test_preds_gmm = np.zeros(len(test))
oof_gmm_2 = np.zeros(len(train))
test_preds_gmm_2 = np.zeros(len(test))
trials = 3
cat_dict = dict()
cluster_report = list()
for i in tqdm(range(512)) :
train2, test2, idx1, idx2, target = extract_wheezy_copper_turtle_magic(train, i)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=1.5)) ])
 train3 = pipe.fit_transform(train2[cols])
 test3 = pipe.fit_transform(test2[cols])
 data2 = StandardScaler().fit_transform(PCA(n_components=40, random_state=4 ).fit_transform(data[cols]))
train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
cat_dict[i] = train3.shape[1]
try:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=3)
 except:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=1)
 oof_gmm[idx1] = gm.predict_proba(train3)[:,0]
test_preds_gmm[idx2] += gm.predict_proba(test3)[:,0]
try:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=3, k_neg=3)
 except:
score, gm, k_pos, k_neg = fit_multicluster_gmm(x=train3, y=target, xt=test3, k_pos=1, k_neg=1)
 clusters = gm.predict_proba(train3 ).shape[1]
oof_gmm_2[idx1] = np.sum(gm.predict_proba(train3)[:,:clusters//2], axis=1)
 test_preds_gmm_2[idx2] += np.sum(gm.predict_proba(test3)[:,:clusters//2], axis=1 )
model.compile(optimizer = "adam",loss ="binary_crossentropy",metrics = ['accuracy'])
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='acc', factor=0.2,patience=3, min_lr=0.0001)
 model.fit(X_train,Y_train,epochs = 30,callbacks = [reduce_lr] )<predict_on_test>
oof_auc_gmm = roc_auc_score(train['target'], oof_gmm)
 print('OOF AUC: =',round(oof_auc_gmm, 5))
oof_auc_gmm_2 = roc_auc_score(train['target'], oof_gmm_2)
 print('OOF AUC: =',round(oof_auc_gmm_2, 5))
oof_auc_blend = roc_auc_score(train['target'],(0.3*oof_gmm+0.7*oof_gmm_2))
print('OOF AUC: =',round(oof_auc_blend, 5))
<categorify><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = test_preds_gmm
sub.to_csv('submission_gmm.csv',index=False)
 sub['target'] = test_preds_gmm_2
sub.to_csv('submission_gmm_2.csv',index=False)
 sub['target'] =(test_preds_gmm + test_preds_gmm_2)/2
sub.to_csv('submission_blend.csv',index=False )