Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<train_on_grid>
%matplotlib inline
warnings.filterwarnings('ignore')
RFC = RandomForestClassifier()
RF_grid = {"max_depth": [None],
"max_features": [4, 6, 8],
"min_samples_split": [3, 5, 7],
"min_samples_leaf": [5, 10, 15],
"bootstrap": [True],
"n_estimators" :[500],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = RF_grid, cv=10, scoring="accuracy", n_jobs= -1, verbose = 1)
 gsRFC.fit(train_X,train_Y)
 RFC_best = gsRFC.best_estimator_
print(gsRFC.best_score_)
 print(gsRFC.best_params_ )<train_on_grid>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
print(train.shape, test.shape )
RF_model = RandomForestClassifier(bootstrap= True,
criterion = 'gini',
max_depth = None,
max_features=8,
min_samples_leaf = 5,
min_samples_split = 7,
n_estimators = 500, random_state=1)
 RF_model.fit(train_X, train_Y)
 CV=cross_val_score(RF_model,train_X,train_Y,cv=KF, scoring="accuracy")
 models.append('RandomForest')
 modelScores.append(round(CV.mean() ,3))
modelSTD.append('NA')
 featureImportance = pd.concat(( pd.DataFrame(train_X.columns, columns = ['Feature']),
pd.DataFrame(RF_model.feature_importances_, columns = ['Importance'])) ,
axis = 1 ).sort_values(by='Importance', ascending = False)[:20]
plt.subplots(figsize=(20,8))
sns.barplot(x=featureImportance['Feature'], y=featureImportance['Importance'], alpha=.5 ).set_title('Feature Importance' )<create_dataframe>
def get_mean_cov(x,y):
ones =(y==1 ).astype(bool)
 x2 = x[ones]
gmm_1 = GaussianMixture(n_components=2)
 gmm_1.fit(x2)
 m1 = gmm_1.means_
p1 = gmm_1.precisions_
zeros =(y==0 ).astype(bool)
 x2b = x[zeros]
gmm_0 = GaussianMixture(n_components=2)
 gmm_0.fit(x2b)
 m2 = gmm_0.means_
p2 = gmm_0.precisions_
ms = np.concatenate(( m1,m2),axis=0)
 ps = np.concatenate(( p1,p2),axis=0)
 return ms,ps
ModelComparison=pd.DataFrame({'CV Score':modelScores, 'Std':modelSTD}, index=models)
 ModelComparison<save_to_csv>
np.random.seed(42)
 oof_gmm = np.zeros(len(train))
preds_gmm = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) , 
('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=2019,shuffle=True)
 for train_index, test_index in skf.split(train2, train2['target']):
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gmm = GaussianMixture(n_components=4, init_params='kmeans', covariance_type='full', max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gmm.fit(np.concatenate([train3,test3],axis = 0))
prob = gmm.predict_proba(train3[test_index,:])
 prob_class = np.zeros(prob.shape[0])
 for j in range(prob.shape[0]):
if(np.argmax(prob,axis=1)[j] in {0, 1}):
prob_class[j] = np.max(prob,axis=1)[j]
else:
prob_class[j] = 1 - np.max(prob,axis=1)[j]
oof_gmm[idx1[test_index]] = prob_class
prob_test = gmm.predict_proba(test3)
 prob_test_class = np.zeros(prob_test.shape[0])
 for j in range(prob_test.shape[0]):
if(np.argmax(prob_test,axis=1)[j] in {0, 1}):
prob_test_class[j] = np.max(prob_test,axis=1)[j]
else:
prob_test_class[j] = 1 - np.max(prob_test,axis=1)[j]
preds_gmm[idx2] += prob_test_class / skf.n_splits
auc = roc_auc_score(train['target'], oof_gmm)
 print(f'AUC: {auc:.5}')
test_ID = pd.read_csv('.. /input/test.csv')
 test_ID = test_ID['PassengerId']
Survival_predictions = RF_model.predict(test)
 ID=np.arange(892,1310,1)
 submission=pd.DataFrame({
"PassengerId": ID,
"Survived": Survival_predictions
})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
for itr in range(3):
test['target'] = preds_gmm
test.loc[test['target'] > 0.96, 'target'] = 1
test.loc[test['target'] < 0.14, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 print(usefull_test.shape[0], "Test Records added for iteration : ", itr)
 new_train.loc[oof_gmm > 0.98, 'target'] = 1
new_train.loc[oof_gmm < 0.02, 'target'] = 0
oof_gmm2 = np.zeros(len(train))
preds_gmm = np.zeros(len(test))
for i in tqdm_notebook(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
train2.reset_index(drop=True, inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 pipe = Pipeline([('vt', VarianceThreshold(threshold=2)) , 
('scaler', StandardScaler())])
 data2 = pipe.fit_transform(data[cols])
 train3 = data2[:train2.shape[0]]
test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=2019,shuffle=True)
 for train_index, test_index in skf.split(train2, train2['target']):
oof_test_index = [t for t in test_index if t < len(idx1)]
ms, ps = get_mean_cov(train3[train_index,:],train2.loc[train_index]['target'].values)
 gmm = GaussianMixture(n_components=4, init_params='kmeans', covariance_type='full', max_iter=100, n_init=1,means_init=ms, precisions_init=ps)
 gmm.fit(np.concatenate([train3,test3],axis = 0))
if len(oof_test_index)> 0:
prob = gmm.predict_proba(train3[oof_test_index,:])
 prob_class = np.zeros(prob.shape[0])
 for j in range(prob.shape[0]):
if(np.argmax(prob,axis=1)[j] in {0, 1}):
prob_class[j] = np.max(prob,axis=1)[j]
else:
prob_class[j] = 1 - np.max(prob,axis=1)[j]
oof_gmm2[idx1[test_index]] = prob_class
prob_test = gmm.predict_proba(test3)
 prob_test_class = np.zeros(prob_test.shape[0])
 for j in range(prob_test.shape[0]):
if(np.argmax(prob_test,axis=1)[j] in {0, 1}):
prob_test_class[j] = np.max(prob_test,axis=1)[j]
else:
prob_test_class[j] = 1 - np.max(prob_test,axis=1)[j]
preds_gmm[idx2] += prob_test_class / skf.n_splits
auc = roc_auc_score(train['target'], oof_gmm)
 print(f'AUC: {auc:.5}')
<set_options><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_gmm
sub.to_csv('submission.csv',index=False )