Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<create_dataframe>
np.random.seed(42)
 x = np.random.choice(np.arange(30),2)
 print('We will submit versions',x[0],'and',x[1])
 np.random.seed(None )
cols = train.columns.values
feature_dataframe = pd.DataFrame(
 {
'Random Forest feature importances': rfc_cv.best_estimator_.feature_importances_,
'Extra Trees feature importances': etc_cv.best_estimator_.feature_importances_,
'AdaBoost feature importances': ada_cv.best_estimator_.feature_importances_,
'Gradient Boost feature importances': gbc_cv.best_estimator_.feature_importances_
}
)<prepare_output>
import numpy as np, pandas as pd, os
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.mixture import GaussianMixture
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import VarianceThreshold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
feature_dataframe['mean'] = feature_dataframe.mean(axis=1)
 feature_dataframe.head(3 )<train_model>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_train.head()
vc = VotingClassifier(estimators=[ 
('rfc', rfc_cv.best_estimator_), 
('etc', etc_cv.best_estimator_), 
('ada', ada_cv.best_estimator_), 
('gbc', gbc_cv.best_estimator_), 
('svm', svc_cv.best_estimator_)
 ], voting='soft', n_jobs=4)
 vc = vc.fit(train_X, train_y )<predict_on_test>
useful = np.zeros(( 256,512))
for i in range(512):
partial = df_train[ df_train['wheezy-copper-turtle-magic']==i ]
useful[:,i] = np.std(partial.iloc[:,1:-1], axis=0)
 useful = useful > 1.5
useful = np.sum(useful, axis=0 )
predictions = vc.predict(test)
 submission = pd.DataFrame({
"PassengerId": PassengerId,
"Survived": predictions
}, dtype='int64')
 submission.head(15 )<save_to_csv>
models = 512
RunLocally = True
if len(df_test)>512*300:
repeat = 1
models = 512 * repeat
RunLocally = False
all_preds = np.zeros(len(df_test))
all_y_pu = np.array([])
 all_y_pr = np.array([])
 all_preds_pu = np.array([])
 all_preds_pr = np.array([])
 for k in range(models):
if RunLocally:
obs = 512
X, y = make_classification(n_samples=1024, n_features=useful[k%512],
n_informative=useful[k%512], n_redundant=0, n_repeated=0,
n_classes=2, n_clusters_per_class=3, weights=None, flip_y=0.05,
class_sep=1.0, hypercube=True, shift=0.0, scale=1.0, shuffle=True,
random_state=None)
 else:
df_train2 = df_train[df_train['wheezy-copper-turtle-magic']==k%512]
df_test2 = df_test[df_test['wheezy-copper-turtle-magic']==k%512]
sel = VarianceThreshold(1.5 ).fit(df_train2.iloc[:,1:-1])
 df_train3 = sel.transform(df_train2.iloc[:,1:-1])
 df_test3 = sel.transform(df_test2.iloc[:,1:])
 obs = df_train3.shape[0]
X = np.concatenate(( df_train3,df_test3),axis=0)
 y = np.concatenate(( df_train2['target'].values,np.zeros(len(df_test2))))
train = X[:obs,:]
train_y = y[:obs]
test = X[obs:,:]
test_y = y[obs:]
comb = X
clf = QuadraticDiscriminantAnalysis(priors = [0.5,0.5])
 clf.fit(train,train_y)
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
clf = QuadraticDiscriminantAnalysis(priors = [0.5, 0.5])
 clf.fit(comb, np.concatenate(( train_y,test_pred)))
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
all_y = np.concatenate(( train_y,test_pred))
least = 0; ct = 1; thx=150
while least<thx:
if ct>=10: thx -= 10
else: thx = 150
clusters = np.zeros(( len(comb),6))
train4 = comb[ all_y==1, :]
clf = GaussianMixture(n_components=3 ).fit(train4)
 clusters[ all_y==1, 3:] = clf.predict_proba(train4)
 train4 = comb[ all_y==0, :]
clf = GaussianMixture(n_components=3 ).fit(train4)
 clusters[ all_y==0, :3] = clf.predict_proba(train4)
 for j in range(5): clusters[:,j+1] += clusters[:,j]
rand = np.random.uniform(0,1,clusters.shape[0])
 for j in range(6): clusters[:,j] = clusters[:,j]>rand
clusters2 = 6 - np.sum(clusters,axis=1)
 least = pd.Series(clusters2 ).value_counts().min() ; ct += 1
clf = QuadraticDiscriminantAnalysis(priors = [0.167, 0.167, 0.167, 0.167, 0.167, 0.167])
 clf.fit(comb,clusters2)
 pds = clf.predict_proba(test)
 test_pred = pds[:,3]+pds[:,4]+pds[:,5]
if RunLocally:
all_y_pu = np.append(all_y_pu, test_y[:256])
 all_y_pr = np.append(all_y_pr, test_y[256:])
 all_preds_pu = np.append(all_preds_pu, test_pred[:256])
 all_preds_pr = np.append(all_preds_pr, test_pred[256:])
 else:
all_preds[df_test2.index] += test_pred / repeat
if(( k+1)%64==0)|(k==0): print('modeled and predicted',k+1,'magic sub datasets')
 if RunLocally:
all_y_pu_pr = np.concatenate(( all_y_pu,all_y_pr))
all_preds_pu_pr = np.concatenate(( all_preds_pu,all_preds_pr))
auc1 = roc_auc_score(all_y_pu_pr, all_preds_pu_pr)
 auc2 = roc_auc_score(all_y_pu, all_preds_pu)
 auc3 = roc_auc_score(all_y_pr, all_preds_pr)
 print()
print('Validation AUC =',np.round(auc1,5))
print('Approx Public LB =',np.round(auc2,5))
print('Approx Private LB =',np.round(auc3,5))
submission.to_csv('titanic.csv', index=False )<import_modules>
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = all_preds
sub.to_csv('submission.csv',index=False)
 plt.hist(test_pred ,bins=100)
 plt.title('Model 512 test predictions')
 plt.show()
import pandas as pd;
import numpy as np;
import seaborn as sns;
import matplotlib.pyplot as plt;<load_from_csv>
for clusters in range(4):
centers = np.array([])
 for k in range(512):
X, y = make_classification(n_samples=768, n_features=useful[k],
n_informative=useful[k], n_redundant=0, n_repeated=0,
n_classes=2, n_clusters_per_class=clusters+1, weights=None,
flip_y=0.05, class_sep=1.0, hypercube=True, shift=0.0, scale=1.0,
shuffle=True, random_state=None)
 centers = np.append(centers,np.mean(X[ np.argwhere(y==0 ).flatten() ,:],axis=0))
centers = np.append(centers,np.mean(X[ np.argwhere(y==1 ).flatten() ,:],axis=0))
plt.hist(centers,bins=100)
 plt.title('Variable means if clusters='+str(clusters+1))
plt.show()
<define_variables><EOS> 
centers = np.array([])
 for k in range(512):
df_train2 = df_train[df_train['wheezy-copper-turtle-magic']==k]
df_test2 = df_test[df_test['wheezy-copper-turtle-magic']==k]
sel = VarianceThreshold(1.5 ).fit(df_train2.iloc[:,1:-1])
 df_train3 = sel.transform(df_train2.iloc[:,1:-1])
 df_test3 = sel.transform(df_test2.iloc[:,1:])
 obs = df_train3.shape[0]
X = np.concatenate(( df_train3,df_test3),axis=0)
 y = np.concatenate(( df_train2['target'].values,np.zeros(len(df_test2))))
train = X[:obs,:]
train_y = y[:obs]
test = X[obs:,:]
test_y = y[obs:]
comb = X
clf = QuadraticDiscriminantAnalysis(priors = [0.5,0.5])
 clf.fit(train,train_y)
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
clf = QuadraticDiscriminantAnalysis(priors = [0.5, 0.5])
 clf.fit(comb, np.concatenate(( train_y,test_pred)))
 test_pred = clf.predict_proba(test)[:,1]
test_pred = test_pred > np.random.uniform(0,1,len(test_pred))
y[obs:] = test_pred
centers = np.append(centers,np.mean(X[ np.argwhere(y==0 ).flatten() ,:],axis=0))
centers = np.append(centers,np.mean(X[ np.argwhere(y==1 ).flatten() ,:],axis=0))
plt.hist(centers,bins=100)
 plt.title('Real Data Variable Means(match clusters=3)')
 plt.show()