Kaggle competition: Instant Gratification
big['Last_Name'] = big['Name'].apply(lambda x: str.split(x, ",")[0])
 big['Fare'].fillna(big['Fare'].mean() , inplace=True)
 DEFAULT_SURVIVAL_VALUE = 0.5
big['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in big[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
big.loc[big['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
big.loc[big['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:", big.loc[big['Family_Survival']!=0.5].shape[0] )<groupby>
warnings.filterwarnings('ignore')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
for _, grp_df in big.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
big.loc[big['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
big.loc[big['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "+str(big[big['Family_Survival']!=0.5].shape[0]))<drop_column>
RANDOM_SEED = 4123
cols = [
c for c in train.columns
if c not in ['id', 'target', 'wheezy-copper-turtle-magic']
]
def get_mean_cov(x,y):
model = GraphicalLasso()
ones =(y==1 ).astype(bool)
 x2 = x[ones]
model.fit(x2)
 p1 = model.precision_
m1 = model.location_
onesb =(y==0 ).astype(bool)
 x2b = x[onesb]
model.fit(x2b)
 p2 = model.precision_
m2 = model.location_
ms = np.stack([m1,m2])
 ps = np.stack([p1,p2])
 return ms,ps
del big['Ticket'], big['Cabin'], big['Name'], big['XName'], big['Last_Name']<count_values>
%%time
SKIP_COMMIT = True
if SKIP_COMMIT:
sub = pd.read_csv('.. /input/sample_submission.csv')
 if sub.shape[0] < 200000:
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.to_csv('submission.csv', index=False)
 raise ValueError('Stop!!!')
 oof_nusvc = np.zeros(len(train))
preds_nusvc = np.zeros(len(test))
oof_nb= np.zeros(len(train))
preds_nb = np.zeros(len(test))
oof_lr = np.zeros(len(train))
preds_lr = np.zeros(len(test))
oof_qda = np.zeros(len(train))
preds_qda = np.zeros(len(test))
oof_lp = np.zeros(len(train))
preds_lp = np.zeros(len(test))
oof_lgbm = np.zeros(len(train))
preds_lgbm = np.zeros(len(test))
oof_gm = np.zeros(len(train))
preds_gm = np.zeros(len(test))
oof_rf = np.zeros(len(train))
preds_rf = np.zeros(len(test))
params_lgbm_1 = {
'boosting_type': 'gbdt',
'objective': 'xentropy',
'metric': ['auc'],
'num_leaves': 31,
'learning_rate': 0.5,
'feature_fraction': 1.0,
'bagging_fraction': 1.0,
'bagging_freq': 18,
'num_threads': 8,
'lambda_l2': 5.0,
'max_bin': 3
}
for i in range(512):
print(i, end=' ')
 train2 = train[train['wheezy-copper-turtle-magic']==i]
idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx2 = test2.index
data = pd.concat(
 [
train2,
test2
],
axis=0 
)
 train2.reset_index(drop=True, inplace=True)
 train_size = train2.shape[0]
sel = VarianceThreshold(threshold=1.5)
 tmp = sel.fit_transform(
 data[cols] 
)
 train3 = tmp[:train_size, :]
test3 = tmp[train_size:, :]
ss = StandardScaler()
tmp_scaled = ss.fit_transform(tmp)
 train3_scaled = tmp_scaled[:train_size, :]
test3_scaled = tmp_scaled[train_size:, :]
poly = PolynomialFeatures(degree=2)
 tmp_poly = poly.fit_transform(tmp_scaled)
 train3_poly = tmp_poly[:train_size, :]
test3_poly = tmp_poly[train_size:, :]
gm_clf_4 = mixture.GaussianMixture(
 n_components=4,
random_state=RANDOM_SEED 
)
 gm_tmp_4 = gm_clf_4.fit_predict(tmp ).reshape(-1, 1)
 le_4 = OneHotEncoder()
gm_tmp_4 = le_4.fit_transform(gm_tmp_4 ).todense()
gm_train3_4 = gm_tmp_4[:train_size, :]
gm_test3_4 = gm_tmp_4[train_size:, :]
gm_clf_6 = mixture.GaussianMixture(
 n_components=6,
random_state=RANDOM_SEED 
)
 gm_tmp_6 = gm_clf_6.fit_predict(tmp ).reshape(-1, 1)
 le_6 = OneHotEncoder()
gm_tmp_6 = le_6.fit_transform(gm_tmp_6 ).todense()
gm_train3_6 = gm_tmp_6[:train_size, :]
gm_test3_6 = gm_tmp_6[train_size:, :]
skf = StratifiedKFold(
 n_splits=11,
random_state=RANDOM_SEED,
shuffle=True 
)
 for train_index, test_index in skf.split(train3, train2['target']):
train_train_index, train_val_index = train_test_split(
 train_index,
test_size=0.3,
random_state=RANDOM_SEED 
)
 train_dataset = lgb.Dataset(
 np.hstack(
(
 train3_scaled[train_train_index,:],
gm_tmp_4[:train_size, :][train_train_index, :].tolist() ,
gm_tmp_6[:train_size, :][train_train_index, :].tolist() ,
train3_poly[train_train_index,:] 
)
),
train2.loc[train_train_index]['target'],
free_raw_data=False 
)
 valid_dataset = lgb.Dataset(
 np.hstack(
(
 train3_scaled[train_val_index,:],
gm_tmp_4[:train_size, :][train_val_index, :].tolist() ,
gm_tmp_6[:train_size, :][train_val_index, :].tolist() ,
train3_poly[train_val_index,:] 
)
),
train2.loc[train_val_index]['target'],
free_raw_data=False 
)
 gm = lgb.train(
 params_lgbm_1,
train_dataset,
num_boost_round=1000,
early_stopping_rounds=20,
valid_sets=(train_dataset, valid_dataset),
valid_names=('train', 'valid'),
feature_name=[str(l)for l in range(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_tmp_4[:train_size, :][test_index, :].tolist() ,
gm_tmp_6[:train_size, :][test_index, :].tolist() ,
train3_poly[test_index,:] 
)
 ).shape[1] 
)],
categorical_feature=[str(l)for l in
range(
 train3_scaled.shape[1],
train3_scaled.shape[1] + gm_tmp_4.shape[1] + gm_tmp_6.shape[1] 
)
 ],
verbose_eval=0 
)
 oof_lgbm[idx1[test_index]] = gm.predict(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_tmp_4[:train_size, :][test_index, :].tolist() ,
gm_tmp_6[:train_size, :][test_index, :].tolist() ,
train3_poly[test_index,:] 
)
)
)
 preds_lgbm[idx2] += gm.predict(
 np.hstack(
(
 test3_scaled,
gm_test3_4.tolist() ,
gm_test3_6.tolist() ,
test3_poly 
)
)
)/ skf.n_splits
ms, ps = get_mean_cov(
 train3[train_index, :],
train2.loc[train_index]['target'].values 
)
 gm = mixture.GaussianMixture(
 n_components=2,
init_params='random',
covariance_type='full',
tol=0.001,
reg_covar=0.001,
max_iter=100,
n_init=1,
means_init=ms,
precisions_init=ps,
random_state=RANDOM_SEED 
)
 gm.fit(tmp)
 oof_gm[idx1[test_index]] = gm.predict_proba(
 train3[test_index,:] 
)[:, 0]
preds_gm[idx2] += gm.predict_proba(
 test3 
)[:, 0] / skf.n_splits
lp = LabelPropagation(
 kernel='rbf',
gamma=0.15301581563198507,
n_jobs=-1 
)
 lp.fit(
 train3_scaled[train_index,:],
train2.loc[train_index]['target'] 
)
 oof_lp[idx1[test_index]] = lp.predict_proba(
 train3_scaled[test_index, :] 
)[:,1]
preds_lp[idx2] += lp.predict_proba(
 test3_scaled 
)[:,1] / skf.n_splits
clf = NuSVC(
 probability=True,
kernel='poly',
degree=2,
gamma='auto',
random_state=RANDOM_SEED,
nu=0.27312143533915767,
coef0=0.4690615598786931 
)
 clf.fit(
 np.hstack(
(
 train3_scaled[train_index,:],
gm_train3_4[train_index, :],
gm_train3_6[train_index, :] 
)
),
train2.loc[train_index]['target'] 
)
 oof_nusvc[idx1[test_index]] = clf.predict_proba(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_train3_4[test_index, :],
gm_train3_6[test_index, :] 
)
)
)[:,1]
preds_nusvc[idx2] += clf.predict_proba(
 np.hstack(
(
 test3_scaled,
gm_test3_4,
gm_test3_6 
)
)
)[:,1] / skf.n_splits
clf = RandomForestClassifier(
 max_depth=4,
n_jobs=-1,
n_estimators=20 
)
 clf.fit(
 np.hstack(
(
 train3_scaled[train_index,:],
gm_train3_4[train_index, :],
gm_train3_6[train_index, :] 
)
),
train2.loc[train_index]['target'] 
)
 oof_rf[idx1[test_index]] = clf.predict_proba(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_train3_4[test_index, :],
gm_train3_6[test_index, :] 
)
)
)[:,1]
preds_rf[idx2] += clf.predict_proba(
 np.hstack(
(
 test3_scaled,
gm_test3_4,
gm_test3_6 
)
)
)[:,1] / skf.n_splits
clf = QuadraticDiscriminantAnalysis(
 reg_param=0.5674164995882528 
)
 clf.fit(
 train3[train_index,:],
train2.loc[train_index]['target'] 
)
 oof_qda[idx1[test_index]] += clf.predict_proba(
 train3[test_index, :] 
)[:,1]
preds_qda[idx2] += clf.predict_proba(
 test3 
)[:,1] / skf.n_splits
clf = linear_model.LogisticRegression(
 solver='saga',
penalty='l2',
C=0.01,
tol=0.001,
random_state=RANDOM_SEED 
)
 clf.fit(
 train3_poly[train_index,:],
train2.loc[train_index]['target'] 
)
 oof_lr[idx1[test_index]] = clf.predict_proba(
 train3_poly[test_index,:] 
)[:,1]
preds_lr[idx2] += clf.predict_proba(
 test3_poly 
)[:,1] / skf.n_splits
clf = GaussianNB()
clf.fit(
 np.hstack(
(
 train3_scaled[train_index,:],
gm_train3_6[train_index, :],
gm_train3_4[train_index, :] 
)
),
train2.loc[train_index]['target'] 
)
 oof_nb[idx1[test_index]] = clf.predict_proba(
 np.hstack(
(
 train3_scaled[test_index,:],
gm_train3_6[test_index, :],
gm_train3_4[test_index, :] 
)
)
)[:,1]
preds_nb[idx2] += clf.predict_proba(
 np.hstack(
(
 test3_scaled,
gm_test3_6,
gm_test3_4 
)
)
)[:,1] / skf.n_splits
print('
svcnu', roc_auc_score(train['target'], oof_nusvc))
print('gm', roc_auc_score(train['target'], oof_gm))
print('qda', roc_auc_score(train['target'], oof_qda))
print('log reg poly', roc_auc_score(train['target'], oof_lr))
print('gnb', roc_auc_score(train['target'], oof_nb))
print('lp', roc_auc_score(train['target'], oof_lp))
print('lgbm', roc_auc_score(train['target'], oof_lgbm))
print('rf', roc_auc_score(train['target'], oof_rf))
oof_qda = oof_qda.reshape(-1, 1)
 preds_qda = preds_qda.reshape(-1, 1)
 oof_lr = oof_lr.reshape(-1, 1)
 preds_lr = preds_lr.reshape(-1, 1)
 oof_nusvc = oof_nusvc.reshape(-1, 1)
 preds_nusvc = preds_nusvc.reshape(-1, 1)
 oof_nb = oof_nb.reshape(-1, 1)
 preds_nb = preds_nb.reshape(-1, 1)
 oof_lp = oof_lp.reshape(-1, 1)
 preds_lp = preds_lp.reshape(-1, 1)
 oof_gm = oof_gm.reshape(-1, 1)
 preds_gm = preds_gm.reshape(-1, 1)
 oof_lgbm = oof_lgbm.reshape(-1, 1)
 preds_lgbm = preds_lgbm.reshape(-1, 1)
 oof_rf = oof_rf.reshape(-1, 1)
 preds_rf = preds_rf.reshape(-1, 1)
 tr_2 = np.concatenate(
(
 oof_qda,
oof_nusvc,
oof_lr,
oof_nb,
oof_lp,
oof_gm,
oof_lgbm,
oof_rf 
),
axis=1 
)
 te_2 = np.concatenate(
(
 preds_qda,
preds_nusvc,
preds_lr,
preds_nb,
preds_lp,
preds_gm,
preds_lgbm,
preds_rf 
),
axis=1 
)
 print(np.corrcoef(tr_2, rowvar=False))
params = {
'boosting_type': 'gbdt',
'objective': 'xentropy',
'metric': ['auc'],
'num_leaves': 3,
'learning_rate': 0.1,
'feature_fraction': 0.4,
'bagging_fraction': 0.4,
'bagging_freq': 5,
'num_threads': 8
}
params.update(
 {
'bagging_fraction': 0.9687497922020039,
'bagging_freq': 100,
'feature_fraction': 0.7578027095458152,
'lambda_l2': 4.871836452096843,
'learning_rate': 0.41230192513715164,
'max_bin': 20,
'num_leaves': 8
} 
)
 oof_boosting_2_bad_cv = np.zeros(train.shape[0])
 pred_te_boosting_2_bad_cv = np.zeros(test.shape[0])
 train2 = train.copy()
train2.reset_index(drop=True,inplace=True)
 skf = StratifiedKFold(
 n_splits=11,
random_state=RANDOM_SEED,
shuffle=True 
)
 for train_index, test_index in skf.split(tr_2, train2['target']):
train_dataset = lgb.Dataset(
 np.hstack(
(
 tr_2[train_index, :],
tr_2[train_index, :] ** 2.587508645172711 
)
),
train2['target'][train_index],
free_raw_data=False 
)
 valid_dataset = lgb.Dataset(
 np.hstack(
(
 tr_2[test_index, :] ,
tr_2[test_index, :] ** 2.587508645172711 
)
),
train2['target'][test_index],
free_raw_data=False 
)
 gbm = lgb.train(
 params,
train_dataset,
num_boost_round=1000,
early_stopping_rounds=100,
valid_sets=(train_dataset, valid_dataset),
valid_names=('train', 'valid'),
verbose_eval=100 
)
 oof_boosting_2_bad_cv[test_index] = gbm.predict(
 np.hstack(
(
 tr_2[test_index, :],
tr_2[test_index, :] ** 2.587508645172711 
)
)
)
 pred_te_boosting_2_bad_cv += gbm.predict(
 np.hstack(
(
 te_2,
te_2 ** 2.587508645172711 
)
)
)/ skf.n_splits
print('gnb', roc_auc_score(train['target'], oof_boosting_2_bad_cv))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = pred_te_boosting_2_bad_cv
sub.to_csv('submission.csv', index=False )