Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<feature_engineering>
warnings.filterwarnings('ignore')
dataset['LastName'] = dataset['Name'].apply(lambda x: str.split(x, ",")[0])
 DEFAULT_SURVIVAL_VALUE = 0.5
dataset['FamilySurvival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in dataset.groupby(['LastName', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
dataset.loc[dataset['PassengerId'] == passID, 'FamilySurvival'] = 1
elif(smin==0.0):
dataset.loc[dataset['PassengerId'] == passID, 'FamilySurvival'] = 0
for _, grp_df in dataset.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['FamilySurvival'] == 0)|(row['FamilySurvival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
dataset.loc[dataset['PassengerId'] == passID, 'FamilySurvival'] = 1
elif(smin==0.0):
dataset.loc[dataset['PassengerId'] == passID, 'FamilySurvival'] = 0<normalization>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train['wheezy-copper-turtle-magic'] = train['wheezy-copper-turtle-magic'].astype('category')
 test['wheezy-copper-turtle-magic'] = test['wheezy-copper-turtle-magic'].astype('category' )
dataset.drop(['Name', 'LastName', 'PassengerId', 'SibSp', 'Parch', 'Ticket', 'Cabin'], axis=1, inplace=True)
 ss = StandardScaler()
ss.fit_transform(dataset)
 train = dataset[:891]
test = dataset[891:]<groupby>
magicNum = 131073
default_cols = [c for c in train.columns if c not in ['id', 'target','target_pred', 'wheezy-copper-turtle-magic']]
cols = [c for c in default_cols]
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub.to_csv('submission.csv',index=False)
 train.shape,test.shape
print(train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean())
 print(train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean())
 print(train[['Alone', 'Survived']].groupby(['Alone'], as_index=False ).mean())
 print(train[['HasCabin', 'Survived']].groupby(['HasCabin'], as_index=False ).mean())
 print(train[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean())
 print(train[['Fare', 'Survived']].groupby(['Fare'], as_index=False ).mean())
 print(train[['Age', 'Survived']].groupby(['Age'], as_index=False ).mean())
 print(train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean() )<prepare_x_and_y>
y_perfect = [0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1]
y_flliped = [1,0,0,1,0,0,0,0,0,0,1,1,1,0,1,1,1,1,0,1]
roc_auc_score(y_perfect,y_flliped )
train_y = train['Survived']
train.drop(['Survived'], axis=1, inplace=True)
 test.drop(['Survived'], axis=1, inplace=True)
 train_X = train<define_search_space>
y_preds = [0.33,0.33,0.33,0.5,0.5,0,0,0,0,0,1,1,0.5,0.5,1,1,1,0.66,0.66,0.66]
roc_auc_score(y_flliped,y_preds )
<train_on_grid><EOS> 
if sub.shape[0] == magicNum:
[].shape
preds=np.zeros(len(test))
train_err=np.zeros(512)
 test_err=np.zeros(512)
 for i in range(512):
X = train[train['wheezy-copper-turtle-magic']==i].copy()
Y = X.pop('target' ).values
X_test = test[test['wheezy-copper-turtle-magic']==i].copy()
idx_train = X.index
idx_test = X_test.index
X.reset_index(drop=True,inplace=True)
 X = X[cols].values
X_test = X_test[cols].values
vt = VarianceThreshold(threshold=2 ).fit(X)
 X = vt.transform(X)
 X_test = vt.transform(X_test)
 X_all = np.concatenate([X,X_test])
 train_size = len(X)
 test1_size = test[:131073][test[:131073]['wheezy-copper-turtle-magic']==i].shape[0]
compo_cnt = 6
for ii in range(30):
gmm = GaussianMixture(n_components=compo_cnt,init_params='random',covariance_type='full',max_iter=100,tol=1e-10,reg_covar=0.0001 ).fit(X_all)
 labels = gmm.predict(X_all)
 cntStd = np.std([len(labels[labels==j])for j in range(compo_cnt)])
 if round(cntStd,4)== 0.4714:
check_labels = labels[:train_size]
cvt_labels=np.zeros(len(labels))
for iii in range(compo_cnt):
mean_val = Y[check_labels==iii].mean()
mean_val = 1 if mean_val > 0.5 else 0
cvt_labels[labels==iii] = mean_val
train_err[i] = len(Y[Y != cvt_labels[:train_size]])
 if(train_err[i] >= 10)and(train_err[i] <= 15):
train_err[i] = 12.5
exp_err = max(0,(25 - train_err[i])/(train_size + test1_size))
for iii in range(compo_cnt):
mean_val = Y[check_labels==iii].mean()
mean_val =(1-exp_err)if mean_val > 0.5 else exp_err
cvt_labels[labels==iii] = mean_val
preds[idx_test] = cvt_labels[train_size:]
break
sub['target'] = preds
sub.to_csv('submission.csv',index=False )