Kaggle competition: Instant Gratification
droplist.append('Name' )<count_values>
from sklearn.feature_selection import VarianceThreshold
print(train.Ticket.str.isnumeric().value_counts())
 train.Ticket[train.Ticket.str.isnumeric().apply(lambda x:not x)]<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
train.Ticket=train.Ticket.apply(lambda x:'num' if x.isnumeric() else x)
 train.Ticket=train.Ticket.str.split('/' ).apply(lambda x:x[0] ).str.split(' ' ).apply(lambda x:x[0] ).str.split('.' ).apply(lambda x:x[0] )<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
test.Ticket=test.Ticket.apply(lambda x:'num' if x.isnumeric() else x)
 test.Ticket=test.Ticket.str.split('/' ).apply(lambda x:x[0] ).str.split(' ' ).apply(lambda x:x[0] ).str.split('.' ).apply(lambda x:x[0] )<count_values>
cols = [c for c in train.columns if c not in ['id', 'target', 'wheezy-copper-turtle-magic']]
train.Ticket.value_counts()<categorify>
def instant_model(train, test, cols = cols, clf = KNeighborsClassifier(leaf_size=2000, n_neighbors = 11, p = 2, weights = 'distance'), selection = "PCA"):
oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in tqdm(range(512)) :
train2 = train[train['wheezy-copper-turtle-magic'] == i]
test2 = test[test['wheezy-copper-turtle-magic'] == i]
idx1 = train2.index
idx2 = test2.index
data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 if selection == "variance":
data2 = StandardScaler().fit_transform(VarianceThreshold(threshold=2 ).fit_transform(data[cols]))
train3 = pd.DataFrame(data2[:train2.shape[0]], index = idx1)
 test3 = pd.DataFrame(data2[train2.shape[0]:], index = idx2)
 elif selection == "PCA":
pca = PCA(n_components = 40, random_state= 1234)
 pca.fit(data[:train2.shape[0]])
 train3 = pd.DataFrame(pca.transform(data[:train2.shape[0]]), index = idx1)
 test3 = pd.DataFrame(pca.transform(data[train2.shape[0]:]), index = idx2)
 train3['target'] = train2['target']
skf = StratifiedKFold(n_splits=15, random_state=42)
 for train_index, test_index in skf.split(train3, train3['target']):
clf = clf
X_train = train3.iloc[train_index, :].drop(["target"], axis = 1)
 X_test = train3.iloc[test_index, :].drop(["target"], axis = 1)
 y_train = train3.iloc[train_index, :]['target']
y_test = train3.iloc[test_index, :]['target']
clf.fit(X_train, y_train)
 train_prob = clf.predict_proba(X_train)[:,1]
test_prob = clf.predict_proba(X_test)[:,1]
oof[idx1[test_index]] = test_prob
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'], oof)
 print(f'AUC: {auc:.5}')
 return oof, preds, auc
train.Ticket.replace(dict(zip(['WE','P','SW','Fa','F4','SO','SCO'],['Rare']*7)) ,inplace=True)
 test.Ticket.replace(dict(zip(['WE','P','SW','Fa','F4','SO','SCO','LP','AQ'],['Rare']*9)) ,inplace=True )<feature_engineering>
def get_newtrain(train, test, preds, oof):
test['target'] = preds
test.loc[test['target'] > 0.985, 'target'] = 1
test.loc[test['target'] < 0.015, 'target'] = 0
usefull_test = test[(test['target'] == 1)|(test['target'] == 0)]
print("No.of test records added: {}".format(usefull_test.shape[0]))
new_train = pd.concat([train, usefull_test] ).reset_index(drop=True)
 new_train.loc[oof > 0.985, 'target'] = 1
new_train.loc[oof < 0.015, 'target'] = 0
return new_train
train['family_size']=train.SibSp+train.Parch+1
train['issingle']=train.family_size.apply(lambda x:1 if x==1 else 0)
 train['smallfamily']=train.family_size.apply(lambda x:1 if x<=3 and x>1 else 0)
 train['middfamily']=train.family_size.apply(lambda x:1 if x<=8 and x>3 else 0)
 train['largefamily']=train.family_size.apply(lambda x:1 if x>8 else 0 )<feature_engineering>
newtrain = get_newtrain(train, test, preds, oof)
 oof_qd, preds_qd, auc = instant_model(newtrain, test, clf = QuadraticDiscriminantAnalysis(0.5),selection="variance" )
test['family_size']=test.SibSp+test.Parch+1
test['issingle']=test.family_size.apply(lambda x:1 if x==1 else 0)
 test['smallfamily']=test.family_size.apply(lambda x:1 if x<=3 and x>1 else 0)
 test['middfamily']=test.family_size.apply(lambda x:1 if x<=8 and x>3 else 0)
 test['largefamily']=test.family_size.apply(lambda x:1 if x>8 else 0 )<count_missing_values>
for i in range(1):
newtrain = get_newtrain(newtrain, test, preds_qd, oof_qd)
 oof_qd, preds_qd, auc = instant_model(newtrain, test, clf = QuadraticDiscriminantAnalysis(0.5),selection="variance" )
<count_missing_values><EOS> 
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = preds_qd
sub.to_csv('submission.csv',index=False)