Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<choose_model_class>
warnings.filterwarnings('ignore')
def build_model(embed):
model = Sequential([
Input(shape=[], dtype=tf.string),
embed,
Dense(1024, activation='elu'),
BatchNormalization() ,
Dropout(0.5),
Dense(512, activation='elu'),
BatchNormalization() ,
Dropout(0.35),
Dense(256, activation='relu'),
BatchNormalization() ,
Dropout(0.1),
Dense(1, activation='sigmoid')
 ])
 model.compile(Adam(lr=0.001), loss='binary_crossentropy', metrics=['accuracy'])
 return model<choose_model_class>
%%time
path=Path('.. /input')
 def load_data(data):
return pd.read_csv(data)
 with multiprocessing.Pool() as pool:
train, test, sub = pool.map(load_data, [path/'train.csv',
path/'test.csv',
path/'sample_submission.csv'] )
earlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='min')
 mcp_save = ModelCheckpoint('model.hdf5', save_best_only=True, monitor='val_loss', mode='min')
 reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=2, epsilon=1e-4, mode='min' )<train_model>
NFOLDS=5
NTRIALS=100
RS=42
debug=0
lowest=0.01
highest=0.99
with tf.compat.v1.Session() as session:
tf.compat.v1.keras.backend.set_session(session)
 session.run([tf.compat.v1.global_variables_initializer() , tf.compat.v1.tables_initializer() ])
 history = model.fit(
 X_train, y_train,
validation_data=(X_test,y_test),
epochs=35,
callbacks=[earlyStopping,reduce_lr_loss,mcp_save],
batch_size=128
)<predict_on_test>
if debug:
magic_max=2
magic_min=0
NFOLDS=2
NTRIALS=2
else:
magic_max=train['wheezy-copper-turtle-magic'].max()
magic_min=train['wheezy-copper-turtle-magic'].min()
with tf.Session() as session:
tf.compat.v1.keras.backend.set_session(session)
 session.run(tf.global_variables_initializer())
 session.run(tf.tables_initializer())
 model.load_weights('model.hdf5')
 y_pred = model.predict(X_test)
 print(classification_report(y_test, y_pred.round().astype(int)) )<save_to_csv>
def preprocess(clfs=['QDA'], train=train, test=test, magic_min=magic_min, magic_max=magic_max):
prepr = {}
for i in range(magic_min, magic_max+1):
X = train[train['wheezy-copper-turtle-magic']==i].copy()
Y = X.pop('target' ).values
X_test = test[test['wheezy-copper-turtle-magic']==i].copy()
idx_train = X.index
idx_test = X_test.index
X.reset_index(drop=True,inplace=True)
 cols = [c for c in X.columns if c not in ['id', 'wheezy-copper-turtle-magic']]
l=len(X)
 X_all = pd.concat([X[cols], X_test[cols]], ignore_index=True)
 X_vt = VarianceThreshold(threshold=1.5 ).fit_transform(X_all)
 prepr['vt_' + str(i)] = X_vt
prepr['train_size_' + str(i)] = l
prepr['idx_train_' + str(i)] = idx_train
prepr['idx_test_' + str(i)] = idx_test
prepr['target_' + str(i)] = Y
return prepr
with tf.Session() as session:
tf.compat.v1.keras.backend.set_session(session)
 session.run(tf.global_variables_initializer())
 session.run(tf.tables_initializer())
 model.load_weights('model.hdf5')
 sub = model.predict(test_data)
 subm = pd.DataFrame()
subm['id'] = test['id']
subm['target'] = sub.round().astype(int)
 subm.to_csv("pred.csv", index = False )<install_modules>
%%time
data = preprocess()
!pip install fastai --upgrade<import_modules>
def get_data(i, data):
l = data['train_size_' + str(i)]
X_all = data['vt_' + str(i)]
X = X_all[:l, :]
X_test = X_all[l:, :]
Y = data['target_' + str(i)]
idx_train = data['idx_train_' + str(i)]
idx_test = data['idx_test_' + str(i)]
return X, X_test, Y, idx_train, idx_test
from fastai import *
from fastai.tabular import *<load_from_csv>
def pseudolabeling(X_train, X_test, Y_train, Y_pseudo,
idx_test, lowest=lowest, highest=highest, test=test):
assert len(test)== len(Y_pseudo), "The length of test does not match that of Y_pseudo!"
Y_aug = Y_pseudo[idx_test]
assert len(Y_aug)== len(X_test), "The length of Y_aug does not match that of X_test!"
Y_aug[Y_aug > highest] = 1
Y_aug[Y_aug < lowest] = 0
mask =(Y_aug == 1)|(Y_aug == 0)
 Y_useful = Y_aug[mask]
X_test_useful = X_test[mask]
X_train_aug = np.vstack(( X_train, X_test_useful))
Y_train_aug = np.vstack(( Y_train.reshape(-1, 1), Y_useful.reshape(-1, 1)))
 return X_train_aug, Y_train_aug
input_path = '/kaggle/input/'
train_df = pd.read_csv(f'{input_path}train.csv')
 test_df = pd.read_csv(f'{input_path}test.csv' )<feature_engineering>
def train_classifier(clf_name, clfs, data=data, train=train, test=test,
debug=debug, NFOLDS=NFOLDS, RS=RS, Y_pseudo=None,
magic_min=magic_min, magic_max=magic_max,
lowest=lowest, highest=highest, verbose=1):
auc_all = np.array([])
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in range(magic_min, magic_max+1):
X, X_test, Y, idx_train, idx_test = get_data(i=i, data=data)
 folds = StratifiedKFold(n_splits=NFOLDS, random_state=RS)
 auc_folds = np.array([])
 for train_index, val_index in folds.split(X, Y):
X_train, Y_train = X[train_index, :], Y[train_index]
X_val, Y_val = X[val_index, :], Y[val_index]
if Y_pseudo is not None:
X_train_aug, Y_train_aug = pseudolabeling(X_train, X_test,
Y_train, Y_pseudo, idx_test,
lowest=lowest, highest=highest,
test=test)
 clfs[clf_name].fit(X_train_aug, Y_train_aug)
 else:
clfs[clf_name].fit(X_train, Y_train)
 oof[idx_train[val_index]] = clfs[clf_name].predict_proba(X_val)[:,1]
preds[idx_test] += clfs[clf_name].predict_proba(X_test)[:,1]/NFOLDS
auc = roc_auc_score(Y_val, oof[idx_train[val_index]])
 auc_folds = np.append(auc_folds, auc)
 auc_all = np.append(auc_all, np.mean(auc_folds))
auc_combo = roc_auc_score(train['target'].values, oof)
 auc_av = np.mean(auc_all)
 std = np.std(auc_all)/(np.sqrt(NFOLDS)*np.sqrt(magic_max+1))
if verbose:
print(f'The result summary for the {clf_name} classifier:')
 print(f'The combined CV score is {round(auc_combo, 5)}.')
 print(f'The folds average CV score is {round(auc_av, 5)}.')
 print(f'The standard deviation is {round(std, 5)}.
 ')
 return preds, auc_combo
for df in [train_df, test_df]:
df['Title'] = df['Name'].str.split(',' ).str[1].str.split(' ' ).str[1]
df['Deck'] = df['Cabin'].str[0]
all_df = pd.concat([train_df, test_df], sort=False)
 mean_age_by_title = all_df.groupby('Title' ).mean() ['Age']
for df in [train_df, test_df]:
for title, age in mean_age_by_title.iteritems() :
df.loc[df['Age'].isnull() &(df['Title'] == title), 'Age'] = age<correct_missing_values>
%%time
results = {}
results['rp']=np.array([])
 results['auc']=np.array([])
 np.random.seed(RS)
 for j in range(NTRIALS):
rp=10**(-2*np.random.rand())
 clfs_init={'QDA': QuadraticDiscriminantAnalysis(reg_param=rp)}
clfs={'QDA': QuadraticDiscriminantAnalysis(reg_param=rp)}
Y_pseudo, _ = train_classifier('QDA', clfs=clfs_init, verbose=0)
 _, auc = train_classifier('QDA', clfs=clfs, Y_pseudo=Y_pseudo, verbose=0)
 results['rp']=np.append(results['rp'], rp)
 results['auc']=np.append(results['auc'], auc)
 print(f"Trial number {j}: AUC = {round(auc, 5)}, rp={round(rp, 5)}.
 " )
test_df.Fare.fillna(0,inplace=True )<split>
auc_max = np.max(results['auc'])
 i_max = np.argmax(results['auc'])
 rp_best = results['rp'][i_max]
print(f"The highest AUC achived is {round(auc_max, 5)} for rp={round(rp_best, 5)}.")
 auc_min = np.min(results['auc'])
 i_min = np.argmin(results['auc'])
 print(f"The lowest AUC achived is {round(auc_min, 5)} for rp={round(results['rp'][i_min], 5)}.")
 print(f"The smallest value of `reg_param` that was explored during the search is {round(np.min(results['rp']), 5)}.")
 print(f"The larges value of `reg_param` that was explored during the search is {round(np.max(results['rp']), 5)}." )
dep_var = 'Survived'
cat_names = ['Pclass', 'Sex', 'Embarked', 'Title', 'Deck']
cont_names = ['Age', 'Fare', 'SibSp', 'Parch']
procs = [FillMissing, Categorify, Normalize]
test = TabularList.from_df(test_df, cat_names=cat_names, cont_names=cont_names, procs=procs)
 data =(TabularList.from_df(train_df, path='.', cat_names=cat_names, cont_names=cont_names, procs=procs)
.split_by_idx(list(range(0,200)))
.label_from_df(cols=dep_var)
.add_test(test, label=0)
.databunch() )<define_variables>
clfs_best = {'QDA': QuadraticDiscriminantAnalysis(reg_param=rp_best)}
preds_best, auc_best = train_classifier('QDA', clfs=clfs_best, Y_pseudo=Y_pseudo, verbose=0)
 print(f"AUC: {auc_best}" )
<train_model><EOS> 
sub['target'] = preds_best
sub.to_csv('submission.csv',index=False )