Kaggle competition: Titanic - Machine Learning from Disaster
%%time
best_epoch_score = np.inf
print('epoch\ttrain-MSE\tval-MSE\tq-kappa\tlr\t\ttime [min]')
 print('------------------------------------------------------------------')
 for epoch in range(25):
start = time.time()
train_loss = []
for i,(X, y, id_)in enumerate(train_loader):
net.train()
optimizer.zero_grad()
out = net(X.cuda())
 loss = loss_function(out, y.float().cuda().view(-1, 1))
train_loss.append(loss.item())
 loss.backward()
optimizer.step()
validation_loss = []
predictions = np.array([])
 truth = np.array([])
 for X, y, id_ in val_loader:
net.eval()
out = net(X.cuda())
 loss = loss_function(out, y.float().cuda().view(-1, 1))
validation_loss.append(loss.item())
 predictions = np.append(predictions, out.detach().cpu().numpy())
 truth = np.append(truth, y.detach().cpu().numpy().astype(int))
current_lr = optimizer.param_groups[0]['lr']
scheduler.step()
qk = cohen_kappa_score(predictions.round().astype(int), truth, weights='quadratic')
 duration =(time.time() - start)/ 60
print(f'{epoch}:\t{np.mean(train_loss):.4f}\t\t{np.mean(validation_loss):.4f}\t{qk:.4f}\t{current_lr:.6f}\t{duration:.2f}')
 if np.mean(validation_loss)< best_epoch_score:
torch.save(net.state_dict() , 'state_dict_best.pt')
 best_epoch_score = np.mean(validation_loss)
 best_epoch = epoch
print(f'epoch with best validation-score: {best_epoch}')
 plt.hist(predictions, bins=5)
 plt.xlim(-1, 5)
 plt.title('distribution of predictions 
(before rounding)')
 plt.show()
plt.hist(train_df.diagnosis.values, bins=5)
 plt.xlim(-1, 5)
 plt.title('distribution of labels')
 plt.show()<load_from_csv>
sns.set(style="ticks", context="talk")
test_dir = join(DATA_DIR, 'test_images')
 test_df = pd.read_csv(join(DATA_DIR, 'test.csv'))
test_df.head(3 )<categorify>
testing = pd.read_csv('/kaggle/input/titanic/test.csv')
 train = pd.read_csv('/kaggle/input/titanic/train.csv')
 target = 'Survived'
test = testing.copy()
test.info()
print('-'*70)
 train.info()
print('-'*70)
 train.tail(10 )
def sample_images(train_dir: str, test_dir: str, n=10):

 train_files = choice(os.listdir(train_dir), size=n)
 test_files = choice(os.listdir(test_dir), size=n)
 images = []
for train_f, test_f in zip(train_files, test_files):
train_img = Image.open(join(train_dir, train_f))
test_img = Image.open(join(test_dir, test_f))
images.append(( train_img, test_img))
def show_image(i):
train_image, test_image = images[i]
fig,(ax1, ax2)= plt.subplots(1, 2)
 fig.set_size_inches(20, 10)
 ax1.imshow(train_image)
 ax2.imshow(test_image)
 ax1.set_title('train')
 ax2.set_title('test')
 plt.show()
return interactive(show_image, i=(0, n-1))
sample_images(train_dir=train_dir,
test_dir=test_dir,
n=10)
<categorify>
print(train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False ).round(2),
'
',
train[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False ).round(2))
test_transform = transforms.Compose([
transforms.Resize(( 256, 256)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomRotation(( -20, 20)) ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 test_ds = Diabetic_Retionopathy_Data(test_dir,
test_df,
transform=test_transform,
train=False)
 test_loader = DataLoader(test_ds, batch_size=batchsize)
 X, _ = next(iter(test_loader))
print(f'batch-dimension:
X = {X.shape}' )<find_best_params>
train['agebucket'] = pd.cut(train['Age'], 5)
 test['agebucket'] = pd.cut(test['Age'], 5)
 train[['agebucket', 'Survived']].groupby(['agebucket'] ).mean().sort_values(by='agebucket', ascending=True ).round(2 )
net.load_state_dict(torch.load('state_dict_best.pt'))
net.eval()
net.cuda()
id2prediction = {}
for i,(X, id_)in enumerate(test_loader):
out = net(X.cuda())
 preds = out.detach().cpu().numpy().ravel()
id2prediction = {**id2prediction, **dict(zip(id_, preds.round().astype(int ).tolist())) }<save_to_csv>
for dataset in [train, test]:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4
train.head()
submission_df = pd.read_csv(join(DATA_DIR, 'sample_submission.csv'))
submission_df.diagnosis = submission_df.id_code.map(id2prediction)
 submission_df.diagnosis = submission_df.diagnosis.map(lambda p: max(p, 0))
submission_df.diagnosis = submission_df.diagnosis.map(lambda p: min(p, 4))
submission_df.to_csv('submission.csv', index=False)
 display(submission_df.head() )<install_modules>
print(train[['Family', 'Survived']].groupby(['Family'], as_index=False ).mean().sort_values(by='Survived', ascending=False ).round(2),
'
',
train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean().sort_values(by='Survived', ascending=False ).round(2))
!pip install -U '.. /input/install/efficientnet-0.0.3-py2.py3-none-any.whl'<load_from_csv>
train['isalone'] = [1 if x == 1 else 0 for x in train['Family']]
test['isalone'] = [1 if x == 1 else 0 for x in test['Family']]
train[['isalone', 'Survived']].groupby(['isalone'] ).mean().sort_values(by='Survived', ascending=False ).round(2 )
TEST_IMG_PATH = '.. /input/aptos2019-blindness-detection/test_images/'
test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(test_df.shape)
 original_names = test_df['id_code'].values
test_df['id_code'] = test_df['id_code'] + ".png"
test_df['diagnosis'] = np.zeros(test_df.shape[0])
 display(test_df.head() )<choose_model_class>
dummy_features = ['Sex','Title', 'isalone']
drop_features = ['Embarked', 'PassengerId', 'Ticket', 'Name', 'Cabin','Parch','SibSp', 'agebucket']
train = pd.concat([train, pd.get_dummies(train[dummy_features])], axis = 1, sort = False)
 train.drop(columns = train[dummy_features], inplace = True)
 train.drop(columns = train[drop_features], inplace = True)
 test = pd.concat([test, pd.get_dummies(test[dummy_features])], axis = 1, sort = False)
 test.drop(columns = test[dummy_features], inplace = True)
 test.drop(columns = test[drop_features], inplace = True)
 train.tail()
HEIGHT = 300
WIDTH = 300
COEFF = [0.5,1.5,2.5,3.5]
efficientnetb3 = EfficientNetB3(
 weights=None,
input_shape=(HEIGHT,WIDTH,3),
include_top=False 
)
 def build_model() :
model = Sequential()
model.add(efficientnetb3)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.Dense(5, activation=elu))
model.add(layers.Dense(1, activation="linear"))
return model
model = build_model()
model.load_weights('.. /input/aptos-3-best-models/best_weights/val_model_1.h5')
 model.summary()<init_hyperparams>
y = train[target]
x = train.drop(columns = target)
 x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state = 42 )
tta_steps = 4
predictions = []
for i in tqdm(range(tta_steps)) :
test_generator = ImageDataGenerator(rescale=1./255,
horizontal_flip=True,
rotation_range= 90,
vertical_flip=True,
brightness_range=(0.5,2),
zoom_range= 0.2,
fill_mode='constant',
cval = 0 ).flow_from_dataframe(test_df,
x_col='id_code',
y_col = 'diagnosis',
directory = TEST_IMG_PATH,
target_size=(WIDTH, HEIGHT),
batch_size=1,
class_mode='other',
shuffle = False,
preprocessing_function=preprocess_image)
 preds = model.predict_generator(test_generator, steps = test_df.shape[0])
 predictions.append(preds)
 del test_generator
gc.collect()
y_test_1 = np.mean(predictions, axis=0)
 for i, pred in enumerate(y_test_1):
if pred < COEFF[0]:
y_test_1[i] = 0
elif pred >= COEFF[0] and pred < COEFF[1]:
y_test_1[i] = 1
elif pred >= COEFF[1] and pred < COEFF[2]:
y_test_1[i] = 2
elif pred >= COEFF[2] and pred < COEFF[3]:
y_test_1[i] = 3
else:
y_test_1[i] = 4
y_test_1 = y_test_1.flatten()<set_options>
RF = ensemble.RandomForestClassifier()
RF_params = {
'n_estimators':[n for n in range(60,140,10)],
'max_depth':[n for n in range(3, 6)],
'max_features' : ['sqrt', 'log2', None],
'random_state' : [42]
}
RF_model = GridSearchCV(RF, param_grid = RF_params, cv = 5, n_jobs = -1 ).fit(x_train, y_train)
 print("Best Hyper Parameters:",RF_model.best_params_)
 RF_probs = RF_model.predict_proba(x_test)
 RF_probs = RF_probs[:, 1]
RF_auc = roc_auc_score(y_test, RF_probs)
 print('AUC: %.3f' % RF_auc)
 RF_predictions = RF_model.predict(x_test)
 RF_accuracy = accuracy_score(y_test, RF_predictions)
 print("RF accuracy: %.3f" % RF_accuracy)
 RF_fpr, RF_tpr, RF_thresholds = roc_curve(y_test, RF_probs)
 plt.plot([0, 1], [0, 1], linestyle='--')
 plt.plot(RF_fpr, RF_tpr, color = 'tab:green')
 plt.show()
del model
gc.collect()<choose_model_class>
GBT = ensemble.GradientBoostingClassifier()
GBT_params = {
'n_estimators':[n for n in range(180, 240, 20)],
'max_depth':[n for n in range(3, 6)],
'learning_rate': [0.1, 0.25, 0.5],
'random_state' : [42]
}
GBT_model = GridSearchCV(GBT, param_grid = GBT_params, cv = 5, n_jobs = -1)
 GBT_model.fit(x_train, y_train)
 print("Best Hyper Parameters:",GBT_model.best_params_)
 GBT_probs = GBT_model.predict_proba(x_test)
 GBT_probs = GBT_probs[:, 1]
GBT_auc = roc_auc_score(y_test, GBT_probs)
 print('AUC: %.3f' % GBT_auc)
 GBT_predictions = GBT_model.predict(x_test)
 GBT_accuracy = accuracy_score(y_test, GBT_predictions)
 print("GBT accuracy: %.3f" % GBT_accuracy)
 GBT_fpr, GBT_tpr, GBT_thresholds = roc_curve(y_test, GBT_probs)
 plt.plot([0, 1], [0, 1], linestyle='--')
 plt.plot(GBT_fpr, GBT_tpr, color = 'tab:orange')
 plt.show()
HEIGHT = 320
WIDTH = 320
COEFF = [0.53164905, 1.37748383, 2.60330927, 3.40191179]
def build_model() :
efficientnetb3 = EfficientNetB3(
 weights=None,
input_shape=(HEIGHT,WIDTH,3),
include_top=False 
)
 model = Sequential()
model.add(efficientnetb3)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.5))
model.add(layers.BatchNormalization())
 model.add(layers.Dense(5, activation=elu))
model.add(layers.Dense(1, activation="linear"))
return model
model = build_model()
model.load_weights('.. /input/aptos-3-best-models/best_weights/val_model_2.h5')
 model.summary()<init_hyperparams>
print("GBT cohen_kappa_score: %.3f" % cohen_kappa_score(y_test, GBT_predictions))
print("RF cohen_kappa_score: %.3f" % cohen_kappa_score(y_test, RF_predictions))
tta_steps = 4
predictions = []
for i in tqdm(range(tta_steps)) :
test_generator = ImageDataGenerator(rescale=1./255,
horizontal_flip=True,
rotation_range= 90,
vertical_flip=True,
brightness_range=(0.5,2),
zoom_range= 0.2,
fill_mode='constant',
preprocessing_function=preprocess_image,
cval = 0 ).flow_from_dataframe(test_df,
x_col='id_code',
y_col = 'diagnosis',
directory = TEST_IMG_PATH,
target_size=(WIDTH, HEIGHT),
batch_size=1,
class_mode='other',
shuffle = False)
 preds = model.predict_generator(test_generator, steps = test_df.shape[0])
 predictions.append(preds)
 del test_generator
gc.collect()
y_test_2 = np.mean(predictions, axis=0)
 for i, pred in enumerate(y_test_2):
if pred < COEFF[0]:
y_test_2[i] = 0
elif pred >= COEFF[0] and pred < COEFF[1]:
y_test_2[i] = 1
elif pred >= COEFF[1] and pred < COEFF[2]:
y_test_2[i] = 2
elif pred >= COEFF[2] and pred < COEFF[3]:
y_test_2[i] = 3
else:
y_test_2[i] = 4
y_test_2 = y_test_2.flatten()<set_options>
print("GBT", classification_report(y_test, GBT_predictions))
print("-"*100)
 print("RF", classification_report(y_test, RF_predictions))
del model
gc.collect()<define_variables>
predict_RF = RF_model.predict(test)
 predict_GBT = GBT_model.predict(test)
 submit_RF = pd.DataFrame({'PassengerId':testing['PassengerId'],'Survived':predict_RF})
 submit_GBT = pd.DataFrame({'PassengerId':testing['PassengerId'],'Survived':predict_GBT})
 filename_RF = 'Titanic Prediction RF.csv'
submit_RF.to_csv(filename_RF,index=False)
 print('Saved file: ' + filename_RF)
 filename_GBT = 'Titanic Prediction GBT.csv'
submit_GBT.to_csv(filename_GBT,index=False)
 print('Saved file: ' + filename_GBT )