Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<create_dataframe>
%matplotlib inline
base_transform = transforms.Compose([
transforms.Resize(224),
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 data_test = RetinopathyDataset(base_transform, is_test=True)
 data_loader = torch.utils.data.DataLoader(data_test,
batch_size=16, shuffle=False,
num_workers=0, drop_last=False)
 def get_test_features(data_loader, extractor):

 for bi, d in enumerate(data_loader):
if bi % 4 == 0 : print(".", end="")
 img_tensor = d["image"].to(DEVICE)
 with torch.no_grad() : feature = extractor(img_tensor)
 feature = feature.cpu().detach().numpy()
if bi == 0 :
features = feature
else :
features = np.concatenate([features, feature], axis=0)
 return features<choose_model_class>
train_file_path = ".. /input/titanic/train.csv"
test_file_path = ".. /input/titanic/test.csv"
train_data = pd.read_csv(train_file_path)
 test_data = pd.read_csv(test_file_path )
print("................................ v")
 predictions = np.zeros(( len(data_test),5))
for tta in range(1):
print("............ tta"+str(tta)+"................ ")
 for no in range(NUM_FOLDS):
extractor = get_extractor_model(no)
 features = get_test_features(data_loader, extractor)
 print("",no)
 xgb_model = xgb.XGBClassifier()
model_path = os.path.join("xgb_model_"+str(no))
xgb_model = pickle.load(open(model_path, "rb"))
prediction = xgb_model.predict_proba(features)
 predictions = predictions + prediction<create_dataframe>
train_data.groupby(by=['Pclass'] ).count()
batch_size = 8
data_augmentation = transforms.Compose([
transforms.Resize(( MODEL_SIZE, MODEL_SIZE)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 data_test = RetinopathyDataset(data_augmentation, is_test=True)
 data_loader = torch.utils.data.DataLoader(data_test,
batch_size=batch_size, shuffle=False,
num_workers=0, drop_last=False )<train_model>
perc = train_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean().sort_values(by='Survived', ascending=False)
 perc*100
softmax = nn.Softmax(dim=1)
 for tta in range(4):
print("............ tta"+str(tta)+"...............")
 for no in range(NUM_FOLDS):
model = get_trained_model(no)
 batch_slice =(0, 0)
 for bi, d in enumerate(data_loader):
if bi %(64//batch_size)== 0 : print(".", end="")
 img_tensor = d["image"].to(DEVICE, dtype=torch.float)
 batch_slice =(batch_slice[1], batch_slice[1]+img_tensor.size(0))
with torch.no_grad() :
feature = model(img_tensor)
 feature = softmax(feature)
 predictions[batch_slice[0]:batch_slice[1],:] += \
feature.cpu().detach().squeeze(0 ).numpy()
print("" )<save_to_csv>
def extract_title(name):

 for string in name.split() :
if '.' in string:
return string[:-1]
train_data['Title'] = train_data['Name'].apply(lambda n: extract_title(n))
test_data['Title'] = test_data['Name'].apply(lambda n: extract_title(n))
print(test_data['Title'].value_counts() ,'
',train_data['Title'].value_counts())
prediction_final = predictions.argmax(axis=1)
 csv_file = os.path.join(DATA_SOURCE, "sample_submission.csv")
 df = pd.read_csv(csv_file)
 df["diagnosis"] = prediction_final
df.to_csv('submission.csv',index=False )<feature_engineering>
for dataframe in [train_data, test_data]:
dataframe['Title'] = dataframe['Title'].replace('Mlle', 'Miss')
 dataframe['Title'] = dataframe['Title'].replace('Ms', 'Miss')
 dataframe['Title'] = dataframe['Title'].replace('Mme', 'Mrs')
 dataframe['Title'] = dataframe['Title'].replace(['Lady', 'Capt', 'Col','Don', 'Dr',
'Major', 'Rev', 'Sir', 'Dona', 'Countess', 'Jonkheer'], 'Other')
 dataframe.drop('Name', axis=1, inplace=True)
t_start = time.time()<define_variables>
print('% of survived females:', train_data['Survived'][train_data['Sex'] == 'female'].value_counts(normalize = True)[1]*100)
 print('% of survived males:', train_data['Survived'][train_data['Sex'] == 'male'].value_counts(normalize = True)[1]*100)
IMG_WIDTH = 456
IMG_HEIGHT = 456
CHANNEL = 3
BATCH_SIZE = 4
EPOCHS_OLD_DATA = 10
WARMUP_EPOCHS = 3
NUM_CLASSES = 5
SEED = 2
LEARNING_RATE = 1e-4
WARMUP_LEARNING_RATE = 1e-3
ES_PATIENCE = 5
RLROP_PATIENCE = 3
DECAY_DROP = 0.5<define_variables>
train_data[['Sex', 'Survived']].groupby(['Sex'], as_index=False ).mean().sort_values(by='Survived', ascending=False )
BASE_DIR = '/kaggle/input/aptos2019-blindness-detection/'
TRAIN_DIR = '/kaggle/input/aptos2019-blindness-detection/train_images'
TEST_DIR = '/kaggle/input/aptos2019-blindness-detection/test_images'
TRAIN_DIR = '/kaggle/input/diabetic-retinopathy-resized/resized_train/resized_train'<load_from_csv>
train_data['FamilySize'] = train_data['SibSp'] + train_data['Parch']
test_data['FamilySize'] = train_data['SibSp'] + train_data['Parch']
train_data['IsAlone'] = train_data['FamilySize'].apply(lambda fs: 1 if fs == 0 else 0)
 test_data['IsAlone'] = test_data['FamilySize'].apply(lambda fs: 1 if fs == 0 else 0 )
TRAIN_DF = pd.read_csv(BASE_DIR + "train.csv",dtype='object')
 TEST_DF = pd.read_csv(BASE_DIR + "test.csv",dtype='object')
 TRAIN_DF = pd.read_csv("/kaggle/input/diabetic-retinopathy-resized/trainLabels.csv",dtype='object')
 X_COL='id_code'
Y_COL='diagnosis'<rename_columns>
train_data[['FamilySize', 'Survived']].groupby('FamilySize', as_index=False ).mean().sort_values(by='Survived', ascending=False )
TRAIN_DF.columns = ['id_code', 'diagnosis'] 
<categorify>
train_data.drop(['Parch', 'SibSp'], axis=1, inplace=True)
 test_data.drop(['Parch', 'SibSp'], axis=1, inplace=True )
def append_file_ext(file_name):
return file_name + ".png"
def append_file_ext_jpeg(file_name):
return file_name.replace(".png",".jpeg" )<feature_engineering>
train_data[['Ticket', 'PassengerId']].groupby('Ticket', as_index=False ).count().sort_values('PassengerId', ascending=False )
TRAIN_DF[X_COL] = TRAIN_DF[X_COL].apply(append_file_ext)
 TEST_DF[X_COL] = TEST_DF[X_COL].apply(append_file_ext)
 TRAIN_DF[X_COL] = TRAIN_DF[X_COL].apply(append_file_ext_jpeg )<concatenate>
train_data['TicketGroupSize'] = train_data.groupby(['Ticket'])['PassengerId'].transform('count')
 test_data['TicketGroupSize'] = test_data.groupby(['Ticket'])['PassengerId'].transform('count' )
df0 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '0']
df1 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '1']
df2 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '2']
df3 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '3']
df4 = TRAIN_DF.loc[TRAIN_DF['diagnosis'] == '4']
df0 = df0.head(2000)
 df1 = df1.head(2000)
 df2 = df2.head(2000)
 TRAIN_DF = df0.append([df1, df2, df3, df4],ignore_index = True)
 print(TRAIN_DF.head())
 print('**********************************')
 TRAIN_DF = shuffle(TRAIN_DF)
 print('**********************************')
 print(len(TRAIN_DF))
print(TRAIN_DF.head() )<choose_model_class>
train_data.drop('Ticket', axis=1, inplace=True)
 test_data.drop('Ticket', axis=1, inplace=True )
def create_model(input_shape, n_out):
input_tensor = Input(shape=input_shape)
 base_model = EfficientNetB5(weights=None,
include_top=False,
input_tensor=input_tensor)
 base_model.load_weights('/kaggle/input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dropout(0.5 )(x)
 x = Dense(2048, activation='relu' )(x)
 final_output = Dense(n_out, activation='softmax', name='final_output' )(x)
 model = Model(input_tensor, final_output)
 return model<choose_model_class>
fare_median = test_data['Fare'].median()
test_data['Fare'] = test_data['Fare'].fillna(fare_median )
model = create_model(input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNEL), n_out=NUM_CLASSES)
 for layer in model.layers:
layer.trainable = False
for i in range(-7, 0):
model.layers[i].trainable = True
metric_list = ["accuracy"]
optimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE)
 model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=metric_list)
<create_dataframe>
train_data['HasCabin'] = train_data['Cabin'].notnull().astype('int')
 test_data['HasCabin'] = test_data['Cabin'].notnull().astype('int' )
datagen = ImageDataGenerator(
 rescale=1./255.,
validation_split=0.25)
 train_generator = datagen.flow_from_dataframe(
 dataframe=TRAIN_DF,
directory=TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="training",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
valid_generator=datagen.flow_from_dataframe(
 dataframe=TRAIN_DF,
directory=TRAIN_DIR,
x_col=X_COL,
y_col=Y_COL,
subset="validation",
batch_size=BATCH_SIZE,
seed=SEED,
zoom_range=0.2,
horizontal_flip=True,
class_mode="categorical",
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))
test_datagen=ImageDataGenerator(rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(
 dataframe=TEST_DF,
directory=TEST_DIR,
x_col=X_COL,
y_col=None,
batch_size=BATCH_SIZE,
seed=SEED,
shuffle=False,
class_mode=None,
preprocessing_function=preprocess_image,
target_size=(IMG_WIDTH,IMG_HEIGHT))<define_variables>
train_data.drop('Cabin', axis=1, inplace=True)
 test_data.drop('Cabin', axis=1, inplace=True )
STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
STEP_SIZE_TEST=test_generator.n//test_generator.batch_size
print(STEP_SIZE_TRAIN)
 print(STEP_SIZE_VALID)
 print(STEP_SIZE_TEST )<set_options>
train_data['Embarked'].value_counts()
gc.collect()<train_model>
full_data = pd.concat([train_data.drop('Survived', axis=1), test_data], ignore_index=True )
history_warmup = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=WARMUP_EPOCHS,
verbose=1 ).history<set_options>
full_data[['Age', 'Pclass', 'Sex', 'Embarked']].groupby(['Pclass', 'Sex', 'Embarked'])['Age'].mean()
gc.collect()<choose_model_class>
train_data[['Age', 'Pclass', 'Sex', 'Embarked']].groupby(['Pclass', 'Sex', 'Embarked'])['Age'].mean()
for layer in model.layers:
layer.trainable = True
es = EarlyStopping(monitor='val_loss',
mode='min',
patience=ES_PATIENCE,
restore_best_weights=True,
verbose=1)
 rlrop = ReduceLROnPlateau(monitor='val_loss',
mode='min',
patience=RLROP_PATIENCE,
factor=DECAY_DROP,
min_lr=1e-6,
verbose=1)
 model_checkpoint = ModelCheckpoint('EfficientNetB5_Best_KV.h5',
verbose=1,
save_best_only=True)
 callback_list = [es, rlrop, model_checkpoint]
optimizer = optimizers.Adam(lr=LEARNING_RATE)
 model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=metric_list)
<set_options>
collected = gc.collect()
print("Garbage collector: collected","%d objects." % collected )<train_model>
def get_age(element):
age = element[0]
pclass = element[1]
sex = element[2]
embarked = element[3]
if(pd.isnull(age)) :
temp_data = full_data[(full_data['Pclass'] == pclass)&(full_data['Sex'] == sex)&(full_data['Embarked'] == embarked)]
mean_age = temp_data['Age'].mean()
return mean_age
return age
train_data['Age'] = train_data[['Age', 'Pclass', 'Sex', 'Embarked']].apply(get_age, axis=1)
 test_data['Age'] = test_data[['Age', 'Pclass', 'Sex', 'Embarked']].apply(get_age, axis=1 )
history_finetunning = model.fit_generator(generator=train_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
epochs=EPOCHS_OLD_DATA,
callbacks=callback_list,
verbose=1 ).history<set_options>
sex = pd.get_dummies(train_data['Sex'], prefix='Sex', drop_first=True)
 embarked = pd.get_dummies(train_data['Embarked'], prefix='Embarked', drop_first=True)
 title = pd.get_dummies(train_data['Title'], prefix='Title', drop_first=True)
 train_data.drop(['Sex', 'Title', 'Embarked'], axis=1, inplace=True)
 train_data = pd.concat([train_data, sex, embarked, title], axis=1 )
gc.collect()<load_pretrained>
sex = pd.get_dummies(test_data['Sex'], prefix='Sex', drop_first=True)
 embarked = pd.get_dummies(test_data['Embarked'], prefix='Embarked', drop_first=True)
 title = pd.get_dummies(test_data['Title'], prefix='Title', drop_first=True)
 test_data.drop(['Sex', 'Title', 'Embarked'], axis=1, inplace=True)
 test_data = pd.concat([test_data, sex, embarked, title], axis=1 )
model = load_model('EfficientNetB5_Best_KV.h5' )<define_variables>
y_train = train_data['Survived']
X_train = train_data.drop(['Survived', 'Fare'], axis=1)
 test_data = test_data.drop(['Fare'], axis=1)
if test_generator.n%BATCH_SIZE > 0:
PREDICTION_STEPS =(test_generator.n//BATCH_SIZE)+ 1
else:
PREDICTION_STEPS =(test_generator.n//BATCH_SIZE)
 print(PREDICTION_STEPS )<define_variables>
kfold = StratifiedKFold(n_splits=5)
 random_state = 13
print(test_generator.n)
 print(test_generator.batch_size)
 print(STEP_SIZE_TEST)
 print(BATCH_SIZE )<predict_on_test>
random_forest_classifier = RandomForestClassifier(random_state=random_state)
 cv_result = cross_val_score(random_forest_classifier, X_train, y_train, cv=kfold, scoring='accuracy')
 print("CV result mean: ", cv_result.mean() , "CV result std: ", cv_result.std())
 cv_result
test_generator.reset()
preds = model.predict_generator(test_generator,
steps=PREDICTION_STEPS,
verbose=1)
 predictions = [np.argmax(pred)for pred in preds]<set_options>
xgbClassifier = XGBClassifier(random_state=random_state)
 cv_result = cross_val_score(xgbClassifier, X_train, y_train, cv=kfold, scoring='accuracy')
 print("CV result mean: ", cv_result.mean() , "CV result std: ", cv_result.std())
 cv_result
gc.collect()<define_variables>
rf_param_grid = {'max_depth' :[1, 2, 3, 4, 5, 6],
'max_features' :[2, 4, 6, 8, 10],
'min_samples_split':[2, 4, 6, 8, 10],
'bootstrap' :[False, True],
'n_estimators' :[50, 100, 200, 500],
'criterion' :['gini']}
grid_search_random_forest_classifier = GridSearchCV(random_forest_classifier, param_grid=rf_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)
 grid_search_random_forest_classifier.fit(X_train, y_train)
 random_forest_best = grid_search_random_forest_classifier.best_estimator_
grid_search_random_forest_classifier.best_score_
filenames = test_generator.filenames 
<save_to_csv>
grid_search_random_forest_classifier.best_params_
results = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})
 results['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])
 results.astype({'diagnosis': 'int64'})
 results.to_csv('submission.csv',index=False)
 print(results.head(10))<set_options>
xgb_param_grid={'colsample_bylevel':[0.1, 0.9, 1],
'colsample_bytree' :[0.2, 0.8, 1],
'gamma' :[0.99, 9, 99],
'max_depth' :[2, 4, 6, 8, 10],
'min_child_weight' :[1, 2, 4, 6, 8, 10],
'n_estimators' :[10, 20, 50, 70, 100, 200, 500, 1000]}
grid_search_xgboost_classifier = GridSearchCV(xgbClassifier, param_grid=xgb_param_grid, cv=kfold, scoring='accuracy', n_jobs=4, verbose=1)
 grid_search_xgboost_classifier.fit(X_train, y_train)
 xgb_best = grid_search_xgboost_classifier.best_estimator_
grid_search_xgboost_classifier.best_score_
gc.collect()<import_modules>
grid_search_xgboost_classifier.best_params_
from fastai import *
from fastai.vision import *
import pandas as pd
import matplotlib.pyplot as plt
import pandas as pd
import os
import numpy as np
import pandas as pd
import glob
import matplotlib.pyplot as plt
import imagehash
import psutil
from PIL import Image
from joblib import Parallel, delayed
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython import display
import time 
<feature_engineering>
x_tr, x_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.3, random_state=random_state )
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 df.head()<define_variables>
random_forest_classifier = RandomForestClassifier(n_estimators=200, max_depth=3, max_features=8, min_samples_split=2, bootstrap=True, random_state=random_state)
 random_forest_classifier.fit(x_tr, y_tr)
 predicted_test = random_forest_classifier.predict(x_val)
 accuracy_random_forest = accuracy_score(y_val, predicted_test)
 f1_random_forest = f1_score(y_val, predicted_test)
 print("Random forest scores: ", accuracy_random_forest, f1_random_forest )
bs = 32
sz=140<compute_test_metric>
xgbClassifier = XGBClassifier(max_depth=4,n_estimators=20,gamma=9,colsample_bylevel=0.9,random_state=random_state)
 xgbClassifier.fit(x_tr, y_tr)
 predicted_test = xgbClassifier.predict(x_val)
 accuracy_xgb = accuracy_score(y_val, predicted_test)
 f1_xgb = f1_score(y_val, predicted_test)
 print("XGBooster scores: ", accuracy_xgb, f1_xgb )
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.argmax(y_hat,1), y, weights='quadratic'),device='cuda:0' )<choose_model_class>
xgbClassifier.fit(X_train, y_train)
 submission_prediction = xgbClassifier.predict(test_data)
 submission = pd.DataFrame({
'PassengerId': test_data['PassengerId'],
'Survived': submission_prediction
})
 submission_file_path = './titanic_submission.csv'
submission.to_csv(submission_file_path, index=False )