Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
sns.set()
%matplotlib inline
warnings.filterwarnings('ignore')
COEFF = [0.5, 1.5, 2.5, 3.5]
for i, pred in enumerate(y_test):
if pred < COEFF[0]:
y_test[i] = 0
elif pred >= COEFF[0] and pred < COEFF[1]:
y_test[i] = 1
elif pred >= COEFF[1] and pred < COEFF[2]:
y_test[i] = 2
elif pred >= COEFF[2] and pred < COEFF[3]:
y_test[i] = 3
else:
y_test[i] = 4<save_to_csv>
path_train = '.. /input/train.csv'
path_test = '.. /input/test.csv'
test_df['diagnosis'] = y_test.astype(int)
 test_df['id_code'] = test_df['id_code'].str.replace(r'.png$', '')
 test_df.to_csv('submission.csv',index=False)
 print("Submission Distribution:")
 print(round(test_df.diagnosis.value_counts() /len(test_df)*100,4))<define_variables>
train_df_raw = pd.read_csv(path_train)
 train_df_raw.head()
DEVICE = torch.device("cuda:0")
 DATA_SOURCE = os.path.join(".. ","input","aptos2019-blindness-detection")
 MODEL_SOURCE = os.path.join(".. ","input","densenet161-1-18-v2-pth")
 MODEL_SIZE = 224<prepare_x_and_y>
draw_missing_data_table(train_df_raw )
def crop_image(img,tol=7):
w, h = img.shape[1],img.shape[0]
gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 gray_img = cv2.blur(gray_img,(5,5))
shape = gray_img.shape
gray_img = gray_img.reshape(-1,1)
 quant = quantile_transform(gray_img, n_quantiles=256, random_state=0, copy=True)
 quant =(quant*256 ).astype(int)
 gray_img = quant.reshape(shape)
 xp =(gray_img.mean(axis=0)>tol)
 yp =(gray_img.mean(axis=1)>tol)
 x1, x2 = np.argmax(xp), w-np.argmax(np.flip(xp))
y1, y2 = np.argmax(yp), h-np.argmax(np.flip(yp))
if x1 >= x2 or y1 >= y2 :
return img
else:
img1=img[y1:y2,x1:x2,0]
img2=img[y1:y2,x1:x2,1]
img3=img[y1:y2,x1:x2,2]
img = np.stack([img1,img2,img3],axis=-1)
 return img
def process_image(image, size=512):
image = cv2.resize(image,(size,int(size*image.shape[0]/image.shape[1])))
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 try:
image = crop_image(image, tol=15)
 except Exception as e:
image = image
print(str(e))
return image<load_from_csv>
def preprocess_data(df):
processed_df = df
processed_df['Embarked'].fillna('C', inplace=True)
 processed_df['Age'] = processed_df.groupby(['Pclass','Sex','Parch','SibSp'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Age'] = processed_df.groupby(['Pclass','Sex','Parch'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Age'] = processed_df.groupby(['Pclass','Sex'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Fare'] = processed_df['Fare'].interpolate()
processed_df['Cabin'].fillna('U', inplace=True)
 processed_df['Title'] = pd.Series(( name.split('.')[0].split(',')[1].strip() for name in train_df_raw['Name']), index=train_df_raw.index)
 processed_df['Title'] = processed_df['Title'].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 processed_df['Title'] = processed_df['Title'].replace(['Mlle', 'Ms'], 'Miss')
 processed_df['Title'] = processed_df['Title'].replace('Mme', 'Mrs')
 processed_df['Title'] = processed_df['Title'].map({"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5})
 processed_df['Age'] = processed_df.groupby(['Title'])['Age'].transform(lambda x: x.fillna(x.mean()))
processed_df['Sex'] = processed_df['Sex'].map({'male': 0, 'female': 1})
 processed_df['Embarked'] = processed_df['Embarked'].map({'S': 0, 'C': 1, 'Q': 2})
 processed_df['FamillySize'] = processed_df['SibSp'] + processed_df['Parch'] + 1
processed_df['FamillySize'][processed_df['FamillySize'].between(1, 5, inclusive=False)] = 2
processed_df['FamillySize'][processed_df['FamillySize']>5] = 3
processed_df['IsAlone'] = np.where(processed_df['FamillySize']!=1, 0, 1)
 processed_df['IsChild'] = processed_df['Age'] < 18
processed_df['IsChild'] = processed_df['IsChild'].astype(int)
 processed_df['Cabin'] = processed_df['Cabin'].str[:1]
processed_df['Cabin'] = processed_df['Cabin'].map({cabin: p for p, cabin in enumerate(set(cab for cab in processed_df['Cabin'])) })
 processed_df['TicketSurvivor'] = pd.Series(0, index=processed_df.index)
 tickets = processed_df['Ticket'].value_counts().to_dict()
for t, occ in tickets.items() :
if occ != 1:
table = train_df_raw['Survived'][train_df_raw['Ticket'] == t]
if sum(table)!= 0:
processed_df['TicketSurvivor'][processed_df['Ticket'] == t] = 1
processed_df = processed_df.drop(['Name', 'Ticket', 'PassengerId'], 1)
 return processed_df
class RetinopathyDataset(Dataset):
def __init__(self, transform, is_test=False):
self.transform = transform
self.base_transform = transforms.Resize(( MODEL_SIZE, MODEL_SIZE))
self.is_test = is_test
if not os.path.exists("cache"): os.mkdir("cache")
 if is_test : file = "test.csv"
else : file = "train.csv"
csv_file = os.path.join(DATA_SOURCE, file)
 df = pd.read_csv(csv_file)
 self.data = df.reset_index(drop=True)
 def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if self.is_test : archive = "test_images"
else : archive = "train_images"
folder = os.path.join(DATA_SOURCE, archive)
 code = str(self.data.loc[idx, 'id_code'])
 file = code + ".png"
cache_path = os.path.join("cache",code+".png")
 cached = os.path.exists(cache_path)
 if not cached :
path = os.path.join(folder, file)
 image = cv2.imread(path)
 image = process_image(image)
 imgpil = Image.fromarray(image)
 imgpil = self.base_transform(imgpil)
 imgpil.save(cache_path,"PNG")
 imgpil = Image.open(cache_path)
 img_tensor = self.transform(imgpil)
 if self.is_test : return {'image': img_tensor}
else :
label = self.data.loc[idx, "diagnosis"]
return {'image': img_tensor, 'label': label}
def get_df(self):
return self.data<split>
train_df = train_df_raw.copy()
X = train_df.drop(['Survived'], 1)
 Y = train_df['Survived']
X = preprocess_data(X)
 sc = StandardScaler()
X = pd.DataFrame(sc.fit_transform(X.values), index=X.index, columns=X.columns)
 X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)
 X_train.head()
NUM_FOLDS = 5
data_augmentation = transforms.Compose([
transforms.RandomRotation(( -15, 15)) ,
transforms.Resize(224),
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
 ])
 DATA = RetinopathyDataset(data_augmentation)
 df = DATA.get_df()
skf = StratifiedKFold(n_splits=NUM_FOLDS)
 folds_generator = skf.split(df.index.values, df.diagnosis.values)
 data_train, data_eval = [], []
for t, e in folds_generator:
data_train.append(t)
 data_eval.append(e )<load_pretrained>
lg = LogisticRegression(solver='lbfgs', random_state=42)
 lg.fit(X_train, Y_train)
 logistic_prediction = lg.predict(X_test)
 score = metrics.accuracy_score(Y_test, logistic_prediction)
 display_confusion_matrix(Y_test, logistic_prediction, score=score )
def get_dataloader_for_fold(n, data, train_data, eval_data, batch_size):

 train_sampler = SubsetRandomSampler(train_data[n])
 valid_sampler = SubsetRandomSampler(eval_data[n])
 data_loader_train = torch.utils.data.DataLoader(data,
batch_size=batch_size, drop_last=False,
sampler=train_sampler)
 data_loader_eval = torch.utils.data.DataLoader(data,
batch_size=batch_size, drop_last=False,
sampler=valid_sampler)
 return data_loader_train, data_loader_eval<choose_model_class>
dt = DecisionTreeClassifier(min_samples_split=15, min_samples_leaf=20, random_state=42)
 dt.fit(X_train, Y_train)
 dt_prediction = dt.predict(X_test)
 score = metrics.accuracy_score(Y_test, dt_prediction)
 display_confusion_matrix(Y_test, dt_prediction, score=score )
class Classificator0(nn.Module):

 def __init__(self, size=128):
super(Classificator0, self ).__init__()
self.size = size
self.network = nn.Sequential(
 nn.BatchNorm1d(size),
nn.Dropout(p=0.3),
nn.Linear(in_features=size, out_features=5, bias=True), 
)
 def forward(self, x):
return self.network(x)
 class Classificator(nn.Module):

 def __init__(self, size=128):
super(Classificator, self ).__init__()
self.size = size
self.network = nn.Sequential(
 nn.BatchNorm1d(size),
nn.Dropout(p=0.25),
nn.Linear(in_features=size, out_features=size, bias=True),
nn.ReLU() ,
nn.BatchNorm1d(size),
nn.Dropout(p=0.5),
nn.Linear(in_features=size, out_features=5, bias=True), 
)
 def forward(self, x):
return self.network(x )<choose_model_class>
svm = SVC(gamma='auto', random_state=42)
 svm.fit(X_train, Y_train)
 svm_prediction = svm.predict(X_test)
 score = metrics.accuracy_score(Y_test, svm_prediction)
 display_confusion_matrix(Y_test, svm_prediction, score=score )
def get_base_model() :

 model = torchvision.models.densenet161(pretrained=False)
 in_features = model.classifier.in_features
model.classifier = Classificator0(in_features)
 model_path = os.path.join(MODEL_SOURCE, "densenet161.1.18.v2.pth")
 model.load_state_dict(torch.load(model_path))
model.classifier = Classificator(in_features)
 model = model.to(DEVICE)
 model.eval()
return model<choose_model_class>
rf = RandomForestClassifier(n_estimators=200, random_state=42)
 rf.fit(X_train, Y_train)
 rf_prediction = rf.predict(X_test)
 score = metrics.accuracy_score(Y_test, rf_prediction)
 display_confusion_matrix(Y_test, rf_prediction, score=score )
def train_model(model, optimizer, scheduler, train_data_loader, eval_data_loader,
file_name, num_epochs = 50, patience = 7, prev_loss = 1000.00):

 criterion = nn.CrossEntropyLoss()
countdown = patience
best_loss = 1000.00
since = time.time()
for epoch in range(num_epochs):
running_loss = 0.0
counter = 0
for bi, d in enumerate(train_data_loader):
inputs = d["image"].to(DEVICE, dtype=torch.float)
 labels = d["label"].to(DEVICE, dtype=torch.long)
 if inputs.shape[0] > 1 :
counter += inputs.size(0)
 model.to(DEVICE)
 model.train()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 loss_val = running_loss / counter
print("{:7} {:.4f} {:.4f}".format(counter, loss.item() *1, loss_val), end="\r")
 epoch_loss = running_loss /(len(train_data_loader)* train_data_loader.batch_size)
 time_elapsed = time.time() - since
print(" T{:3}/{:3} loss: {:.4f}({:3.0f}m {:2.0f}s)".format(
 epoch, num_epochs - 1, epoch_loss,time_elapsed // 60, time_elapsed % 60))
running_loss = 0.0
counter = 0
for bi, d in enumerate(eval_data_loader):
inputs = d["image"].to(DEVICE, dtype=torch.float)
 counter += inputs.size(0)
 labels = d["label"].to(DEVICE, dtype=torch.long)
 model.to(DEVICE)
 model.eval()
with torch.no_grad() :
outputs = model(inputs)
 loss = criterion(outputs, labels)
 running_loss += loss.item() * inputs.size(0)
 loss_val = running_loss / counter
print("{:7} {:.4f} {:.4f}".format(counter, loss.item() *1, loss_val), end="\r")
 epoch_loss = running_loss /(len(eval_data_loader)* eval_data_loader.batch_size)
 if epoch_loss < best_loss :
best_loss = epoch_loss
if epoch_loss < prev_loss:
torch.save(model.state_dict() , file_name)
 prev_loss = epoch_loss
print("*", end="")
 else:
print(".", end="")
 countdown = patience
else:
print("{:1}".format(countdown), end="")
 countdown -= 1
time_elapsed = time.time() - since
print("E{:3}/{:3} loss: {:.4f}({:3.0f}m {:2.0f}s)".format(
 epoch, num_epochs - 1, epoch_loss,time_elapsed // 60, time_elapsed % 60))
scheduler.step()
if countdown <= 0 : break
return prev_loss
print("done.")
<load_pretrained>
def build_ann(optimizer='adam'):
ann = Sequential()
ann.add(Dense(units=32, kernel_initializer='glorot_uniform', activation='relu', input_shape=(13,)))
 ann.add(Dense(units=64, kernel_initializer='glorot_uniform', activation='relu'))
ann.add(Dropout(rate=0.5))
ann.add(Dense(units=64, kernel_initializer='glorot_uniform', activation='relu'))
ann.add(Dropout(rate=0.5))
ann.add(Dense(units=1, kernel_initializer='glorot_uniform', activation='sigmoid'))
ann.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
 return ann
batch_size = 56
num_round_per_fold = 2
for no in range(NUM_FOLDS):
print("-"*22, "fold",no)
 bst_loss = 10000.00
for r in range(num_round_per_fold):
print("-"*11,"round",r)
 data_loader_train, data_loader_eval = get_dataloader_for_fold(no,
DATA, data_train, data_eval, batch_size)
 model = get_base_model()
plist = [{"params": model.features.denseblock4.parameters() , "lr":0.0001},
{"params": model.classifier.parameters() }]
optimizer = optim.Adam(plist, lr=0.001, amsgrad=True)
 scheduler = optim.lr_scheduler.MultiStepLR(optimizer, [1,5], gamma=0.1, last_epoch=-1)
 bst_loss = train_model(model, optimizer, scheduler,
data_loader_train, data_loader_eval,
"tmp"+str(no)+".pth", prev_loss=bst_loss,
num_epochs=13, patience=3)
 print("-"*22, "best loss", bst_loss)
 print("")
<load_pretrained>
opt = optimizers.Adam(lr=0.001)
 ann = build_ann(opt)
 history = ann.fit(X_train, Y_train, batch_size=16, epochs=30, validation_data=(X_test, Y_test))
def get_trained_model(no):

 extractor = torchvision.models.densenet161(pretrained=False)
 in_features = extractor.classifier.in_features
extractor.classifier = Classificator(in_features)
 model_path = os.path.join("tmp"+str(no)+".pth")
 extractor.load_state_dict(torch.load(model_path))
extractor = extractor.to(DEVICE)
 extractor.eval()
return extractor<choose_model_class>
ann_prediction = ann.predict(X_test)
 ann_prediction =(ann_prediction > 0.5)
 score = metrics.accuracy_score(Y_test, ann_prediction)
 display_confusion_matrix(Y_test, ann_prediction, score=score )
def get_extractor_model(no):

 extractor = get_trained_model(no)
 extractor.classifier = nn.Identity()
extractor = extractor.to(DEVICE)
 extractor.eval()
return extractor<categorify>
n_folds = 10
cv_score_lg = cross_val_score(estimator=lg, X=X_train, y=Y_train, cv=n_folds, n_jobs=-1)
 cv_score_dt = cross_val_score(estimator=dt, X=X_train, y=Y_train, cv=n_folds, n_jobs=-1)
 cv_score_svm = cross_val_score(estimator=svm, X=X_train, y=Y_train, cv=n_folds, n_jobs=-1)
 cv_score_rf = cross_val_score(estimator=rf, X=X_train, y=Y_train, cv=n_folds, n_jobs=-1)
 cv_score_ann = cross_val_score(estimator=KerasClassifier(build_fn=build_ann, batch_size=16, epochs=20, verbose=0),
X=X_train, y=Y_train, cv=n_folds, n_jobs=-1 )
def get_train_features(data_loader, extractor):

 for bi, d in enumerate(data_loader):
print(".", end="")
 img_tensor = d["image"].to(DEVICE)
 target = d["label"].numpy()
with torch.no_grad() : feature = extractor(img_tensor)
 feature = feature.cpu().detach().squeeze(0 ).numpy()
if bi == 0 :
features = feature
targets = target
else :
features = np.concatenate([features, feature], axis=0)
 targets = np.concatenate([targets, target], axis=0)
 return features, targets<init_hyperparams>
cv_result = {'lg': cv_score_lg, 'dt': cv_score_dt, 'svm': cv_score_svm, 'rf': cv_score_rf, 'ann': cv_score_ann}
cv_data = {model: [score.mean() , score.std() ] for model, score in cv_result.items() }
cv_df = pd.DataFrame(cv_data, index=['Mean_accuracy', 'Variance'])
 cv_df
XGBOOST_PARAM = {
"random_state" : 42,
"n_estimators" : 200,
"objective" : "multi:softmax",
"num_class" : 5,
"eval_metric" : "mlogloss",
}<feature_engineering>
class EsemblingClassifier:
def __init__(self, verbose=True):
self.ann = build_ann(optimizer=optimizers.Adam(lr=0.001))
self.rf = RandomForestClassifier(n_estimators=300, max_depth=11, random_state=42)
 self.svm = SVC(random_state=42)
 self.trained = False
self.verbose = verbose
def fit(self, X, y):
if self.verbose:
print('-------- Fitting models --------')
 self.ann.fit(X, y, epochs=30, batch_size=16, verbose=0)
 self.rf.fit(X, y)
 self.svm.fit(X, y)
 self.trained = True
def predict(self, X):
if self.trained == False:
raise NotFittedError('Please train the classifier before making a prediction')
 if self.verbose:
print('-------- Making and combining predictions --------')
 predictions = list()
pred_ann = self.ann.predict(X)
 pred_ann =(pred_ann > 0.5)*1
pred_rf = self.rf.predict(X)
 pred_svm = self.svm.predict(X)
 for n in range(len(pred_ann)) :
combined = pred_ann[n] + pred_rf[n] + pred_svm[n]
p = 0 if combined == 1 or combined == 0 else 1
predictions.append(p)
 return predictions
batch_size = 64
eval_set = []
for no in range(NUM_FOLDS):
print("-"*22, "fold",no)
 data_loader_train, data_loader_eval = get_dataloader_for_fold(no,
DATA, data_train, data_eval, batch_size)
 extractor = get_extractor_model(no)
 print("...........|.............................................|")
 features_eval, targets_eval = get_train_features(data_loader_eval,
extractor)
 features_train, targets_train = get_train_features(data_loader_train,
extractor)
 print("")
 xgb_model = xgb.XGBClassifier(**XGBOOST_PARAM)
 xgb_model = xgb_model.fit(features_train,targets_train.reshape(-1),
eval_set=[(features_eval, targets_eval.reshape(-1)) ],
early_stopping_rounds=20,
verbose=False)
 print("score",xgb_model.evals_result() ["validation_0"]["mlogloss"][-1])
 pickle.dump(xgb_model, open("xgb_model_"+str(no), "wb"))<categorify>
ens = EsemblingClassifier()
ens.fit(X_train, Y_train)
 ens_prediction = ens.predict(X_test)
 score = metrics.accuracy_score(Y_test, ens_prediction)
 display_confusion_matrix(Y_test, ens_prediction, score=score )
base_transform = transforms.Compose([
transforms.Resize(224),
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 DATA.transform = base_transform<categorify>
test_df_raw = pd.read_csv(path_test)
 test = test_df_raw.copy()
test = preprocess_data(test)
 test = pd.DataFrame(sc.fit_transform(test.values), index=test.index, columns=test.columns)
 test.head()
<choose_model_class><EOS> 
model_test = EsemblingClassifier()
model_test.fit(X, Y)
 prediction = model_test.predict(test)
 result_df = test_df_raw.copy()
result_df['Survived'] = prediction
result_df.to_csv('submission.csv', columns=['PassengerId', 'Survived'], index=False )