Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<categorify>
warnings.filterwarnings('ignore' )
class Alaska2Dataset(Dataset):
def __init__(self, df, augmentations=None, test = False):
self.data = df
self.augment = augmentations
self.test = test
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if self.test:
fn = self.data.loc[idx][0]
else:
fn, label = self.data.loc[idx]
im = imread(fn)
 if self.augment:
im = self.augment(image=im)
 if self.test:
item = {'features': im['image']}
else:
item = {'features': im['image'], 'targets':label, 'bool_targets': get_one_hot(label, 10)}
return item
AUGMENTATIONS_TRAIN = Compose([
VerticalFlip(p=0.5),
HorizontalFlip(p=0.5),
ToFloat(max_value=255),
ToTensorV2()
], p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=255),
ToTensorV2()
], p=1 )<create_dataframe>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 dataset = pd.concat([train, test], ignore_index = True)
 PassengerId = test['PassengerId']
batch_size = 24
num_workers = 8
train_dataset = Alaska2Dataset(train_df, augmentations=AUGMENTATIONS_TRAIN)
 valid_dataset = Alaska2Dataset(val_df.sample(5000 ).reset_index(drop=True), augmentations=AUGMENTATIONS_TEST)
 train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=batch_size*2,
num_workers=num_workers,
shuffle=False )<feature_engineering>
dataset = dataset.fillna(np.nan)
 dataset.isnull().sum()
class Net(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.model = EfficientNet.from_name('efficientnet-b0')
 self.dense_output = nn.Linear(1280, num_classes)
 def forward(self, x):
feat = self.model.extract_features(x)
 feat = F.avg_pool2d(feat, feat.size() [2:] ).reshape(-1, 1280)
 return self.dense_output(feat )<choose_model_class>
dataset['Title'] = dataset['Name'].apply(lambda x:x.split(',')[1].split('.')[0].strip())
 Title_Dict = {}
Title_Dict.update(dict.fromkeys(['Capt', 'Col', 'Major', 'Dr', 'Rev'], 'Officer'))
Title_Dict.update(dict.fromkeys(['Don', 'Sir', 'the Countess', 'Dona', 'Lady'], 'Royalty'))
Title_Dict.update(dict.fromkeys(['Mme', 'Ms', 'Mrs'], 'Mrs'))
Title_Dict.update(dict.fromkeys(['Mlle', 'Miss'], 'Miss'))
Title_Dict.update(dict.fromkeys(['Mr'], 'Mr'))
Title_Dict.update(dict.fromkeys(['Master','Jonkheer'], 'Master'))
dataset['Title'] = dataset['Title'].map(Title_Dict)
 sns.barplot(x="Title", y="Survived", data=dataset, palette='Set3' )
loaders = {
"train": train_loader,
"valid": valid_loader
}
model = Net(num_classes=len(class_labels))
model.load_state_dict(torch.load('.. /input/alaska2trainvalsplit/epoch_5_val_loss_3.75_auc_0.833.pth'))
optimizer = torch.optim.AdamW(model.parameters() , lr=0.0003)
 criterion = torch.nn.CrossEntropyLoss()
callbacks = [
AccuracyCallback() ,
AUCCallback(input_key='bool_targets', num_classes = 1)
 ]
runner = SupervisedRunner()<train_model>
age = dataset[['Age','Pclass','Sex','Title']]
age = pd.get_dummies(age)
 known_age = age[age.Age.notnull() ].values
null_age = age[age.Age.isnull() ].values
x = known_age[:, 1:]
y = known_age[:, 0]
rf = RandomForestRegressor(n_jobs=-1)
 rf.fit(x, y)
 predictedAge = rf.predict(null_age[:, 1:])
 dataset.loc[(dataset.Age.isnull()),'Age'] = predictedAge
runner.train(
 model=model,
criterion=criterion,
optimizer=optimizer,
loaders=loaders,
num_epochs=5,
verbose=True,
callbacks=callbacks,
logdir="logs",
main_metric="auc/class_0",
minimize_metric = False,
)<create_dataframe>
dataset[dataset['Embarked'].isnull() ]
test_filenames = sorted(glob(f"{data_dir}/Test/*.jpg"))
test_df = pd.DataFrame({'ImageFileName': list(
 test_filenames)}, columns=['ImageFileName'])
 batch_size = 16
num_workers = 4
test_dataset = Alaska2Dataset(test_df, augmentations=AUGMENTATIONS_TEST, test=True)
 test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=False,
drop_last=False )<save_to_csv>
C = dataset[(dataset['Embarked']=='C')&(dataset['Pclass'] == 1)]['Fare'].median()
print(C)
 S = dataset[(dataset['Embarked']=='S')&(dataset['Pclass'] == 1)]['Fare'].median()
print(S)
 Q = dataset[(dataset['Embarked']=='S')&(dataset['Pclass'] == 1)]['Fare'].median()
print(Q)
 dataset['Embarked'] = dataset['Embarked'].fillna('C' )
model.load_state_dict(torch.load('logs/checkpoints/best.pth')["model_state_dict"])
 model.cuda()
preds = []
for outputs in tqdm(runner.predict_loader(loader=test_loader, model=model)) :
preds.append(softmax(outputs))
preds = np.array(preds)
 test_df['Id'] = test_df['ImageFileName'].apply(lambda x: x.split(os.sep)[-1])
 test_df['Label'] = 1-preds[:, 0]
test_df = test_df.drop('ImageFileName', axis=1)
 test_df.to_csv('submission.csv', index=False)
 print(test_df.head() )<import_modules>
dataset[dataset['Fare'].isnull() ]
!pip install -q efficientnet_pytorch
Compose, HorizontalFlip, CLAHE, HueSaturationValue,
RandomBrightness, RandomContrast, RandomGamma, OneOf, Resize,
ToFloat, ShiftScaleRotate, GridDistortion, RandomRotate90, Cutout,
RGBShift, RandomBrightness, RandomContrast, Blur, MotionBlur, MedianBlur, GaussNoise, CoarseDropout,
IAAAdditiveGaussianNoise, GaussNoise, OpticalDistortion, RandomSizedCrop, VerticalFlip 
)
<set_options>
fare=dataset[(dataset['Embarked'] == "S")&(dataset['Pclass'] == 3)].Fare.median()
dataset['Fare']=dataset['Fare'].fillna(fare )
seed = 42
print(f'setting everything to seed {seed}')
 random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
dataset['Surname']=dataset['Name'].apply(lambda x:x.split(',')[0].strip())
 Surname_Count = dict(dataset['Surname'].value_counts())
 dataset['FamilyGroup'] = dataset['Surname'].apply(lambda x:Surname_Count[x])
 Female_Child_Group=dataset.loc[(dataset['FamilyGroup']>=2)&(( dataset['Age']<=12)|(dataset['Sex']=='female')) ]
Male_Adult_Group=dataset.loc[(dataset['FamilyGroup']>=2)&(dataset['Age']>12)&(dataset['Sex']=='male')]
data_dir = '.. /input/alaska2-image-steganalysis'
folder_names = ['JMiPOD/', 'JUNIWARD/', 'UERD/']
class_names = ['Normal', 'JMiPOD_75', 'JMiPOD_90', 'JMiPOD_95',
'JUNIWARD_75', 'JUNIWARD_90', 'JUNIWARD_95',
'UERD_75', 'UERD_90', 'UERD_95']
class_labels = { name: i for i, name in enumerate(class_names)}<load_from_csv>
Female_Child=pd.DataFrame(Female_Child_Group.groupby('Surname')['Survived'].mean().value_counts())
 Female_Child.columns=['GroupCount']
Female_Child
train_df = pd.read_csv('.. /input/alaska2trainvalsplit/alaska2_train_df.csv')
 val_df = pd.read_csv('.. /input/alaska2trainvalsplit/alaska2_val_df.csv')
 print(train_df.sample(10))
train_df.Label.hist()
plt.title('Distribution of Classes' )<normalization>
Male_Adult=pd.DataFrame(Male_Adult_Group.groupby('Surname')['Survived'].mean().value_counts())
 Male_Adult.columns=['GroupCount']
Male_Adult
class Alaska2Dataset(Dataset):
def __init__(self, df, augmentations=None):
self.data = df
self.augment = augmentations
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
fn, label = self.data.loc[idx]
im = cv2.imread(fn)[:, :, ::-1]
if self.augment:
im = self.augment(image=im)
 return im, label
img_size = 512
AUGMENTATIONS_TRAIN = Compose([
VerticalFlip(p=0.5),
HorizontalFlip(p=0.5),
RandomRotate90(p=0.5),
OneOf([
Cutout(max_h_size=50,max_w_size=50),
CoarseDropout(max_height=50, max_width=50)], p=0.5),
ToFloat(max_value=255),
ToTensor()
], p=1)
 AUGMENTATIONS_TEST = Compose([
ToFloat(max_value=255),
ToTensor()
], p=1)
<create_dataframe>
Female_Child_Group=Female_Child_Group.groupby('Surname')['Survived'].mean()
Dead_List=set(Female_Child_Group[Female_Child_Group.apply(lambda x:x==0)].index)
 print(Dead_List)
 Male_Adult_List=Male_Adult_Group.groupby('Surname')['Survived'].mean()
Survived_List=set(Male_Adult_List[Male_Adult_List.apply(lambda x:x==1)].index)
 print(Survived_List )
temp_df = train_df.sample(64 ).reset_index(drop=True)
 train_dataset = Alaska2Dataset(temp_df, augmentations=AUGMENTATIONS_TEST)
 batch_size = 64
num_workers = 0
temp_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers, shuffle=False)
 images, labels = next(iter(temp_loader))
images = images['image'].permute(0, 2, 3, 1)
 max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width,
figsize=(grid_width+1, grid_height+2))
for i,(im, label)in enumerate(zip(images, labels)) :
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im.squeeze())
 ax.set_title(str(label.item()))
ax.axis('off')
 plt.suptitle("0: Cover, 1: JMiPOD_75, 2: JMiPOD_90, 3: JMiPOD_95, 4: JUNIWARD_75, 5:JUNIWARD_90,
6: JUNIWARD_95, 7:UERD_75, 8:UERD_90, 9:UERD_95")
 plt.show()
del images
gc.collect()<create_dataframe>
train=dataset.loc[dataset['Survived'].notnull() ]
test=dataset.loc[dataset['Survived'].isnull() ]
test.loc[(test['Surname'].apply(lambda x:x in Dead_List)) ,'Sex'] = 'male'
test.loc[(test['Surname'].apply(lambda x:x in Dead_List)) ,'Age'] = 60
test.loc[(test['Surname'].apply(lambda x:x in Dead_List)) ,'Title'] = 'Mr'
test.loc[(test['Surname'].apply(lambda x:x in Survived_List)) ,'Sex'] = 'female'
test.loc[(test['Surname'].apply(lambda x:x in Survived_List)) ,'Age'] = 5
test.loc[(test['Surname'].apply(lambda x:x in Survived_List)) ,'Title'] = 'Miss'
train_dataset = Alaska2Dataset(temp_df, augmentations=AUGMENTATIONS_TRAIN)
 batch_size = 64
num_workers = 0
temp_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers, shuffle=False)
 images, labels = next(iter(temp_loader))
images = images['image'].permute(0, 2, 3, 1)
 max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width,
figsize=(grid_width+1, grid_height+2))
for i,(im, label)in enumerate(zip(images, labels)) :
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im.squeeze())
 ax.set_title(str(label.item()))
ax.axis('off')
 plt.suptitle("0: Cover, 1: JMiPOD_75, 2: JMiPOD_90, 3: JMiPOD_95, 4: JUNIWARD_75, 5:JUNIWARD_90,
6: JUNIWARD_95, 7:UERD_75, 8:UERD_90, 9:UERD_95")
 plt.show()
del images, temp_df
gc.collect()<categorify>
dataset = pd.concat([train, test])
 dataset=dataset[['Survived','Pclass','Sex','Age','Fare','Embarked','Title','FamilyLabel','Deck','TicketGroup']]
dataset=pd.get_dummies(dataset)
 trainset=dataset[dataset['Survived'].notnull() ]
testset=dataset[dataset['Survived'].isnull() ].drop('Survived',axis=1)
 X = trainset.values[:,1:]
Y = trainset.values[:,0]
class Net(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.model = EfficientNet.from_pretrained('efficientnet-b0')
 self.dense_output = nn.Linear(1280, num_classes)
 def forward(self, x):
feat = self.model.extract_features(x)
 feat = F.avg_pool2d(feat, feat.size() [2:] ).reshape(-1, 1280)
 return self.dense_output(feat )<create_dataframe>
pipe=Pipeline([('select',SelectKBest(k=20)) , 
('classify', RandomForestClassifier(random_state = 10, max_features = 'sqrt')) ])
 param_test = {'classify__n_estimators':list(range(20,50,2)) ,
'classify__max_depth':list(range(3,60,3)) }
gsearch = GridSearchCV(estimator = pipe, param_grid = param_test, scoring='accuracy', cv=10)
 gsearch.fit(X,Y)
 print(gsearch.best_params_, gsearch.best_score_ )
batch_size = 8
num_workers = 8
train_dataset = Alaska2Dataset(train_df, augmentations=AUGMENTATIONS_TRAIN)
 valid_dataset = Alaska2Dataset(val_df.sample(1000 ).reset_index(drop=True), augmentations=AUGMENTATIONS_TEST)
 train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=batch_size*2,
num_workers=num_workers,
shuffle=False)
 device = 'cuda'
model = Net(num_classes=len(class_labels)).to(device)
 model.load_state_dict(torch.load('.. /input/alaska2trainvalsplit/val_loss_6.08_auc_0.875.pth'))
optimizer = torch.optim.AdamW(model.parameters() , lr=1e-6 )<compute_train_metric>
select = SelectKBest(k = 20)
 clf = RandomForestClassifier(random_state = 10, warm_start = True,
n_estimators = 30,
max_depth = 6,
max_features = 'sqrt')
 pipeline = make_pipeline(select, clf)
 pipeline.fit(X, Y )
def alaska_weighted_auc(y_true, y_valid):
tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization<train_on_grid>
cv_score = model_selection.cross_val_score(pipeline, X, Y, cv= 10)
 print("CV Score : Mean - %.7g | Std - %.7g " %(np.mean(cv_score), np.std(cv_score)) )
<create_dataframe><EOS> 
predictions = pipeline.predict(testset)
 submission = pd.DataFrame({"PassengerId": PassengerId, "Survived": predictions.astype(np.int32)})
 submission.to_csv("submission.csv", index=False )