Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<prepare_x_and_y>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape<load_pretrained>
df=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv" )
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )<categorify>
df=df.drop(['Cabin'],axis=1)
 test=test.drop(['Cabin'],axis=1)
 df.columns
class FATTestDataset(Dataset):
def __init__(self, fnames, mels, transforms, tta=5):
super().__init__()
self.fnames = fnames
self.mels = mels
self.transforms = transforms
self.tta = tta
def __len__(self):
return len(self.fnames)* self.tta
def __getitem__(self, idx):
new_idx = idx % len(self.fnames)
 image = Image.fromarray(self.mels[new_idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - base_dim)
 image = image.crop([crop, 0, crop + base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 fname = self.fnames[new_idx]
return image, fname<categorify>
print("Number of people embarking in Southampton(S):")
 southampton = df[df["Embarked"] == "S"].shape[0]
print(southampton)
 print("Number of people embarking in Cherbourg(C):")
 cherbourg = df[df["Embarked"] == "C"].shape[0]
print(cherbourg)
 print("Number of people embarking in Queenstown(Q):")
 queenstown = df[df["Embarked"] == "Q"].shape[0]
print(queenstown )
transforms_dict = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
}<define_search_model>
df=df.drop(['Ticket'],axis=1)
 test=test.drop(['Ticket'],axis=1)
 test.columns
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x<choose_model_class>
combine = [df, test]
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 print(pd.crosstab(df['Title'], df['Sex']))
class Classifier(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = torch.mean(x, dim=3)
 x, _ = torch.max(x, dim=2)
 x = self.fc(x)
 return x<choose_model_class>
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()
Classifier(num_classes=num_classes )<split>
df['Age'] = df.groupby(['Title'])['Age'].transform(lambda x: x.fillna(x.mean()))
test['Age'] = test.groupby(['Title'])['Age'].transform(lambda x: x.fillna(x.mean()))
df['Age'] = df['Age'].astype(int)
 test['Age'] = test['Age'].astype(int)
 df.loc[ df['Age'] <= 16, 'Age'] = 0
df.loc[(df['Age'] > 16)&(df['Age'] <= 32), 'Age'] = 1
df.loc[(df['Age'] > 32)&(df['Age'] <= 48), 'Age'] = 2
df.loc[(df['Age'] > 48)&(df['Age'] <= 64), 'Age'] = 3
df.loc[(df['Age'] > 64), 'Age'] = 4
test.loc[ test['Age'] <= 16, 'Age'] = 0
test.loc[(test['Age'] > 16)&(test['Age'] <= 32), 'Age'] = 1
test.loc[(test['Age'] > 32)&(test['Age'] <= 48), 'Age'] = 2
test.loc[(test['Age'] > 48)&(test['Age'] <= 64), 'Age'] = 3
test.loc[(test['Age'] > 64), 'Age'] = 4
df['Age'].head(5 )
def train_model(x_train, y_train, train_transforms):
num_epochs = 118
batch_size = 128
test_batch_size = 256
lr = 1e-3
eta_min = 1e-5
t_max = 5
num_classes = y_train.shape[1]
x_trn, x_val, y_trn, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=SEED)
 train_dataset = FATTrainDataset(x_trn, y_trn, train_transforms)
 valid_dataset = FATTrainDataset(x_val, y_val, train_transforms)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = DataLoader(valid_dataset, batch_size=test_batch_size, shuffle=False)
 model = Classifier(num_classes=num_classes ).cuda()
criterion = nn.BCEWithLogitsLoss().cuda()
optimizer = Adam(params=model.parameters() , lr=lr, amsgrad=False)
 scheduler = CosineAnnealingLR(optimizer, T_max=t_max, eta_min=eta_min)
 best_epoch = -1
best_lwlrap = 0.
 mb = master_bar(range(num_epochs))
for epoch in mb:
start_time = time.time()
model.train()
avg_loss = 0.
 for x_batch, y_batch in train_loader:
preds = model(x_batch.cuda())
 loss = criterion(preds, y_batch.cuda())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds = np.zeros(( len(x_val), num_classes))
avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(valid_loader):
preds = model(x_batch.cuda() ).detach()
loss = criterion(preds, y_batch.cuda())
 preds = torch.sigmoid(preds)
 valid_preds[i * test_batch_size:(i+1)* test_batch_size] = preds.cpu().numpy()
avg_val_loss += loss.item() / len(valid_loader)
 score, weight = calculate_per_class_lwlrap(y_val, valid_preds)
 lwlrap =(score * weight ).sum()
scheduler.step()
if(epoch + 1)% 1 == 0:
elapsed = time.time() - start_time
mb.write(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f} avg_val_loss: {avg_val_loss:.4f} val_lwlrap: {lwlrap:.6f} time: {elapsed:.0f}s')
 if lwlrap > best_lwlrap:
best_epoch = epoch + 1
best_lwlrap = lwlrap
torch.save(model.state_dict() , 'weight_best.pt')
 return {
'best_epoch': best_epoch,
'best_lwlrap': best_lwlrap,
}<train_model>
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 test.head(5 )
result = train_model(x_train, y_train, transforms_dict['train'] )<predict_on_test>
df=df.drop(['Name'],axis=1)
 test=test.drop(['Name'],axis=1)
 df.columns
test_preds = predict_model(test_df['fname'], x_test, transforms_dict['test'], num_classes, tta=35 )<save_to_csv>
df.drop('AgeGroup',axis=1,inplace=True )
test_df[labels] = test_preds.values
test_df.to_csv('submission.csv', index=False)
 test_df.head()<define_search_model>
df = pd.concat([df.drop('Sex', axis=1), pd.get_dummies(df['Sex'])], axis=1)
 test = pd.concat([test.drop('Sex', axis=1), pd.get_dummies(test['Sex'])], axis=1)
 test.head(5 )
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
class Classifier(nn.Module):
def __init__(self, num_classes=1000):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.PReLU() ,
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = self.avgpool(x)
 x = self.fc(x)
 return x<define_variables>
df.drop('Class',axis=1,inplace=True)
 df.head()
DATA = Path('.. /input/freesound-audio-tagging-2019')
 PREPROCESSED = Path('.. /input/fat2019_prep_mels1')
 WORK = Path('work')
 Path(WORK ).mkdir(exist_ok=True, parents=True)
 CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = DATA/'train_noisy.csv'
CSV_TRN_NOISY_BEST50S = PREPROCESSED/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
MELS_TRN_CURATED = PREPROCESSED/'mels_train_curated.pkl'
MELS_TRN_NOISY = PREPROCESSED/'mels_train_noisy.pkl'
MELS_TRN_NOISY_BEST50S = PREPROCESSED/'mels_trn_noisy_best50s.pkl'
MELS_TEST = PREPROCESSED/'mels_test.pkl'
trn_curated_df = pd.read_csv(CSV_TRN_CURATED)
 trn_noisy_df = pd.read_csv(CSV_TRN_NOISY)
 trn_noisy50s_df = pd.read_csv(CSV_TRN_NOISY_BEST50S)
 test_df = pd.read_csv(CSV_SUBMISSION)
 df = pd.concat([trn_curated_df, trn_noisy50s_df], ignore_index=True, sort=True)
 test_df = pd.read_csv(CSV_SUBMISSION)
 X_train = pickle.load(open(MELS_TRN_CURATED, 'rb')) + pickle.load(open(MELS_TRN_NOISY_BEST50S, 'rb'))<define_variables>
df['Embarked'].replace({'S':1,'C':2,'Q':3},inplace=True)
 df['Embarked']=df['Embarked'].fillna(1)
 test['Embarked'].replace({'S':1,'C':2,'Q':3},inplace=True)
 test['Embarked']=test['Embarked'].fillna(1)
 test.head(5 )
data.show_batch(3 )<train_model>
df=df.drop(['Title'],axis=1)
 test=test.drop(['Title'],axis=1)
 test.columns
learn.fit_one_cycle(10, slice(1e-6, 1e-1))<train_model>
predictors=df.drop(['Survived','PassengerId'],axis=1)
 target=df['Survived']
x_train,x_cv,y_train,y_cv=train_test_split(predictors,target,test_size=0.35,random_state=0 )
learn.fit_one_cycle(100, 3e-3 )<save_model>
knn = KNeighborsClassifier()
knn.fit(x_train, y_train)
 y_pred = knn.predict(x_cv)
 acc_knn = round(accuracy_score(y_pred,y_cv)* 100, 2)
 print(acc_knn )
learn.save('fat2019_fastai_cnn2d_stage-2')
 learn.export()<load_from_csv>
logreg = LogisticRegression()
logreg.fit(x_train, y_train)
 y_pred = logreg.predict(x_cv)
 acc_logreg = round(accuracy_score(y_pred, y_cv)* 100, 2)
 print(acc_logreg )
del X_train
X_test = pickle.load(open(MELS_TEST, 'rb'))
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK, Path('.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK, test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test )<save_to_csv>
randomforest = RandomForestClassifier()
randomforest.fit(x_train, y_train)
 y_pred = randomforest.predict(x_cv)
 acc_randomforest = round(accuracy_score(y_pred, y_cv)* 100, 2)
 print(acc_randomforest )
test_df[learn.data.classes] = preds
test_df.to_csv('submission.csv', index=False)
 test_df.head()<load_pretrained>
svc = SVC()
svc.fit(x_train, y_train)
 y_pred = svc.predict(x_cv)
 acc_svc = round(accuracy_score(y_pred, y_cv)* 100, 2)
 print(acc_svc )
del X_test
X_train = pickle.load(open(MELS_TRN_CURATED, 'rb'))
CUR_X_FILES, CUR_X = list(df.fname.values), X_train
learn = cnn_learner(data, borrowed_model, pretrained=False, metrics=[f_score])
 learn.load('fat2019_fastai_cnn2d_stage-2');<load_from_csv>
models = pd.DataFrame({
'Method': ['KNN', 'Logistic Regression',
'Random Forest', 'Support Vector Machine'],
'Score': [acc_knn, acc_logreg,
acc_randomforest, acc_svc]})
 models.sort_values(by='Score', ascending=False )
CSV_TRN_CURATED = DATA/'train_curated.csv'
CSV_TRN_NOISY = PREPROCESSED/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
MELS_TRN_CURATED = PREPROCESSED/'mels_train_curated.pkl'
MELS_TRN_NOISY = '.. /input/fat2019_prep_mels1/mels_trn_noisy_best50s.pkl'
MELS_TEST = PREPROCESSED/'mels_test.pkl'
trn_curated_df = pd.read_csv(CSV_TRN_CURATED)
 trn_noisy_df = pd.read_csv(CSV_TRN_NOISY)
 test_df = pd.read_csv(CSV_SUBMISSION)
 df = pd.concat([trn_curated_df, trn_noisy_df], ignore_index=True)
 test_df = pd.read_csv(CSV_SUBMISSION)
 X_train = pickle.load(open(MELS_TRN_CURATED, 'rb'))
with open(".. /input/fat2019_prep_mels1/mels_trn_noisy_best50s.pkl", 'rb')as f:
X_train.append(pickle.load(f))
X_test = pickle.load(open(MELS_TEST, 'rb'))<define_variables>
svc = MLPClassifier()
svc.fit(x_train, y_train)
 y_pred = svc.predict(x_cv)
 acc_svc = round(accuracy_score(y_pred, y_cv)* 100, 2)
 print(acc_svc)
list_model = [ models.vgg16_bn, models.vgg19_bn]
predicts = None
i = 0<train_on_grid>
svc = NearestCentroid()
svc.fit(x_train, y_train)
 y_pred = svc.predict(x_cv)
 acc_svc = round(accuracy_score(y_pred, y_cv)* 100, 2)
 print(acc_svc)
print(list_model[i])
 CUR_X_FILES, CUR_X = list(df.fname.values), X_train
f_score = partial(fbeta, thresh=0.2)
 learn = cnn_learner(data, list_model[i], pretrained=False, metrics=[f_score])
 learn.unfreeze()
learn.lr_find()
learn.fit_one_cycle(5, slice(1e-6, 1e-1))
learn.lr_find()
learn.fit_one_cycle(100, slice(1e-6, 1e-2))
learn.export()
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK, Path('.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK, test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 if predicts is None:
predicts = preds
else:
predicts += preds
i+=1<train_on_grid>
svc = SVC()
svc.fit(x_train, y_train)
 y_pred = svc.predict(test.drop('PassengerId',axis=1))
print(y_pred )
print(list_model[i])
 CUR_X_FILES, CUR_X = list(df.fname.values), X_train
f_score = partial(fbeta, thresh=0.2)
 learn = cnn_learner(data, list_model[i], pretrained=False, metrics=[f_score])
 learn.unfreeze()
learn.lr_find()
learn.fit_one_cycle(5, slice(1e-6, 1e-1))
learn.lr_find()
learn.fit_one_cycle(100, slice(1e-6, 1e-2))
learn.export()
CUR_X_FILES, CUR_X = list(test_df.fname.values), X_test
test = ImageList.from_csv(WORK, Path('.. ')/CSV_SUBMISSION, folder='test')
 learn = load_learner(WORK, test=test)
 preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 if predicts is None:
predicts = preds
else:
predicts += preds
i+=1<feature_engineering>
submission = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':y_pred})
 submission.head(5 )
<save_to_csv><EOS> 
filename = 'TPredictions.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )