Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<install_modules>
warnings.filterwarnings("ignore")
 %matplotlib inline
!pip install -q efficientnet_pytorch > /dev/null<set_options>
def drop_col_not_req(df, cols):
df.drop(cols, axis = 1, inplace = True )
warnings.filterwarnings('ignore')
 SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<define_variables>
def pclass_fare_category(df, Pclass_1_mean_fare, Pclass_2_mean_fare, Pclass_3_mean_fare):
if(df['Pclass'] == 1):
if(df['Fare'] <= Pclass_1_mean_fare):
return 'Pclass_1_Low_Fare'
else:
return 'Pclass_1_High_Fare'
elif(df['Pclass'] == 2):
if(df['Fare'] <= Pclass_2_mean_fare):
return 'Pclass_2_Low_Fare'
else:
return 'Pclass_2_High_Fare'
elif(df['Pclass'] == 3):
if(df['Fare'] <= Pclass_3_mean_fare):
return 'Pclass_3_Low_Fare'
else:
return 'Pclass_3_High_Fare'
%%time
dataset = []
for label, kind in enumerate(['Cover', 'JMiPOD', 'JUNIWARD', 'UERD']):
for path in glob('.. /input/alaska2-image-steganalysis/Cover/*.jpg'):
dataset.append({
'kind': kind,
'image_name': path.split('/')[-1],
'label': label
} )<split>
def fill_missing_age(missing_age_train, missing_age_test):
missing_age_X_train = missing_age_train.drop(['Age'], axis = 1)
 missing_age_y_train = missing_age_train['Age']
missing_age_X_test = missing_age_test.drop(['Age'], axis = 1)
 gbm_reg = ensemble.GradientBoostingRegressor(random_state = 42)
 gbm_reg_param_grid = {'n_estimators': [2000], 'max_depth': [3], 'learning_rate': [0.01], 'max_features': [3]}
gbm_reg_grid = model_selection.GridSearchCV(gbm_reg, gbm_reg_param_grid, cv = 5, n_jobs = 25, verbose = 1, scoring = 'neg_mean_squared_error')
 gbm_reg_grid.fit(missing_age_X_train, missing_age_y_train)
 print("Age feature Best GB Params: " + str(gbm_reg_grid.best_params_))
print("Age feature Best GB Score: " + str(gbm_reg_grid.best_score_))
print("GB Train Error for 'Age' Feature Regressor: " + str(gbm_reg_grid.score(missing_age_X_train, missing_age_y_train)))
 missing_age_test['Age_GB'] = gbm_reg_grid.predict(missing_age_X_test)
 print(missing_age_test['Age_GB'][:4])
 lrf_reg = LinearRegression()
lrf_reg_param_grid = {'fit_intercept': [True], 'normalize': [True]}
lrf_reg_grid = model_selection.GridSearchCV(lrf_reg, lrf_reg_param_grid, cv = 5, n_jobs = 25, verbose = 1, scoring = 'neg_mean_squared_error')
 lrf_reg_grid.fit(missing_age_X_train, missing_age_y_train)
 print("Age feature Best LR Params: " + str(lrf_reg_grid.best_params_))
print("Age feature Best LR Score: " + str(lrf_reg_grid.best_score_))
print("LR Train Error for 'Age' Feature Regressor: " + str(lrf_reg_grid.score(missing_age_X_train, missing_age_y_train)))
 missing_age_test['Age_LRF'] = lrf_reg_grid.predict(missing_age_X_test)
 print(missing_age_test['Age_LRF'][:4])
 missing_age_test['Age'] = missing_age_test[['Age_GB', 'Age_LRF']].mean(axis = 1)
 print(missing_age_test['Age'][:4])
 drop_col_not_req(missing_age_test, ['Age_GB', 'Age_LRF'])
 return missing_age_test
dataset_small = dataset[0:5000]+dataset[75000:80000]+dataset[150000:155000]+dataset[225000:223000]
random.shuffle(dataset_small)
 dataset = pd.DataFrame(dataset_small)
 gkf = GroupKFold(n_splits=5)
 dataset.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(gkf.split(X=dataset.index, y=dataset['label'], groups=dataset['image_name'])) :
dataset.loc[dataset.iloc[val_index].index, 'fold'] = fold_number<normalization>
def get_top_n_features(titanic_train_data_X, titanic_train_data_y, top_n_features):
rf_est = RandomForestClassifier(random_state = 42)
 rf_param_grid = {'n_estimators' : [500], 'min_samples_split':[2, 3], 'max_depth':[20]}
rf_grid = model_selection.GridSearchCV(rf_est, rf_param_grid, n_jobs = 25, cv = 10, verbose = 1)
 rf_grid.fit(titanic_train_data_X, titanic_train_data_y)
 print("Top N Features Best RF Params: " + str(rf_grid.best_params_))
print("Top N Features Best RF Score: " + str(rf_grid.best_score_))
print("Top N Features RF Train Error: " + str(rf_grid.score(titanic_train_data_X, titanic_train_data_y)))
 feature_imp_sorted_rf = pd.DataFrame({'feature': list(titanic_train_data_X), 'importance': rf_grid.best_estimator_.feature_importances_} ).sort_values('importance', ascending = False)
 features_top_n_rf = feature_imp_sorted_rf.head(top_n_features)['feature']
print("Sample 25 Features from RF Classifier:")
 print(str(features_top_n_rf[:25]))
ada_est = ensemble.AdaBoostClassifier(random_state = 42)
 ada_param_grid = {'n_estimators' : [500], 'learning_rate': [0.5, 0.6]}
ada_grid = model_selection.GridSearchCV(ada_est, ada_param_grid, n_jobs = 25, cv = 10, verbose = 1)
 ada_grid.fit(titanic_train_data_X, titanic_train_data_y)
 print("Top N Features Best Ada Params: " + str(ada_grid.best_params_))
print("Top N Features Best Ada Score: " + str(ada_grid.best_score_))
print("Top N Features Ada Train Error: " + str(ada_grid.score(titanic_train_data_X, titanic_train_data_y)))
 feature_imp_sorted_ada = pd.DataFrame({'feature': list(titanic_train_data_X), 'importance': ada_grid.best_estimator_.feature_importances_} ).sort_values('importance', ascending = False)
 features_top_n_ada = feature_imp_sorted_ada.head(top_n_features)['feature']
print("Sample 25 Features from Ada Classifier:")
 print(str(features_top_n_ada[:25]))
et_est = ensemble.ExtraTreesClassifier(random_state = 42)
 et_param_grid = {'n_estimators' : [500], 'min_samples_split':[3, 4], 'max_depth':[15]}
et_grid = model_selection.GridSearchCV(et_est, et_param_grid, n_jobs = 25, cv = 10, verbose = 1)
 et_grid.fit(titanic_train_data_X, titanic_train_data_y)
 print("Top N Features Best ET Params: " + str(et_grid.best_params_))
print("Top N Features Best ET Score: " + str(et_grid.best_score_))
print("Top N Features ET Train Error: " + str(et_grid.score(titanic_train_data_X, titanic_train_data_y)))
 feature_imp_sorted_et = pd.DataFrame({'feature': list(titanic_train_data_X), 'importance': et_grid.best_estimator_.feature_importances_} ).sort_values('importance', ascending = False)
 features_top_n_et = feature_imp_sorted_et.head(top_n_features)['feature']
print("Sample 25 Features from ET Classifier:")
 print(str(features_top_n_et[:25]))
features_top_n = pd.concat([features_top_n_rf, features_top_n_ada, features_top_n_et], ignore_index = True ).drop_duplicates()
return features_top_n
def get_train_transforms() :
return A.Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
def Stacking_Ensemble(models, X_train, y_train, X_test, n_folds):
X_train = np.array(X_train)
 y_train = np.array(y_train)
 X_test = np.array(X_test)
 Stacking_train = np.zeros(( X_train.shape[0],(len(models)* 2)))
 Stacking_test = np.zeros(( X_test.shape[0],(len(models)* 2)))
 Strat_KFold = model_selection.StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = 42)
 skf = list(Strat_KFold.split(X_train, y_train))
for model_counter, model in enumerate(models):
print("Model_Counter" + str(model_counter))
S_test_temp_p1 = np.zeros(( titanic_test_data_X.shape[0], n_folds))
S_test_temp_p2 = np.zeros(( titanic_test_data_X.shape[0], n_folds))
for fold_counter,(tr_index, te_index)in enumerate(skf):
print("Fold : " + str(fold_counter))
X_tr = X_train[tr_index]
y_tr = y_train[tr_index]
X_te = X_train[te_index]
y_te = y_train[te_index]
model.fit(X_tr, y_tr)
 print("----------------------")
 print("Model : " + str(model_counter)+ "Fold : " + str(fold_counter)+ "Score is - " + str(model.score(X_tr, y_tr)))
 print("----------------------")
 tr_pred = model.predict_proba(X_te)
 Stacking_train[te_index, model_counter * 2] = tr_pred[:,0]
Stacking_train[te_index,(model_counter * 2)+ 1] = tr_pred[:,1]
te_pred = model.predict_proba(X_test)
 S_test_temp_p1[:, fold_counter] = te_pred[:,0]
S_test_temp_p2[:, fold_counter] = te_pred[:,1]
Stacking_test[:, model_counter * 2] = np.mean(S_test_temp_p1, axis = 1)
 Stacking_test[:,(model_counter * 2)+ 1] = np.mean(S_test_temp_p2, axis = 1)
 return(Stacking_train, Stacking_test)
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, kinds, image_names, labels, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
target = onehot(4, label)
 return image, target
def __len__(self)-> int:
return self.image_names.shape[0]
def get_labels(self):
return list(self.labels )<define_variables>
train_data_orig = pd.read_csv('.. /input/train.csv')
 test_data_orig = pd.read_csv('.. /input/test.csv' )
fold_number = 0
train_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] != fold_number].kind.values,
image_names=dataset[dataset['fold'] != fold_number].image_name.values,
labels=dataset[dataset['fold'] != fold_number].label.values,
transforms=get_train_transforms() , 
)
 validation_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] == fold_number].kind.values,
image_names=dataset[dataset['fold'] == fold_number].image_name.values,
labels=dataset[dataset['fold'] == fold_number].label.values,
transforms=get_valid_transforms() ,
)<compute_train_metric>
print(combined_train_test.groupby(['Survived', 'Embarked'])['Survived'].count())
 print(combined_train_test['PassengerId'].groupby(by = combined_train_test['Embarked'] ).count().sort_values(ascending = False))
print(combined_train_test['Fare'].groupby(by = combined_train_test['Embarked'] ).mean().sort_values(ascending = False))
if(combined_train_test['Embarked'].isnull().sum() != 0):
combined_train_test['Embarked'].fillna(combined_train_test['Embarked'].mode().iloc[0], inplace=True)
 combined_train_test.info()
emb_dummies_df = pd.get_dummies(combined_train_test['Embarked'],
prefix = combined_train_test[['Embarked']].columns[0])
 combined_train_test = pd.concat([combined_train_test, emb_dummies_df], axis = 1 )
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def alaska_weighted_auc(y_true, y_valid):

 tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1 ).clip(min=0, max=1 ).astype(int)
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = alaska_weighted_auc(self.y_true, self.y_pred)
 @property
def avg(self):
return self.score<normalization>
print(combined_train_test['Sex'].groupby(by = combined_train_test['Sex'] ).count().sort_values(ascending = False))
print(combined_train_test.groupby(['Survived', 'Sex'])['Survived'].count())

sex_dummies_df = pd.get_dummies(combined_train_test['Sex'],
prefix = combined_train_test[['Sex']].columns[0])
 combined_train_test = pd.concat([combined_train_test, sex_dummies_df], axis = 1 )
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.05):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )<init_hyperparams>
title_Dict = {}
title_Dict.update(dict.fromkeys(["Capt", "Col", "Major", "Dr", "Rev"], "Officer"))
title_Dict.update(dict.fromkeys(["Jonkheer", "Don", "Sir", "the Countess", "Dona", "Lady"], "Royalty"))
title_Dict.update(dict.fromkeys(["Mme", "Ms", "Mrs"], "Mrs"))
title_Dict.update(dict.fromkeys(["Mlle", "Miss"], "Miss"))
title_Dict.update(dict.fromkeys(["Mr", "Ms"], "Mr"))
title_Dict.update(dict.fromkeys(["Master"], "Master"))
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = './'
self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = LabelSmoothing().to(self.device)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss, final_scores = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss, final_scores = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
targets = targets.to(self.device ).float()
batch_size = images.shape[0]
images = images.to(self.device ).float()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 targets = targets.to(self.device ).float()
images = images.to(self.device ).float()
batch_size = images.shape[0]
self.optimizer.zero_grad()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 loss.backward()
final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss, final_scores
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<choose_model_class>
combined_train_test['Title'] = combined_train_test['Title'].map(title_Dict)
 print(combined_train_test['Title'].groupby(by = combined_train_test['Title'] ).count().sort_values(ascending = False))
title_dummies_df = pd.get_dummies(combined_train_test['Title'],
prefix = combined_train_test[['Title']].columns[0])
 combined_train_test = pd.concat([combined_train_test, title_dummies_df], axis = 1 )
def get_net() :
net = EfficientNet.from_pretrained('efficientnet-b2')
 net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)
 return net
net = get_net().cuda()<train_model>
combined_train_test['Name_Length'] = combined_train_test['Name'].str.len()
print(combined_train_test['Name_Length'].groupby(by = combined_train_test['Name_Length'] ).count().sort_values(ascending = False)[:5] )
class TrainGlobalConfig:
num_workers = 4
batch_size = 16
n_epochs = 25
lr = 0.001
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)<load_pretrained>
combined_train_test['Name_Length_Category'] = combined_train_test['Name_Length'].map(name_len_category)
 print(combined_train_test['Name_Length_Category'].groupby(by = combined_train_test['Name_Length_Category'] ).count().sort_values(ascending = False))
le_fare = LabelEncoder()
le_fare.fit(np.array(['Very_Short_Name', 'Short_Name', 'Medium_Name', 'Long_Name', 'Very_High_Fare']))
combined_train_test['Name_Length_Category'] = le_fare.transform(combined_train_test['Name_Length_Category'])
 print(combined_train_test[['Name_Length_Category', 'Survived']].corr())
 first_name_dummies_df = pd.get_dummies(combined_train_test['Name_Length_Category'],
prefix = combined_train_test[['Name_Length_Category']].columns[0])
 combined_train_test = pd.concat([combined_train_test, first_name_dummies_df], axis = 1 )
def run_training() :
device = torch.device('cuda:0')
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
batch_size=TrainGlobalConfig.batch_size,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )<train_model>
combined_train_test['First_Name'] = combined_train_test['Name'].str.extract('^ (.+?),' ).str.strip()
print(combined_train_test['First_Name'].groupby(by = combined_train_test['First_Name'] ).count().sort_values(ascending = False)[:5])
 first_name_dummies_df = pd.get_dummies(combined_train_test['First_Name'],
prefix = combined_train_test[['First_Name']].columns[0])
 combined_train_test = pd.concat([combined_train_test, first_name_dummies_df], axis = 1 )
run_training()<install_modules>
combined_train_test['Last_Name'] = combined_train_test['Name'].str.split("\." ).str[1].str.strip()
combined_train_test['Last_Name'] = combined_train_test['Last_Name'].str.strip("\([^)]*\)")
 combined_train_test['Last_Name'].fillna(combined_train_test['Name'].str.split("\." ).str[1].str.strip())
 print(combined_train_test['Last_Name'].groupby(by = combined_train_test['Last_Name'] ).count().sort_values(ascending = False)[:5])
 last_name_dummies_df = pd.get_dummies(combined_train_test['Last_Name'],
prefix = combined_train_test[['Last_Name']].columns[0])
 combined_train_test = pd.concat([combined_train_test, last_name_dummies_df], axis = 1 )
!pip install -q efficientnet_pytorch > /dev/null<set_options>
combined_train_test['Original_Name'] = combined_train_test['Name'].str.split("\((.*?)\)" ).str[1].str.strip (
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<normalization>
if(combined_train_test['Fare'].isnull().sum() != 0):
combined_train_test['Fare'] = combined_train_test[['Fare']].fillna(combined_train_test.groupby('Pclass' ).transform('mean'))
combined_train_test.info()
def get_train_transforms() :
return A.Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
combined_train_test['Group_Ticket'] = combined_train_test['Fare'].groupby(by = combined_train_test['Ticket'] ).transform('count')
 combined_train_test['Fare'] = combined_train_test['Fare']/combined_train_test['Group_Ticket']
combined_train_test.drop(['Group_Ticket'], axis = 1, inplace = True )
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, kinds, image_names, labels, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
target = onehot(4, label)
 return image, target
def __len__(self)-> int:
return self.image_names.shape[0]
def get_labels(self):
return list(self.labels )<define_variables>
if(sum(n == 0 for n in combined_train_test.Fare.values.flatten())> 0):
combined_train_test.loc[combined_train_test.Fare == 0, 'Fare'] = np.nan
combined_train_test['Fare'] = combined_train_test[['Fare']].fillna(combined_train_test.groupby('Pclass' ).transform('mean'))
combined_train_test['Fare'].describe()
fold_number = 0
train_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] != fold_number].kind.values,
image_names=dataset[dataset['fold'] != fold_number].image_name.values,
labels=dataset[dataset['fold'] != fold_number].label.values,
transforms=get_train_transforms() , 
)
 validation_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] == fold_number].kind.values,
image_names=dataset[dataset['fold'] == fold_number].image_name.values,
labels=dataset[dataset['fold'] == fold_number].label.values,
transforms=get_valid_transforms() ,
)<compute_train_metric>
combined_train_test['Fare_Category'] = combined_train_test['Fare'].map(fare_category)
 le_fare = LabelEncoder()
le_fare.fit(np.array(['Very_Low_Fare', 'Low_Fare', 'Med_Fare', 'High_Fare', 'Very_High_Fare']))
combined_train_test['Fare_Category'] = le_fare.transform(combined_train_test['Fare_Category'])
 fare_cat_dummies_df = pd.get_dummies(combined_train_test['Fare_Category'],
prefix = combined_train_test[['Fare_Category']].columns[0])
 combined_train_test = pd.concat([combined_train_test, fare_cat_dummies_df], axis = 1)
 print(combined_train_test['Fare_Category'].groupby(by = combined_train_test['Fare_Category'] ).count().sort_values(ascending = False))
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def alaska_weighted_auc(y_true, y_valid):

 tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1 ).clip(min=0, max=1 ).astype(int)
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = alaska_weighted_auc(self.y_true, self.y_pred)
 @property
def avg(self):
return self.score<normalization>
print(combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean())
 Pclass_1_mean_fare = combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().get([1] ).values[0]
Pclass_2_mean_fare = combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().get([2] ).values[0]
Pclass_3_mean_fare = combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().get([3] ).values[0]
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.05):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )<init_hyperparams>
combined_train_test['Pclass_Fare_Category'] = combined_train_test.apply(pclass_fare_category, args=(Pclass_1_mean_fare, Pclass_2_mean_fare, Pclass_3_mean_fare), axis = 1)
 print(combined_train_test['Pclass_Fare_Category'].groupby(by = combined_train_test['Pclass_Fare_Category'] ).count().sort_values(ascending = False))
le_fare = LabelEncoder()
le_fare.fit(np.array(['Pclass_1_Low_Fare', 'Pclass_1_High_Fare', 'Pclass_2_Low_Fare', 'Pclass_2_High_Fare', 'Pclass_3_Low_Fare', 'Pclass_3_High_Fare']))
combined_train_test['Pclass_Fare_Category'] = le_fare.transform(combined_train_test['Pclass_Fare_Category'] )
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = './'
self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = LabelSmoothing().to(self.device)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss, final_scores = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss, final_scores = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
targets = targets.to(self.device ).float()
batch_size = images.shape[0]
images = images.to(self.device ).float()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(images, targets)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 targets = targets.to(self.device ).float()
images = images.to(self.device ).float()
batch_size = images.shape[0]
self.optimizer.zero_grad()
outputs = self.model(images)
 loss = self.criterion(outputs, targets)
 loss.backward()
final_scores.update(targets, outputs)
 summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss, final_scores
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<choose_model_class>
print(combined_train_test['Fare'].groupby(by = combined_train_test['Pclass'] ).mean().sort_values(ascending = True))
combined_train_test['Pclass'].replace([1, 2, 3],[Pclass_1_mean_fare, Pclass_2_mean_fare, Pclass_3_mean_fare], inplace = True )
def get_net() :
net = EfficientNet.from_pretrained('efficientnet-b2')
 net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)
 return net
net = get_net().cuda()<train_model>
combined_train_test['Family_Size'] = combined_train_test['Parch'] + combined_train_test['SibSp'] + 1
print(combined_train_test['Family_Size'].groupby(by = combined_train_test['Family_Size'] ).count().sort_values(ascending = False))
combined_train_test['Family_Size_Category'] = combined_train_test['Family_Size'].map(family_size_category)
 print(combined_train_test['Family_Size_Category'].groupby(by = combined_train_test['Family_Size_Category'] ).count().sort_values(ascending = False))
print(combined_train_test.groupby(['Survived', 'Family_Size_Category'])['Survived'].count())
 le_family = LabelEncoder()
le_family.fit(np.array(['Single', 'Small_Family', 'Large_Family']))
combined_train_test['Family_Size_Category'] = le_family.transform(combined_train_test['Family_Size_Category'])
 fam_size_cat_dummies_df = pd.get_dummies(combined_train_test['Family_Size_Category'],
prefix = combined_train_test[['Family_Size_Category']].columns[0])
 combined_train_test = pd.concat([combined_train_test, fam_size_cat_dummies_df], axis = 1 )
class TrainGlobalConfig:
num_workers = 4
batch_size = 16
n_epochs = 25
lr = 0.001
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)<load_pretrained>
print(combined_train_test['Age'].groupby(by = combined_train_test['Title'] ).mean().sort_values(ascending = True))
def run_training() :
device = torch.device('cuda:0')
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
batch_size=TrainGlobalConfig.batch_size,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )<load_from_csv>
combined_train_test['Age_Null'] = combined_train_test['Age'].apply(lambda x: 1 if(pd.notnull(x)) else 0 )
file = open('.. /input/alaska2-public-baseline/log.txt', 'r')
 for line in file.readlines() :
print(line[:-1])
 file.close()<load_pretrained>
missing_age_df = pd.DataFrame(combined_train_test[['Age', 'Parch', 'Sex', 'SibSp', 'Family_Size', 'Family_Size_Category', 'Title', 'Fare']])
 missing_age_df = pd.get_dummies(missing_age_df, columns = ['Title', 'Family_Size_Category', 'Sex'])
 missing_age_df.shape
missing_age_df.info()
missing_age_train = missing_age_df[missing_age_df['Age'].notnull() ]
missing_age_test = missing_age_df[missing_age_df['Age'].isnull() ]
checkpoint = torch.load('.. /input/alaska2-public-baseline/best-checkpoint-033epoch.bin')
 net.load_state_dict(checkpoint['model_state_dict']);
net.eval() ;<data_type_conversions>
combined_train_test.loc[(combined_train_test.Age.isnull()), 'Age'] = fill_missing_age(missing_age_train, missing_age_test )
class DatasetSubmissionRetriever(Dataset):
def __init__(self, image_names, transforms=None):
super().__init__()
self.image_names = image_names
self.transforms = transforms
def __getitem__(self, index: int):
image_name = self.image_names[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/Test/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image_name, image
def __len__(self)-> int:
return self.image_names.shape[0]<define_variables>
if(sum(n < 0 for n in combined_train_test.Age.values.flatten())> 0):
combined_train_test.loc[combined_train_test.Age < 0, 'Age'] = np.nan
combined_train_test['Age'] = combined_train_test[['Age']].fillna(combined_train_test.groupby('Title' ).transform('mean'))
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')]),
transforms=get_valid_transforms() , 
)
 data_loader = DataLoader(
 dataset,
batch_size=8,
shuffle=False,
num_workers=2,
drop_last=False,
)<prepare_output>
print(combined_train_test['Age'].groupby(by = combined_train_test['Title'] ).mean().sort_values(ascending = True))
submission_1 = pd.DataFrame(result)
 submission_1.head(5 )<train_model>
combined_train_test['Age_Category'] = combined_train_test['Age'].map(age_group_cat)
 le_age = LabelEncoder()
le_age.fit(np.array(['Baby', 'Toddler', 'Child', 'Teenager', 'Adult', 'Middle_Aged', 'Senior_Citizen', 'Old']))
combined_train_test['Age_Category'] = le_age.transform(combined_train_test['Age_Category'])
 age_cat_dummies_df = pd.get_dummies(combined_train_test['Age_Category'],
prefix = combined_train_test[['Age_Category']].columns[0])
 combined_train_test = pd.concat([combined_train_test, age_cat_dummies_df], axis = 1 )
class TrainGlobalConfig_2:
num_workers = 4
batch_size = 8
n_epochs = 25
lr = 0.002
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
<load_pretrained>
combined_train_test['Ticket_Letter'] = combined_train_test['Ticket'].str.split().str[0]
combined_train_test['Ticket_Letter'] = combined_train_test['Ticket_Letter'].apply(lambda x: np.NaN if x.isnumeric() else x)
 combined_train_test['Ticket_Number'] = combined_train_test['Ticket'].apply(lambda x: pd.to_numeric(x, errors='coerce'))
combined_train_test['Ticket_Number'].fillna(0, inplace = True)
 combined_train_test = pd.get_dummies(combined_train_test, columns = ['Ticket', 'Ticket_Letter'])
 combined_train_test.shape
def run_training_2() :
device = torch.device('cuda:0')
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
batch_size=TrainGlobalConfig_2.batch_size,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig_2.num_workers, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig_2.batch_size,
num_workers=TrainGlobalConfig_2.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig_2)
 fitter.fit(train_loader, val_loader )<define_variables>
combined_train_test['Cabin_Letter'] = combined_train_test['Cabin'].apply(lambda x: str(x)[0] if(pd.notnull(x)) else x)
 combined_train_test = pd.get_dummies(combined_train_test, columns = ['Cabin', 'Cabin_Letter'])
 combined_train_test.shape
results = []
for mode in range(0, 4):
dataset = DatasetSubmissionRetriever(
 image_names=np.array([path.split('/')[-1] for path in glob('.. /input/alaska2-image-steganalysis/Test/*.jpg')]),
transforms=get_test_transforms(mode), 
)
 data_loader = DataLoader(
 dataset,
batch_size=8,
shuffle=False,
num_workers=2,
drop_last=False, 
)
 result = {'Id': [], 'Label': []}
for step,(image_names, images)in enumerate(data_loader):
print(step, end='\r')
 y_pred = net(images.cuda())
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
result['Id'].extend(image_names)
 result['Label'].extend(y_pred)
 results.append(result )<prepare_output>
scale_age_fare = preprocessing.StandardScaler().fit(combined_train_test[['Age', 'Fare']])
 combined_train_test[['Age', 'Fare']] = scale_age_fare.transform(combined_train_test[['Age', 'Fare']] )
y_pred = net(images.cuda())
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
result['Id'].extend(image_names)
 result['Label'].extend(y_pred )<create_dataframe>
combined_train_test.drop(['Name', 'PassengerId', 'Embarked', 'Sex', 'Title', 'Fare_Category', 'Family_Size_Category', 'Age_Category', 'First_Name', 'Last_Name', 'Original_Name', 'Name_Length_Category'], axis = 1, inplace = True )
submissions_2 = []
for mode in range(0,4):
submission = pd.DataFrame(results[mode])
 submissions_2.append(submission )<save_to_csv>
train_data = combined_train_test[:891]
test_data = combined_train_test[891:]
titanic_train_data_X = train_data.drop(['Survived'], axis = 1)
 titanic_train_data_y = train_data['Survived']
titanic_test_data_X = test_data.drop(['Survived'], axis = 1 )
for mode in range(0,4):
submissions_2[mode].to_csv(f'submission_{mode}.csv', index=False )<feature_engineering>
base_models = [ensemble.RandomForestClassifier(n_estimators = 750, criterion = 'gini', max_features = 'sqrt', max_depth = 3, min_samples_split = 4, min_samples_leaf = 2, n_jobs = 15, random_state = 42, verbose = 1),
ensemble.GradientBoostingClassifier(n_estimators = 900, learning_rate = 0.001, loss = 'exponential', min_samples_split = 3, min_samples_leaf = 2, max_features ='sqrt', max_depth = 3, random_state = 42, verbose = 1),
ensemble.ExtraTreesClassifier(n_estimators = 750, max_depth = 35, n_jobs = 15, criterion = 'entropy', random_state = 42, verbose = 1),
ensemble.AdaBoostClassifier(n_estimators = 500, learning_rate = 0.5, random_state = 42),
XGBClassifier(n_estimators = 500, max_depth = 2, eta = 0.05, n_jobs = 15, random_state = 42),
LogisticRegression(C = 0.5, max_iter = 200, random_state = 42),
ensemble.BaggingClassifier(n_estimators = 500, max_samples = 5, max_features = 5, random_state = 42),
DecisionTreeClassifier(max_features = 5, max_depth = 5, min_samples_split = 3, min_samples_leaf =2, random_state =42),
GaussianNB() ,
KNeighborsClassifier(n_neighbors = 2),
SVC(C = 0.4, probability = True, random_state = 42)]
Stacked_train, Stacked_test = Stacking_Ensemble(base_models, titanic_train_data_X, titanic_train_data_y, titanic_test_data_X, n_folds = 7 )
submissions_2[0]['Label'] =(submissions_2[0]['Label']*3 + submissions_2[1]['Label'] + submissions_2[2]['Label'] + submissions_2[3]['Label'])/ 6<feature_engineering>
rf_est = ensemble.RandomForestClassifier(n_estimators = 750, criterion = 'gini', max_features = 'sqrt', max_depth = 3, min_samples_split = 4, min_samples_leaf = 2, n_jobs = 15, random_state = 42, verbose = 1)
 gbm_est = ensemble.GradientBoostingClassifier(n_estimators = 900, learning_rate = 0.001, loss = 'exponential', min_samples_split = 3, min_samples_leaf = 2, max_features ='sqrt', max_depth = 3, random_state = 42, verbose = 1)
 et_est = ensemble.ExtraTreesClassifier(n_estimators = 750, max_depth = 35, n_jobs = 15, criterion = 'entropy', random_state = 42, verbose = 1)
 voting_est = ensemble.VotingClassifier(estimators = [('rf', rf_est),('gbm', gbm_est),('et', et_est)],
voting = 'hard',
n_jobs = 35)
 voting_est.fit(Stacked_train, titanic_train_data_y)
 print("VotingClassifier Score: " + str(voting_est.score(Stacked_train, titanic_train_data_y)))
 print("VotingClassifier Estimators: " + str(voting_est.estimators_))
submissions_2[0]['Label'] =(submissions_2[0]['Label'] * 0.72289157 + submission_1['Label'] * 0.27710843)
 submissions_2[0].head(5 )<save_to_csv>
titanic_test_data_X['Survived'] = voting_est.predict(Stacked_test )
submissions_2[0].to_csv(f'submission.csv', index=False )<import_modules>
Stacked_train_df = pd.DataFrame(Stacked_train)
 Stacked_test_df = pd.DataFrame(Stacked_test)
 titanic_comb_X = pd.concat([Stacked_train_df, Stacked_test_df])
 titanic_comb_y = pd.concat([titanic_train_data_y, titanic_test_data_X['Survived']] )
!pip install -q efficientnet_pytorch
Compose, HorizontalFlip,
ToFloat, VerticalFlip 
)
<define_variables>
rf_est = ensemble.RandomForestClassifier(n_estimators = 750, criterion = 'gini', max_features = 'sqrt', max_depth = 3, min_samples_split = 4, min_samples_leaf = 2, n_jobs = 35, random_state = 42)
 gbm_est = ensemble.GradientBoostingClassifier(n_estimators = 900, learning_rate = 0.001, loss = 'exponential', min_samples_split = 3, min_samples_leaf = 2, max_features ='sqrt', max_depth = 3, random_state = 42)
 et_est = ensemble.ExtraTreesClassifier(n_estimators = 750, max_depth = 35, n_jobs = 15, criterion = 'entropy', random_state = 42)
 voting_est = ensemble.VotingClassifier(estimators = [('rf', rf_est),('gbm', gbm_est),('et', et_est)],
voting = 'hard',
n_jobs = 35)
 voting_est.fit(titanic_comb_X, titanic_comb_y)
 print("VotingClassifier Score: " + str(voting_est.score(titanic_comb_X, titanic_comb_y)))
 print("VotingClassifier Estimators: " + str(voting_est.estimators_))
titanic_test_data_X['Survived_new'] = voting_est.predict(Stacked_test_df )
<load_from_csv><EOS> 
submission = pd.DataFrame({'PassengerId': test_data_orig.loc[:, 'PassengerId'],
'Survived': titanic_test_data_X.loc[:, 'Survived_new']})
 submission.to_csv(".. /working/submission.csv", index = False )