Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<find_best_model_class>
random.seed(123)
 sns.set_style("darkgrid")
 train = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv("/kaggle/input/titanic/test.csv")
 len_train = len(train)
 train_y = train["Survived"]
attention = get_model('attention')
 attention.summary()
attention.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 attention.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y), callbacks=[earlystop, reducelr])
 predictions_attention_real = attention.predict(test_X)
 predictions_attention =(predictions_attention_real >= thresh ).astype(int)
 predictions_val_attention = attention.predict(val_X, batch_size=1024)
 print_f1s(predictions_val_attention)
 del attention
gc.collect()
time.sleep(10)
<data_type_conversions>
lm = ols('Age~ C(Pclass)+ C(Sex)* C(Embarked)+ C(SibSp)+ C(Parch)', pd.concat([train,test],axis=0)).fit()
anova_lm(lm,typ=2 )
val_preds = 0.50*predictions_val_gru + 0.25*predictions_val_lstm + 0.25*predictions_val_attention
val_preds =(val_preds > thresh ).astype(int)
 print_f1s(val_preds )<save_to_csv>
lm = ols('Fare~ C(Pclass)+ C(Sex)+ C(Embarked)+ C(SibSp)+ C(Parch)', pd.concat([train,test],axis=0)).fit()
anova_lm(lm,typ=2 )
final_preds = 0.50*predictions_gru_real + 0.25*predictions_lstm_real + 0.25*predictions_attention_real
final_preds =(final_preds > thresh ).astype(int)
 final_prediction = pd.DataFrame({"qid":test_set["qid"].values})
 final_prediction['prediction'] = final_preds
final_prediction.to_csv("submission.csv", index=False )<import_modules>
mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'the Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attribute_names):
self.attribute_names = attribute_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attribute_names]
class FirstTransformer(BaseEstimator, TransformerMixin):
def __init__(self, name):
self.name = name
def fit(self, X, y=None):
return self
def transform(self, X):
value = [i[0][0] for i in X[self.name]]
X[self.name] = value
return X
class ValueImputer(BaseEstimator, TransformerMixin):
def __init__(self, name, value):
self.value = value
self.name = name
def fit(self, X, y=None):
return self
def transform(self, X):
X[self.name].fillna(self.value,inplace=True)
 return X
class GroupbyImputer(BaseEstimator, TransformerMixin):
def __init__(self, name, group_by):
self.name = name
self.group_by = group_by
def fit(self, X, y=None):
self.value = X.groupby(self.group_by)[self.name]
return self
def transform(self, X):
X[self.name] = self.value.apply(lambda x: x.fillna(x.median()))
i=1
while(X[self.name].isnull().sum() >0):
k = X.groupby(self.group_by[:-i])[self.name]
X[self.name] =(k.apply(lambda x: x.fillna(x.median())))
 i+=1
if i == len(self.group_by):
break
return X
class MostTransformer(BaseEstimator, TransformerMixin):
def __init__(self, names,mapping):
self.names = names
self.mapping = mapping
def fit(self, X, y=None):
self.freq = X[self.names].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]
return self
def transform(self, X, y=None):
X[self.names + "_map"] = self.freq
X.replace({self.names+"_map": self.mapping}, inplace=True)
 return X.drop("Name",axis=1)
 class FrequentTransformer(BaseEstimator, TransformerMixin):
def __init__(self, names):
self.names = names
def fit(self, X, y=None):
self.freq = X.groupby(self.names)[self.names].transform('count')
 return self
def transform(self, X, y=None):
X[self.names + "_freq"] = self.freq
return X.drop(self.names, axis=1)
 imputer_pipeline = Pipeline([ 
('Cabin_imputer', ValueImputer(name = "Cabin",value="None")) , 
('Embarked_imputer', ValueImputer(name= "Embarked",value="S")) , 
('Fare_imputer', GroupbyImputer(name= "Fare",group_by = ['Pclass', 'Parch','SibSp'])) , 
('Age_imputer', GroupbyImputer(name= "Age",group_by = ['Pclass','Parch','Embarked'])) ,
])
 transform_pipeline = Pipeline([ 
('Cabin', FirstTransformer(name= "Cabin")) , 
('Name_transformer', MostTransformer(names = "Name", mapping = mapping)) , 
('Ticket_transformer', FrequentTransformer(names= "Ticket")) ,
] )
FOLD = 0
SEED = 1337
NOTIFY_EACH_EPOCH = False
WORKERS = 0
BATCH_SIZE = 512
N_SPLITS = 10
random.seed(SEED)
 os.environ['PYTHONHASHSEED'] = str(SEED)
 np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def save_checkpoint(state):

 print(" Saving checkpoint")
 filename = f'./checkpoint-{state["epoch"]}.pt.tar'
torch.save(state, filename)
 def initialize(model, path=None, optimizer=None):
if path == None:
checkpoints = glob.glob('./*.pt.tar')
 path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0])for checkpoint in checkpoints])]
checkpoint = torch.load(path)
 model.load_state_dict(checkpoint['model'])
 print(f' Loaded checkpoint {path} | Trained for {checkpoint["epoch"] + 1} epochs')
 if optimizer:
optimizer.load_state_dict(checkpoint['optimizer'])
 epoch = checkpoint['epoch'] + 1
train_iteration = checkpoint['train_iteration']
val_iteration = checkpoint['val_iteration']
return model, optimizer, epoch, train_iteration, val_iteration
else:
return model<load_from_csv>
class TaitanicProcessing(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
self.y = y
return self
def transform(self, X):
X["Sex"] = X["Sex"].map({"male":1, "female":0})
 X["Family"] = X["SibSp"] + X["Parch"] + 1
X["Family"] = X["Family"].astype(str)
 X['Family'] = X['Family'].replace("1", 'Alone')
 X['Family'] = X['Family'].replace(["2","3","4"], 'Normal')
 X['Family'] = X['Family'].replace(["5","6"], 'Mid')
 X['Family'] = X['Family'].replace(["7","8","11"], 'Big')
 X["Fare"] = np.log1p(X["Fare"])
 X["Fare"] = MinMaxScaler().fit_transform(X['Fare'].values.reshape(-1, 1))
X['Cabin'] = X['Cabin'].replace(['A', 'B', 'C','T'], 'ABCT')
 X['Cabin'] = X['Cabin'].replace(['D', 'E'], 'DE')
 X['Cabin'] = X['Cabin'].replace(['F', 'G'], 'FG')
 X['Age'] = pd.qcut(X['Age'], 10)
 X['Age'] = LabelEncoder().fit_transform(X['Age'])
 return X
class DummyCategory(BaseEstimator, TransformerMixin):
def __init__(self, names):
self.names = names
def fit(self, X, y=None):
return self
def transform(self, X):
encoded_fea = []
for c in self.names:
encoded = OneHotEncoder().fit_transform(X[c].values.reshape(-1, 1)).toarray()
n = X[c].nunique()
cols = ['{}_{}'.format(c, n)for n in range(1, n + 1)]
encoded_df = pd.DataFrame(encoded, columns=cols)
 encoded_df.index = X.index
encoded_fea.append(encoded_df)
 return pd.concat([X, *encoded_fea[:5]], axis=1)
 process_pipeline = Pipeline([ 
("process", TaitanicProcessing()), 
("cat_pipeline", DummyCategory(["Embarked", "Name_map","Cabin","Family"])) ,
])
 full_pipeline = Pipeline([ 
("imputer", imputer_pipeline), 
("transform", transform_pipeline), 
("process", process_pipeline),
] )
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<feature_engineering>
df = pd.concat([train.drop("Survived",axis=1),test],axis=0,sort=False)
 df.set_index("PassengerId",drop=True, inplace=True)
 df = full_pipeline.fit_transform(df,train_y)
 df.reset_index(drop=True,inplace=True)
 df.info()
def add_features(df):
df2 = df.copy(deep=True)
 df2['question_text'] = df2['question_text'].apply(lambda x:str(x))
df2['total_length'] = df2['question_text'].apply(len)
 df2['capitals'] = df2['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df2['caps_vs_length'] = df2.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)
 df2['num_words'] = df2.question_text.str.count('\S+')
 df2['num_unique_words'] = df2['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df2['words_vs_unique'] = df2['num_unique_words'] / df2['num_words']
return np.concatenate(( df2['caps_vs_length'].values.reshape(-1, 1), df2['words_vs_unique'].values.reshape(-1, 1)) , axis=1 )<save_to_csv>
df.drop(["Embarked", "Name_map","Cabin","Family",'SibSp','Parch'],axis=1,inplace=True)
 df.head()
kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 train_idx, val_idx = list(kfold.split(train)) [FOLD]
x_train, x_val = train.iloc[train_idx], train.iloc[val_idx]
x_train_meta = add_features(x_train)
 x_val_meta = add_features(x_val)
 test_meta = add_features(test)
 x_train = x_train.reset_index()
x_val = x_val.reset_index()
x_test = test.reset_index()
x_train.to_csv('train.csv')
 x_val.to_csv('val.csv')
 x_test.to_csv('test.csv' )<define_variables>
train = df.iloc[:len_train,:].reset_index(drop=True)
 test = df.iloc[len_train:,:].reset_index(drop=True )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]<string_transform>
def rf_cv(n_estimators, max_depth, min_samples_split, min_samples_leaf, ccp_alpha, x_data=None, y_data=None, n_splits=5, output='score'):
score = 0
kf = StratifiedKFold(n_splits=n_splits, random_state=5, shuffle=True)
 models = []
for train_index, valid_index in kf.split(x_data, y_data):
x_train, y_train = x_data.iloc[train_index], y_data[train_index]
x_valid, y_valid = x_data.iloc[valid_index], y_data[valid_index]
model = RandomForestClassifier(
 criterion='gini',
max_features='auto',
n_estimators = int(n_estimators),
max_depth = int(max_depth),
min_samples_split = int(min_samples_split),
min_samples_leaf = int(min_samples_leaf),
ccp_alpha = ccp_alpha,
random_state = 123,
oob_score=True,
n_jobs=-1 
)
 model.fit(x_train, y_train)
 models.append(model)
 pred = model.predict_proba(x_valid)[:, 1]
true = y_valid
score += roc_auc_score(true, pred)/n_splits
if output == 'score':
return score
if output == 'model':
return models
func_fixed = partial(rf_cv, x_data=train, y_data=train_y, n_splits=5, output='score')
 rf_ba = BayesianOptimization(
 func_fixed,
{
'n_estimators':(1000, 2000),
'max_depth' :(5,13),
'min_samples_split' :(4,8),
'min_samples_leaf' :(4,8),
'ccp_alpha' :(0.0001, 0.01)
 },
random_state=4321 
)
 rf_ba.maximize(init_points=5, n_iter=20 )
nlp = English()
def tokenize(sentence):
sentence = str(sentence)
 for punct in puncts:
sentence = sentence.replace(punct, f' {punct} ')
 x = nlp(sentence)
 return [token.text for token in x]<define_variables>
params = rf_ba.max['params']
rf_model = rf_cv(
 params['n_estimators'],
params['max_depth'],
params['min_samples_split'],
params['min_samples_leaf'],
params['ccp_alpha'],
x_data=train, y_data=train_y, n_splits=5, output='model')
 importances = pd.DataFrame(np.zeros(( train.shape[1], 5)) , columns=['Fold_{}'.format(i)for i in range(1, 6)], index=train.columns)
 preds = []
for i, model in enumerate(rf_model):
importances.iloc[:, i] = model.feature_importances_
pred = model.predict(test)
 preds.append(pred)
 pred = np.mean(preds, axis=0 )
%%time
index_field = data.Field(sequential=False, use_vocab=False, batch_first=True)
 question_field = data.Field(tokenize=tokenize, lower=True, batch_first=True, include_lengths=True)
 target_field = data.Field(sequential=False, use_vocab=False, batch_first=True)
 train_fields = [ 
('id', index_field), 
('index', None), 
('qid', None), 
('question_text', question_field), 
('target', target_field)
 ]
test_fields = [ 
('id', index_field), 
('index', None), 
('qid', None), 
('question_text', question_field)
 ]
train_dataset, val_dataset = data.TabularDataset.splits('./', train='train.csv', validation='val.csv', format='CSV', skip_header=True, fields=train_fields)
 test_dataset = data.TabularDataset('./test.csv', format='CSV', skip_header=True, fields=test_fields)
 question_field.build_vocab(train_dataset, val_dataset, max_size=95000)
 train_dataloader, val_dataloader = data.BucketIterator.splits(( train_dataset, val_dataset),(BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True)
 test_dataloader = data.BucketIterator(test_dataset, 1024, sort=False, shuffle=False)
 print(f'Train Dataset: {len(train_dataset)}')
 print(f'Val Dataset: {len(val_dataset)}')
 print(f'Test Dataset: {len(test_dataset)}' )<categorify>
y_pred = pred
y_pred[y_pred >= 0.5] = 1
y_pred = y_pred.astype(int)
 submission = pd.read_csv("/kaggle/input/titanic/gender_submission.csv")
 submission["Survived"] = y_pred
submission.to_csv("submission.csv",index = False )
class SelfAttention(nn.Module):
def __init__(self, hidden_size, batch_first=False):
super(SelfAttention, self ).__init__()
self.hidden_size = hidden_size
self.batch_first = batch_first
self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)
 nn.init.xavier_uniform_(self.att_weights.data)
 def get_mask(self):
pass
def forward(self, inputs, lengths):
if self.batch_first:
batch_size, max_len = inputs.size() [:2]
else:
max_len, batch_size = inputs.size() [:2]
weights = torch.bmm(inputs,
self.att_weights 
.permute(1, 0)
.unsqueeze(0)
.repeat(batch_size, 1, 1)
)
 attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)
 mask = torch.ones(attentions.size() , requires_grad=True ).cuda()
for i, l in enumerate(lengths):
if l < max_len:
mask[i, l:] = 0
masked = attentions * mask
_sums = masked.sum(-1 ).unsqueeze(-1)
 attentions = masked.div(_sums)
 weighted = torch.mul(inputs, attentions.unsqueeze(-1 ).expand_as(inputs))
representations = weighted.sum(1 ).squeeze()
return representations, attentions
class net(nn.Module):
def __init__(self, embedding):
super(net, self ).__init__()
self.embedding = nn.Embedding.from_pretrained(embedding)
 self.skip_lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=1, batch_first=True, bidirectional=True)
 self.skip_attention = SelfAttention(128*2, batch_first=True)
 self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=1, batch_first=True, bidirectional=True)
 self.attention = SelfAttention(128*2, batch_first=True)
 self.avg_pool = nn.AdaptiveAvgPool1d(1)
 self.max_pool = nn.AdaptiveMaxPool1d(1)
 self.fc = nn.Linear(128*2*4+2, 1)
 self.logit = nn.Linear(1, 1)
 def forward(self, x, x_meta, x_len):
x = self.embedding(x)
 x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)
 skip_lstm, _ = self.skip_lstm(x)
 lstm, _ = self.lstm(x)
 lstm, lstm_lengths = nn.utils.rnn.pad_packed_sequence(lstm, batch_first=True)
 skip_lstm, skip_lstm_lengths = nn.utils.rnn.pad_packed_sequence(skip_lstm, batch_first=True)
 skip_attention, _ = self.skip_attention(skip_lstm, skip_lstm_lengths)
 attention, _ = self.attention(lstm, lstm_lengths)
 avg_pool = self.avg_pool(lstm.transpose(1, 2))
max_pool = self.max_pool(lstm.transpose(1, 2))
x = torch.cat([
skip_attention.view(-1, 128*2),
attention.view(-1, 128*2),
avg_pool.view(-1, 128*2),
max_pool.view(-1, 128*2),
x_meta.view(-1, 2)
 ], dim=1)
 x = self.fc(x)
 x = self.logit(x ).view(-1)
 return x<find_best_params>
rfc_model = RandomForestClassifier(criterion='gini',n_estimators=1800,max_depth=7, min_samples_split=6,min_samples_leaf=6,
max_features='auto', oob_score=True, random_state=123,n_jobs=-1)
 oob = 0
probs = pd.DataFrame(np.zeros(( len(test),10)) , columns=['Fold_{}_Sur_{}'.format(i, j)for i in range(1, 6)for j in range(2)])
 kf = StratifiedKFold(n_splits=5, random_state=5, shuffle=True)
 for fold,(trn_idx, val_idx)in enumerate(kf.split(train, train_y), 1):
rfc_model.fit(train.iloc[trn_idx,:], train_y[trn_idx])
 probs.loc[:, ['Fold_{}_Sur_0'.format(fold),'Fold_{}_Sur_1'.format(fold)]] = rfc_model.predict_proba(test)
 oob += rfc_model.oob_score_ / 5
print('Fold {} OOB : {}'.format(fold, rfc_model.oob_score_))
print('Average Score: {}'.format(oob))
def choose_threshold(val_preds, y_val):
thresholds = np.arange(0.1, 0.501, 0.01)
 val_scores = []
for threshold in thresholds:
threshold = np.round(threshold, 2)
 f1 = f1_score(y_val,(val_preds > threshold ).astype(int))
val_scores.append(f1)
 best_val_f1 = np.max(val_scores)
 best_threshold = np.round(thresholds[np.argmax(val_scores)], 2)
 return best_threshold, best_val_f1<train_on_grid>
survived =[col for col in probs.columns if col.endswith('Sur_1')]
probs['survived'] = probs[survived].mean(axis=1)
 probs['unsurvived'] = probs.drop(columns=survived ).mean(axis=1)
 probs['pred'] = 0
sub = probs[probs['survived'] >= 0.5].index
probs.loc[sub, 'pred'] = 1
y_pred = probs['pred'].astype(int )
<load_pretrained><EOS> 
submission = pd.read_csv("/kaggle/input/titanic/gender_submission.csv")
 submission["Survived"] = y_pred
submission.to_csv("submission.csv",index = False )