Kaggle competition: Titanic - Machine Learning from Disaster
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(model_path+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(model_path+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model<string_transform>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv", index_col='PassengerId')
 train_data.head()
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0): return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))<define_variables>
test_data = pd.read_csv("/kaggle/input/titanic/test.csv", index_col='PassengerId')
 test_data.head()
SEED = 42
EPOCHS = 3
BATCH_SIZE = 32
PAD_ID = 1
LABEL_SMOOTHING = 0.1
np.random.seed(SEED)
 jac = []
VER='v0'
DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=3,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(input_ids,train.sentiment.values)) :
print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) ,
key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3),
reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],
token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
all = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = train.loc[k,'text']
else:
text1 = " "+" ".join(train.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 all.append(jaccard(st,train.loc[k,'selected_text']))
jac.append(np.mean(all))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(all))
print()<string_transform>
cols_with_missing = [col for col in train_data.columns if train_data[col].isnull().any() ]
cols_with_missing
selectedtext = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test.loc[k,'text']
else:
text1 = " "+" ".join(test.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 selectedtext.append(st )<save_to_csv>
cat =(train_data.dtypes == 'object')
 object_cols = list(cat[cat].index)
 object_cols
test['selected_text'] = selectedtext
test[['textID','selected_text']].to_csv('submission.csv',index=False)
 test.sample(10 )<load_pretrained>
test_cols_with_missing = [col for col in test_data.columns if test_data[col].isnull().any() ]
test_cols_with_missing
<install_modules>
test_cat =(test_data.dtypes == 'object')
 test_object_cols = list(test_cat[test_cat].index)
 test_object_cols
!pip install /kaggle/input/fairseq-and-fastbpe/sacrebleu-1.4.9-py3-none-any.whl
!pip install /kaggle/input/fairseq-and-fastbpe/fairseq-0.9.0-cp37-cp37m-linux_x86_64.whl
!pip install /kaggle/input/fairseq-and-fastbpe/fastBPE-0.1.0-cp37-cp37m-linux_x86_64.whl<set_options>
object_nunique = list(map(lambda col: train_data[col].nunique() , object_cols))
d = dict(zip(object_cols, object_nunique))
object_nunique_test = list(map(lambda col: test_data[col].nunique() , test_object_cols))
d_test = dict(zip(test_object_cols, object_nunique_test))
sorted(d.items() , key=lambda x: x[1] )
warnings.filterwarnings("ignore" )<set_options>
sorted(d.items() , key=lambda x: x[1] )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 777
seed_everything(seed )<categorify>
low_cardinality_cols = [col for col in object_cols if train_data[col].nunique() < 10]
high_cardinality_cols = list(set(object_cols)-set(low_cardinality_cols))
print('Categorical columns that will be one-hot encoded:', low_cardinality_cols)
 print('
Categorical columns that will be dropped from the dataset:', high_cardinality_cols )
class BERTweetTokenizer() :
def __init__(self, path):
self.bpe = fastBPE(SimpleNamespace(bpe_codes=os.path.join(path, 'bpe.codes')))
 self.vocab = Dictionary()
self.vocab.add_from_file(os.path.join(path, 'dict.txt'))
def encode(self, text):
tokens = self.tokenize(text)
 input_ids = self.convert_tokens_to_ids(tokens)
 offsets = []
curr_pos = 0
for idx, token in enumerate(tokens):
need_space = idx > 0 and(not tokens[idx-1].endswith('@@'))
curr_pos += need_space
offsets.append(( curr_pos, curr_pos + len(token.replace('@@', ''))))
curr_pos += len(token.replace('@@', ''))
return SimpleNamespace(ids=input_ids, tokens=tokens, offsets=offsets)
 def tokenize(self, text):
return self.bpe.encode(text ).split()
def convert_tokens_to_ids(self, tokens):
input_ids = self.vocab.encode_line(' '.join(tokens), append_eos=False, add_if_not_exist=False ).long().tolist()
return input_ids
def decode_tokens(self, tokens):
decoded = ' '.join(tokens ).replace('@@ ', '' ).strip()
return decoded 
<define_variables>
object_cols = [col for col in train_data.columns if train_data[col].dtype == "object"]
good_label_cols = [col for col in object_cols if
set(train_data[col])== set(test_data[col])]
bad_label_cols = list(set(object_cols)-set(good_label_cols))
print('Categorical columns that will be label encoded:', good_label_cols)
 print('
Categorical columns that will be dropped from the dataset:', bad_label_cols )
class config:
VERSION = 'v0-0-5-bertweet'
INPUT_DATASET = 'tweet-sentiment-extraction'
MODEL_DATASET = f'model-{VERSION}'
ROBERTA_DATASET = 'bertweet-base-transformers'
TRAIN = False
IS_KAGGLE = True
DATA_DIR = 'data' if not IS_KAGGLE else f'.. /input/{INPUT_DATASET}'
MODELS_DIR = os.path.join(
 'models' if not IS_KAGGLE else '.. /input',
MODEL_DATASET 
)
 ROBERTA_DIR = os.path.join(
 'transformers_data' if not IS_KAGGLE else '.. /input',
ROBERTA_DATASET 
)
 FOLD = None
N_FOLDS = 5
LEARNING_RATE = 3e-5
N_LAST_LAYERS = 3
MAX_LEN = 120
TRAIN_BATCH_SIZE = 32
VALID_BATCH_SIZE = 32
EPOCHS = 5
TRAINING_FILE = os.path.join(DATA_DIR, 'train.csv')
 TEST_FILE = os.path.join(DATA_DIR, 'test.csv')
 SAMPLE_FILE = os.path.join(DATA_DIR, 'sample_submission.csv')
 TOKENIZER = BERTweetTokenizer(ROBERTA_DIR)
 @classmethod
def get_model_paths(cls):
res = []
for filename in os.listdir(cls.MODELS_DIR):
if filename.endswith('.bin'):
res.append(os.path.join(cls.MODELS_DIR, filename))
return res 
<normalization>
X = train_data
X_test = test_data
X.dropna(axis=0, subset=['Survived'], inplace=True)
 y = X.Survived
X.drop(['Survived'], axis=1, inplace=True)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y,
train_size=0.8, test_size=0.2,
random_state=0)
 cols_with_missing = [col for col in X.columns
if X[col].isnull().any() ]
categorical_cols = [cname for cname in X_train.columns if
X_train[cname].nunique() < 10 and
X_train[cname].dtype == "object"]
numerical_cols = [cname for cname in X_train.columns if
X_train[cname].dtype in ['int64', 'float64']]
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
class AverageMeter:

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def upload_kaggle_dataset(dir_path, id_, user='drack3800', title=None):
title = title or id_
metadata = {
"title": title,
"id": f"{user}/{id_}",
"licenses": [
{
"name": "CC0-1.0"
}
]
}
with open(os.path.join(dir_path, 'dataset-metadata.json'), 'w')as fout:
fout.write(json.dumps(metadata)+ '
')
 proc = subprocess.run(['kaggle', 'datasets', 'create', '-p' , dir_path], capture_output=True)
 print(proc.stdout.decode())
<feature_engineering>
numerical_transformer = SimpleImputer(strategy='mean')
 categorical_transformer = Pipeline(steps=[ 
('imputer', SimpleImputer(strategy='most_frequent')) , 
('onehot', OneHotEncoder(handle_unknown='ignore'))
])
 preprocessor = ColumnTransformer(
 transformers=[ 
('num', numerical_transformer, numerical_cols), 
('cat', categorical_transformer, categorical_cols)
 ] )
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=config.MAX_LEN):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = config.TOKENIZER
def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets, token_type_ids = self.get_input_data(row)
 data['ids'] = ids
data['mask'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
data['token_type_ids'] = token_type_ids
data['sentiment'] = row.sentiment
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
data['selected_tweet'] = " ".join(str(row.selected_text ).lower().split())
 return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " ".join(str(row.text ).lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 token_type_ids = torch.zeros(len(ids), dtype=torch.int)
 return ids, masks, tweet, offsets, token_type_ids
def get_target_idx(self, row, tweet, offsets):
selected_text = " ".join(str(row.selected_text ).lower().split())
 len_st = len(selected_text)
 idx0 = None
idx1 = None
for ind in range(len(tweet)) :
if tweet[ind: ind + len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx<load_pretrained>
model = RandomForestClassifier(n_estimators=100, criterion='mae', max_leaf_nodes=100, random_state=0 )
class TweetModel(nn.Module):
def __init__(self):
super().__init__()
conf = transformers.RobertaConfig.from_pretrained(
 config.ROBERTA_DIR,
output_hidden_states=True, 
)
 self.roberta = transformers.RobertaModel.from_pretrained(
 os.path.join(config.ROBERTA_DIR, 'model.bin'),
config=conf 
)
 self.drop_out = nn.Dropout(0.5)
 self.l0 = nn.Linear(768, 2)
 nn.init.normal_(self.l0.weight, std=0.02)
 nn.init.normal_(self.l0.bias, 0)
 self.layer_weights = nn.Parameter(torch.ones(config.N_LAST_LAYERS)/ config.N_LAST_LAYERS)
 def forward(self, ids, mask, token_type_ids=None):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 last_layers_out = torch.stack([out[-i] for i in range(1, config.N_LAST_LAYERS + 1)], dim=0)
 layer_weights = torch.softmax(self.layer_weights, dim=0)
 out = torch.sum(layer_weights[:, None, None, None] * last_layers_out, dim=0)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits
def build_model(state_dict_path=None, train=True, device=None):
model = TweetModel()
if device:
model.to(device)
 if state_dict_path:
print(f"Loaded model from {state_dict_path}")
 model.load_state_dict(torch.load(state_dict_path))
model.train(train)
 return model<compute_train_metric>
def get_score(n_estimators, max_leaf_nodes):
my_pipeline = Pipeline(steps=[('preprocessor', preprocessor), 
('model', RandomForestClassifier(n_estimators=n_estimators,max_leaf_nodes=max_leaf_nodes, random_state=0))
])
 scores = -1 * cross_val_score(my_pipeline, X, y,
cv=3,
scoring='neg_mean_absolute_error')
 return scores.mean()
def loss_fn(start_logits, end_logits, start_positions, end_positions):
loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss<train_model>
k=100
results = {}
for i in range(1,9):
for k in range(1,8):
results[50*i, 50*k] = get_score(50*i,50*k)
 results
def train_fn(data_loader, model, optimizer, device, num_batches, scheduler=None):
ewma_loss = None
model.train()
tk0 = tqdm(data_loader, total=num_batches, desc="Training")
 for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d['start_idx']
targets_end = d['end_idx']
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
loss_item = loss.item()
if ewma_loss is None:
ewma_loss = loss_item
else:
ewma_loss = 0.8 * ewma_loss + 0.2 * loss_item
tk0.set_postfix(loss=loss_item, ewma=ewma_loss)
 return ewma_loss<create_dataframe>
%matplotlib inline
plt.plot(list(results.keys()),list(results.values()))
plt.show()
def run(model, df_train, df_valid, model_name, device='cuda'):
model = model.to(device)
 train_dataset = TweetDataset(df_train)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
shuffle=True,
drop_last=True,
num_workers=2 
)
 valid_dataset = TweetDataset(df_valid)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
shuffle=False,
drop_last=False,
num_workers=1 
)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 optimizer = AdamW(
 model.parameters() ,
lr=config.LEARNING_RATE 
)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=int(0.1 * num_train_steps),
num_training_steps=num_train_steps 
)
 best_jac = 0
num_batches = int(len(df_train)/ config.TRAIN_BATCH_SIZE)
 print("Training is Starting.... ")
 for epoch in range(config.EPOCHS):
ewma_loss = train_fn(
 train_data_loader,
model,
optimizer,
device,
num_batches,
scheduler 
)
 jac = eval_fn(
 valid_data_loader,
model,
device 
)
 print(f'Epoch={epoch}, Jaccard={jac:.4f}, Train Loss={ewma_loss:.4f}')
 if jac > best_jac:
model_path = os.path.join(
 config.MODELS_DIR,
f'{model_name}.bin' 
)
 os.makedirs(config.MODELS_DIR, exist_ok=True)
 print(f"Model Improved! Saving model to {model_path}")
 torch.save(model.state_dict() , model_path)
 best_jac = jac
return best_jac
def generate_folds(dfx):
skf = model_selection.StratifiedKFold(n_splits=config.N_FOLDS, random_state=777)
 dfx = dfx.copy()
dfx['kfold'] = 0
for n_fold,(_, val_idx)in enumerate(skf.split(dfx.values, dfx.sentiment.values)) :
dfx.loc[val_idx, 'kfold'] = n_fold
return dfx<data_type_conversions>
my_pipeline = Pipeline(steps=[('preprocessor', preprocessor), 
('model', RandomForestClassifier(n_estimators=400,max_leaf_nodes=50,random_state=0))
])
 scores = -1 * cross_val_score(my_pipeline, X, y,
cv=3,
scoring='neg_mean_absolute_error')
 my_pipeline.fit(X, y)
 preds = my_pipeline.predict(X_test)
<choose_model_class>
output = pd.DataFrame({'PassengerId': X_test.index,
'Survived': preds})
 output.to_csv('submission_final_5.csv', index=False )
if config.TRAIN:
fold_metrics = []
dfx = pd.read_csv(config.TRAINING_FILE)
 dfx = generate_folds(dfx)
 if config.FOLD is not None:
folds = [config.FOLD]
print("Going to train in single-fold mode")
 else:
folds = list(range(config.N_FOLDS))
print("Going to train in all-folds mode")
 for n_fold in folds:
model_name = f'model_{n_fold}'
MX = build_model()
df_train = dfx[dfx.kfold != n_fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == n_fold].reset_index(drop=True)
 print(f'
>>> Running training for fold={n_fold}, model_name={model_name}')
 jac = run(MX, df_train, df_valid, model_name, device='cuda')
 print(f'>>> Finished training for fold={n_fold}.Jaccard = {jac:.4f}')
 fold_metrics.append(jac)
 print('
*********** REPORT **********')
 print(f'CV Jaccard = {np.mean(fold_metrics):.4f} +- {np.std(fold_metrics):.4f}')
<load_pretrained>
my_model_2 = XGBRegressor(n_estimators=300000, learning_rate=1.2)
 my_model_2.fit(X, y,
early_stopping_rounds = 5,
eval_set = [(X_valid, y_valid)],
verbose=False)
 predictions_2 = my_model_2.predict(X_test)
<load_from_csv><EOS> 
output = pd.DataFrame({'PassengerId': X_test.index,
'Survived': predictions_2})
 output.to_csv('submission_final_XGBoost.csv', index=False )