Kaggle competition: Titanic - Machine Learning from Disaster
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'arj_bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_inputs, train_outputs, epochs=epochs, batch_size=batch_size)
 model.save_weights(f'arj_bert-{fold}.h5')
 valid_preds.append(model.predict(valid_inputs))
test_preds.append(model.predict(test_inputs))
rho_val = compute_spearmanr(valid_outputs, valid_preds[-1])
 print('validation score = ', rho_val)
<categorify>
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<load_pretrained>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 y = train.Survived
passengerid = test.PassengerId
titanic = train.append(test, ignore_index = True )
models = []
for i in range(1,4):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
<load_pretrained>
train_index = len(train)
 test_index = len(titanic)- len(test )
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
titanic['Title'] = titanic.Name.apply(lambda x: x.split(',')[1].split('.')[0].strip() )
for i in range(3,5):
model_path = f'.. /input/using-pretrained-kernel/using_pretrained_kernel/arj_bert-{0}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
titanic.Title.value_counts()
for i in range(3-5):
model_path = f'.. /input/bert-pretrained-models/Pretrained_bert_models/bert-{0}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
normalized_title = {
'Mr':"Mr",
'Mrs': "Mrs",
'Ms': "Mrs",
'Mme':"Mrs",
'Mlle':"Miss",
'Miss':"Miss",
'Master':"Master",
'Dr':"Officer",
'Rev':"Officer",
'Col':"Officer",
'Capt':"Officer",
'Major':"Officer",
'Lady':"Royalty",
'Sir':"Royalty",
'the Countess':"Royalty",
'Dona':"Royalty",
'Don':"Royalty",
'Jonkheer':"Royalty"
}
for i in range(2):
model_path = f'.. /input/bertmodelpretrained/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
titanic.Title = titanic.Title.map(normalized_title )
for i in range(2,5):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
print(titanic.Title.value_counts() )
for i in range(3,6):
model_path = f'.. /input/bert-base-tf2-0-training/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<define_variables>
titanic.Age = grouped.Age.apply(lambda x: x.fillna(x.median()))
titanic.isnull().sum()
test_predictions = []<predict_on_test>
titanic.Fare = titanic.Fare.fillna(titanic.Fare.mean())
 titanic.Cabin = titanic.Cabin.fillna('U')
 most_Embarked = titanic.Embarked.value_counts().index[0]
titanic.Embarked = titanic.Embarked.fillna(most_Embarked )
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))<prepare_output>
titanic.isnull().sum()
final_predictions = np.mean(test_predictions, axis=0 )<load_pretrained>
titanic['FamilySize'] = titanic['SibSp'] + titanic['Parch'] + 1
n=df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.) 
<save_to_csv>
titanic['Cabin'].value_counts()
df_sub['question_type_spelling']=spelling
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )<set_options>
titanic.Cabin = titanic.Cabin.map(lambda x: x[0])
 titanic.Cabin.head()
def tqdm(it, *args, **kwargs):
return it
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
np.set_printoptions(suppress=True)
 print(tf.__version__)
 print(torch.__version__ )<load_from_csv>
titanic.Sex = titanic.Sex.map({"male":0,"female":1})
 title_dummies = pd.get_dummies(titanic.Title , prefix = "Title")
 cabin_dummies = pd.get_dummies(titanic.Cabin , prefix = "Cabin")
 pclass_dummies = pd.get_dummies(titanic.Pclass , prefix ="Pclass")
 embarked_dummies = pd.get_dummies(titanic.Embarked , prefix = "Embarked" )
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bertpretrained/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12/'
tokenizer = BertTokenizer.from_pretrained(BERT_PATH)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<categorify>
titanic_dummies = pd.concat([titanic , title_dummies,cabin_dummies,
pclass_dummies,embarked_dummies],axis = 1 )
df_train.question_body = df_train.question_body.apply(html.unescape)
 df_train.question_title = df_train.question_title.apply(html.unescape)
 df_train.answer = df_train.answer.apply(html.unescape)
 df_test.question_body = df_test.question_body.apply(html.unescape)
 df_test.question_title = df_test.question_title.apply(html.unescape)
 df_test.answer = df_test.answer.apply(html.unescape )<categorify>
titanic_dummies.drop(['Pclass', 'Title', 'Cabin', 'Embarked', 'Name', 'Ticket'],axis = 1,inplace = True )
def _preprocess_text(s: str)-> str:
return s
def _trim_input(question_tokens: List[str], answer_tokens: List[str], max_sequence_length: int, q_max_len: int, a_max_len: int)-> Tuple[List[str], List[str]]:
q_len = len(question_tokens)
 a_len = len(answer_tokens)
 if q_len + a_len + 3 > max_sequence_length:
if a_max_len <= a_len and q_max_len <= q_len:
q_new_len_head = floor(( q_max_len - q_max_len/2))
question_tokens = question_tokens[:q_new_len_head] + question_tokens[q_new_len_head - q_max_len:]
a_new_len_head = floor(( a_max_len - a_max_len/2))
answer_tokens = answer_tokens[:a_new_len_head] + answer_tokens[a_new_len_head - a_max_len:]
elif q_len <= a_len and q_len < q_max_len:
a_max_len = a_max_len +(q_max_len - q_len - 1)
 a_new_len_head = floor(( a_max_len - a_max_len/2))
answer_tokens = answer_tokens[:a_new_len_head] + answer_tokens[a_new_len_head - a_max_len:]
elif a_len < q_len:
assert a_len <= a_max_len
q_max_len = q_max_len +(a_max_len - a_len - 1)
 q_new_len_head = floor(( q_max_len - q_max_len/2))
question_tokens = question_tokens[:q_new_len_head] + question_tokens[q_new_len_head - q_max_len:]
else:
raise ValueError("unreachable: q_len: {}, a_len: {}, q_max_len: {}, a_max_len: {}".format(q_len, a_len, q_max_len, a_max_len))
return question_tokens, answer_tokens
def _convert_to_transformer_inputs(title: str, question: str, answer: str, tokenizer: BertTokenizer, question_only=False):
title = _preprocess_text(title)
 question = _preprocess_text(question)
 answer = _preprocess_text(answer)
 question = "{} [SEP] {}".format(title, question)
 question_tokens = tokenizer.tokenize(question)
 if question_only:
answer_tokens = []
else:
answer_tokens = tokenizer.tokenize(answer)
 question_tokens, answer_tokens = _trim_input(question_tokens, answer_tokens, MAX_SEQUENCE_LENGTH,(MAX_SEQUENCE_LENGTH - 3)// 2,(MAX_SEQUENCE_LENGTH - 3)// 2)
 ids = tokenizer.convert_tokens_to_ids(["[CLS]"] + question_tokens + ["[SEP]"] + answer_tokens + ["[SEP]"])
 padded_ids = ids + [tokenizer.pad_token_id] *(MAX_SEQUENCE_LENGTH - len(ids))
token_type_ids = [0] *(1 + len(question_tokens)+ 1)+ [1] *(len(answer_tokens)+ 1)+ [0] *(MAX_SEQUENCE_LENGTH - len(ids))
attention_mask = [1] * len(ids)+ [0] *(MAX_SEQUENCE_LENGTH - len(ids))
return padded_ids, token_type_ids, attention_mask
sample_args = df_train["question_title"].values[0], df_train["question_body"].values[0], df_train["answer"].values[0]
sample_ids = _convert_to_transformer_inputs(*sample_args, tokenizer, question_only=True)
 print(sample_ids)
 print(tokenizer.convert_ids_to_tokens(sample_ids[0]))<categorify>
train_x = titanic_dummies[:train_index]
test_x = titanic_dummies[train_index:]
def compute_input_arrays(df, question_only=False):
input_ids, input_token_type_ids, input_attention_masks = [], [], []
for title, body, answer in zip(df["question_title"].values, df["question_body"].values, df["answer"].values):
ids, type_ids, mask = _convert_to_transformer_inputs(title, body, answer, tokenizer, question_only=question_only)
 input_ids.append(ids)
 input_token_type_ids.append(type_ids)
 input_attention_masks.append(mask)
 return(
 np.asarray(input_ids, dtype=np.int32),
np.asarray(input_token_type_ids, dtype=np.int32),
np.asarray(input_attention_masks, dtype=np.int32), 
)
 def compute_output_arrays(df):
return np.asarray(df[output_categories] )<load_pretrained>
train_x.Survived = train_x.Survived.astype(int )
class Model(torch.nn.Module):
def __init__(self):
super().__init__()
config = BertConfig.from_json_file(BERT_PATH + "/bert_config.json")
 config.output_hidden_states = True
self.bert = BertForPreTraining.from_pretrained(BERT_PATH + "/bert_model.ckpt.index", from_tf=True, config=config ).bert
self.cls_token_head = nn.Sequential(
 nn.Dropout(0.1),
nn.Linear(768 * 4, 768),
nn.ReLU(inplace=True), 
)
 self.qa_sep_token_head = nn.Sequential(
 nn.Dropout(0.1),
nn.Linear(768 * 4, 768),
nn.ReLU(inplace=True), 
)
 self.linear = nn.Sequential(
 nn.Dropout(0.1),
nn.Linear(768 * 2, 30), 
)
 def forward(self, input_ids, attention_mask, token_type_ids):
question_answer_seps =(torch.sum(( token_type_ids == 0)* attention_mask, -1)- 1)
 _, _, hidden_states = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 hidden_states_cls_embeddings = [x[:, 0] for x in hidden_states[-4:]]
x = torch.cat(hidden_states_cls_embeddings, dim=-1)
 x_cls = self.cls_token_head(x)
 tmp = torch.arange(0, len(input_ids), dtype=torch.long)
 hidden_states_qa_sep_embeddings = [x[tmp, question_answer_seps] for x in hidden_states[-4:]]
x = torch.cat(hidden_states_qa_sep_embeddings, dim=-1)
 x_qa_sep = self.qa_sep_token_head(x)
 x = torch.cat([x_cls, x_qa_sep], -1)
 x = self.linear(x)
 return x<data_type_conversions>
X = train_x.drop('Survived', axis=1 ).values
y = train_x.Survived.values
outputs = torch.tensor(compute_output_arrays(df_train), dtype=torch.float)
 inputs = [torch.tensor(x, dtype=torch.long)for x in compute_input_arrays(df_train)]
question_only_inputs = [torch.tensor(x, dtype=torch.long)for x in compute_input_arrays(df_train, question_only=True)]
test_inputs = [torch.tensor(x, dtype=torch.long)for x in compute_input_arrays(df_test)]
test_question_only_inputs = [torch.tensor(x, dtype=torch.long)for x in compute_input_arrays(df_test, question_only=True)]<set_options>
X_test = test_x.drop('Survived', axis=1 ).values
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device )<find_best_params>
label = train_x.Survived
train_X , val_X , train_Y , val_Y = train_test_split(train_x , label,test_size = 0.2,shuffle = True )
for n, _ in Model().named_parameters() :
print(n )<data_type_conversions>
dtrain_X = train_X.drop('Survived',axis = 1 )
LABEL_WEIGHTS = torch.tensor(1.0 / df_train[output_categories].std().values, dtype=torch.float32 ).to(device)
 LABEL_WEIGHTS = LABEL_WEIGHTS / LABEL_WEIGHTS.sum() * 30
for name, weight in zip(output_categories, LABEL_WEIGHTS.cpu().numpy()):
print(name, "\t", weight )<define_search_space>
dval_X = val_X.drop('Survived',axis = 1 )
BEST_BINS = [400, 400, 15, 100, 400, 7, 1600, 100, 100, 400, 100, 9, 8, 50, 9, 8, 15, 400, 400, 5, 400, 400, 800, 50, 200, 1600, 20, 200, 1600, 1600]
def binning_output(preds, n_bins=BEST_BINS):
preds = preds.copy()
for i in range(preds.shape[-1]):
n = n_bins[i]
binned =(preds[:, i] * n ).astype(np.int32 ).astype(np.float32)/ n
unique_values, unique_counts = np.unique(binned, return_counts=True)
 minor_value_ratio =(unique_counts.sum() - unique_counts.max())/ unique_counts.sum()
if minor_value_ratio < 0.005:
keep = np.argsort(preds[:, i])[::-1][:int(len(preds)* 0.005)+ 1]
binned[keep] = preds[keep, i]
preds[:, i] = binned
return preds
def compute_spearmanr(trues, preds, n_bins=None):
rhos = []
if n_bins:
preds = binning_output(preds, n_bins)
 for col_trues, col_pred in zip(trues.T, preds.T):
if len(np.unique(col_pred)) == 1:
col_pred[np.random.randint(0, len(col_pred)- 1)] = col_pred.max() + 1
rhos.append(spearmanr(col_trues, col_pred ).correlation)
 return np.mean(rhos)
 def compute_loss(outputs, targets, alpha=0.5, margin=0.1, question_only=False):
if question_only:
outputs = outputs[:, :21]
targets = targets[:, :21]
bce = F.binary_cross_entropy_with_logits(outputs, targets, reduction="none")
 bce =(bce * LABEL_WEIGHTS[:bce.size(-1)] ).mean()
batch_size = outputs.size(0)
 if batch_size % 2 == 0:
outputs1, outputs2 = outputs.sigmoid().contiguous().view(2, batch_size // 2, outputs.size(-1))
targets1, targets2 = targets.contiguous().view(2, batch_size // 2, outputs.size(-1))
ordering =(targets1 > targets2 ).float() -(targets1 < targets2 ).float()
margin_rank_loss =(-ordering *(outputs1 - outputs2)+ margin ).clamp(min=0.0)
 margin_rank_loss =(margin_rank_loss * LABEL_WEIGHTS[:outputs.size(-1)] ).mean()
else:
margin_rank_loss = 0.0
return alpha * bce +(1 - alpha)* margin_rank_loss
def train_and_predict(train_data, valid_data, test_data, q_train_data, q_valid_data, q_test_data, q_epochs, epochs, batch_size, fold):
dataloader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=batch_size)
 valid_dataloader = torch.utils.data.DataLoader(valid_data, shuffle=False, batch_size=batch_size)
 test_dataloader = torch.utils.data.DataLoader(test_data, shuffle=False, batch_size=batch_size)
 q_dataloader = torch.utils.data.DataLoader(q_train_data, shuffle=True, batch_size=batch_size)
 q_valid_dataloader = torch.utils.data.DataLoader(q_valid_data, shuffle=False, batch_size=batch_size)
 q_test_dataloader = torch.utils.data.DataLoader(q_test_data, shuffle=False, batch_size=batch_size)
 model = Model().to(device)
 no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [p for n, p in model.named_parameters() if p.requires_grad and not any(nd in n for nd in no_decay)and "bert" in n],
"weight_decay": 1e-2,
"lr": 5e-5
},
{
"params": [p for n, p in model.named_parameters() if p.requires_grad and any(nd in n for nd in no_decay)and "bert" in n],
"weight_decay": 0.0,
"lr": 5e-5
},
{
"params": [p for n, p in model.named_parameters() if p.requires_grad and "bert" not in n],
"weight_decay": 1e-2,
"lr": 5e-4
}
]
optimizer = AdamW(optimizer_grouped_parameters)
 scheduler = get_linear_schedule_with_warmup(
 optimizer, num_warmup_steps=int(len(dataloader)*(q_epochs)* 0.05),
num_training_steps=len(dataloader)*(q_epochs)
)
 test_predictions = []
valid_predictions = []
for epoch in range(q_epochs):
start = time.time()
model.train()
train_losses = []
train_preds = []
train_targets = []
for input_ids, token_type_ids, attention_mask, targets in tqdm(q_dataloader, total=len(q_dataloader)) :
input_ids = input_ids.to(device)
 token_type_ids = token_type_ids.to(device)
 attention_mask = attention_mask.to(device)
 targets = targets.to(device)
 outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 train_preds.extend(outputs.detach().sigmoid().cpu().numpy())
 train_targets.extend(targets.detach().cpu().numpy())
 loss = compute_loss(outputs, targets, question_only=True)
 model.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
train_losses.append(loss.detach().cpu().item())
 model.eval()
valid_losses = []
valid_preds = []
valid_targets = []
with torch.no_grad() :
for input_ids, token_type_ids, attention_mask, targets in tqdm(q_valid_dataloader, total=len(q_valid_dataloader)) :
input_ids = input_ids.to(device)
 token_type_ids = token_type_ids.to(device)
 attention_mask = attention_mask.to(device)
 targets = targets.to(device)
 outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 prob = outputs.sigmoid()
prob[:, 21:] = 0.0
valid_preds.extend(prob.cpu().numpy())
 valid_targets.extend(targets.cpu().numpy())
 loss = compute_loss(outputs, targets, question_only=True)
 valid_losses.append(loss.detach().cpu().item())
 valid_predictions.append(np.stack(valid_preds))
test_preds = []
for input_ids, token_type_ids, attention_mask in tqdm(q_test_dataloader, total=len(q_test_dataloader)) :
input_ids = input_ids.to(device)
 token_type_ids = token_type_ids.to(device)
 attention_mask = attention_mask.to(device)
 outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 prob = outputs.sigmoid()
prob[:, 21:] = 0.0
test_preds.extend(prob.cpu().numpy())
 test_predictions.append(np.stack(test_preds))
print()
print("Epoch {}: Train Loss {}, Valid Loss {}".format(epoch + 1, np.mean(train_losses), np.mean(valid_losses)))
 print("\t Train Spearmanr {:.4f}, Valid Spearmanr(avg){:.4f}, Valid Spearmanr(last){:.4f}".format(
 compute_spearmanr(np.stack(train_targets), np.stack(train_preds)) ,
compute_spearmanr(np.stack(valid_targets), sum(valid_predictions)/ len(valid_predictions)) ,
compute_spearmanr(np.stack(valid_targets), valid_predictions[-1])
))
print("\t elapsed: {}s".format(time.time() - start))
model = Model().to(device)
 no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [p for n, p in model.named_parameters() if p.requires_grad and not any(nd in n for nd in no_decay)and "bert" in n],
"weight_decay": 1e-2,
"lr": 5e-5
},
{
"params": [p for n, p in model.named_parameters() if p.requires_grad and any(nd in n for nd in no_decay)and "bert" in n],
"weight_decay": 0.0,
"lr": 5e-5
},
{
"params": [p for n, p in model.named_parameters() if p.requires_grad and "bert" not in n],
"weight_decay": 1e-2,
"lr": 5e-4
}
]
optimizer = AdamW(optimizer_grouped_parameters)
 scheduler = get_linear_schedule_with_warmup(
 optimizer, num_warmup_steps=int(len(dataloader)*(epochs)* 0.05),
num_training_steps=len(dataloader)*(epochs)
)
 for epoch in range(epochs):
start = time.time()
model.train()
train_losses = []
train_preds = []
train_targets = []
for input_ids, token_type_ids, attention_mask, targets in tqdm(dataloader, total=len(dataloader)) :
input_ids = input_ids.to(device)
 token_type_ids = token_type_ids.to(device)
 attention_mask = attention_mask.to(device)
 targets = targets.to(device)
 outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 train_preds.extend(outputs.detach().sigmoid().cpu().numpy())
 train_targets.extend(targets.detach().cpu().numpy())
 loss = compute_loss(outputs, targets)
 model.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
train_losses.append(loss.detach().cpu().item())
 model.eval()
valid_losses = []
valid_preds = []
valid_targets = []
with torch.no_grad() :
for input_ids, token_type_ids, attention_mask, targets in tqdm(valid_dataloader, total=len(valid_dataloader)) :
input_ids = input_ids.to(device)
 token_type_ids = token_type_ids.to(device)
 attention_mask = attention_mask.to(device)
 targets = targets.to(device)
 outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 valid_preds.extend(outputs.sigmoid().cpu().numpy())
 valid_targets.extend(targets.cpu().numpy())
 loss = compute_loss(outputs, targets)
 valid_losses.append(loss.detach().cpu().item())
 valid_predictions.append(np.stack(valid_preds))
test_preds = []
for input_ids, token_type_ids, attention_mask in tqdm(test_dataloader, total=len(test_dataloader)) :
input_ids = input_ids.to(device)
 token_type_ids = token_type_ids.to(device)
 attention_mask = attention_mask.to(device)
 outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
 test_preds.extend(outputs.sigmoid().cpu().numpy())
 test_predictions.append(np.stack(test_preds))
print()
print("Epoch {}: Train Loss {}, Valid Loss {}".format(epoch + 1, np.mean(train_losses), np.mean(valid_losses)))
 print("\t Train Spearmanr {:.4f}, Valid Spearmanr(avg){:.4f}, Valid Spearmanr(last){:.4f}".format(
 compute_spearmanr(np.stack(train_targets), np.stack(train_preds)) ,
compute_spearmanr(np.stack(valid_targets), sum(valid_predictions)/ len(valid_predictions)) ,
compute_spearmanr(np.stack(valid_targets), valid_predictions[-1])
))
print("\t elapsed: {}s".format(time.time() - start))
return valid_predictions, test_predictions<define_search_model>
params = dict(
 max_depth = [n for n in range(9,15)],
min_samples_split = [n for n in range(4, 11)],
min_samples_leaf = [n for n in range(2, 5)],
n_estimators = [n for n in range(10, 60, 10)],
)
class Fold(object):
def __init__(self, n_splits=5, shuffle=True, random_state=71):
self.n_splits = n_splits
self.shuffle = shuffle
self.random_state = random_state
def get_groupkfold(self, train, group_name):
group = train[group_name]
unique_group = group.unique()
kf = KFold(
 n_splits=self.n_splits,
shuffle=self.shuffle,
random_state=self.random_state 
)
 folds_ids = []
for trn_group_idx, val_group_idx in kf.split(unique_group):
trn_group = unique_group[trn_group_idx]
val_group = unique_group[val_group_idx]
is_trn = group.isin(trn_group)
 is_val = group.isin(val_group)
 trn_idx = train[is_trn].index
val_idx = train[is_val].index
folds_ids.append(( trn_idx, val_idx))
return folds_ids<statistical_test>
model_forest = RandomForestClassifier()
gkf = Fold(n_splits=3, shuffle=True, random_state=71)
 fold_ids = gkf.get_groupkfold(df_train, group_name="url")
 for train_idx, valid_idx in fold_ids:
print(( df_train.loc[train_idx, "question_type_spelling"] > 0 ).sum())
 print(( df_train.loc[valid_idx, "question_type_spelling"] > 0 ).sum() )<create_dataframe>
forest_gs = GridSearchCV(param_grid=params,
estimator=model_forest,
cv=5)
 forest_gs.fit(dtrain_X,train_Y )
histories = []
test_dataset = torch.utils.data.TensorDataset(*test_inputs)
 q_test_dataset = torch.utils.data.TensorDataset(*test_question_only_inputs)
 for fold,(train_idx, valid_idx)in enumerate(fold_ids):
gc.collect()
train_inputs = [inputs[i][train_idx] for i in range(3)]
q_train_inputs = [question_only_inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
train_dataset = torch.utils.data.TensorDataset(*train_inputs, train_outputs)
 q_train_dataset = torch.utils.data.TensorDataset(*q_train_inputs, train_outputs)
 valid_inputs = [inputs[i][valid_idx] for i in range(3)]
q_valid_inputs = [question_only_inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
valid_dataset = torch.utils.data.TensorDataset(*valid_inputs, valid_outputs)
 q_valid_dataset = torch.utils.data.TensorDataset(*q_valid_inputs, valid_outputs)
 history = train_and_predict(
 train_data=train_dataset,
valid_data=valid_dataset,
test_data=test_dataset,
q_train_data=q_train_dataset,
q_valid_data=q_valid_dataset,
q_test_data=q_test_dataset,
q_epochs=3, epochs=3, batch_size=8, fold=fold 
)
 histories.append(history )<define_variables>
forest_gs.best_score_
val_preds_list = []
n_epochs = len(histories[0][0])
 for epoch in range(n_epochs):
val_preds_one_epoch = np.zeros([len(df_train), 30])
 for fold,(train_idx, valid_idx)in enumerate(fold_ids):
val_pred = histories[fold][0][epoch]
val_preds_one_epoch[valid_idx, :] += val_pred
val_preds_list.append(val_preds_one_epoch )<define_variables>
forest_gs.best_estimator_
oof_predictions = np.zeros(( n_epochs, len(df_train), len(output_categories)) , dtype=np.float32)
 for j, name in enumerate(output_categories):
for epoch in range(n_epochs):
col = "{}_{}".format(epoch, name)
 oof_predictions[epoch, :, j] = val_preds_list[epoch][:, j]
oof_predictions.shape<define_variables>
p = forest_gs.predict(dval_X)
 print(mean_absolute_error(p,val_Y))
test_preds_list = []
for epoch in range(n_epochs):
test_preds_one_epoch = 0
for fold in range(len(fold_ids)) :
test_preds = histories[fold][1][epoch]
test_preds_one_epoch += test_preds
test_preds_one_epoch = test_preds_one_epoch / len(fold_ids)
 test_preds_list.append(test_preds_one_epoch )<define_variables>
prediction_Random_forest = forest_gs.predict(X_test)
 prediction_Random_forest
<find_best_model_class><EOS> 
output = pd.DataFrame({"PassengerId":passengerid , "Survived" : prediction_Random_forest})
 output.to_csv("Submission5.csv",index = False )