Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<compute_test_metric>
%matplotlib inline
py.init_notebook_mode(connected=True)
 @contextmanager
def timer(title):
t0 = time.time()
yield
print("{} - done in {:.0f}s".format(title, time.time() - t0))
warnings.filterwarnings('ignore' )
def lgb_compute_spearmanr(preds, trues):
rhos = spearmanr(trues.get_label() , preds ).correlation
return "spearmanr", rhos, True
def compute_spearmanr_each_col(trues, preds, n_bins=None):
if n_bins:
preds = binning_output(preds, n_bins)
 rhos = spearmanr(trues, preds ).correlation
return rhos<train_model>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
class LightGBM(Base_Model):
def fit(self, x_train, y_train, x_valid, y_valid, config):
d_train = lgb.Dataset(x_train, label=y_train)
 d_valid = lgb.Dataset(x_valid, label=y_valid)
 lgb_model_params = config["model"]["model_params"]
lgb_train_params = config["model"]["train_params"]
model = lgb.train(
 params=lgb_model_params,
train_set=d_train,
valid_sets=[d_valid],
valid_names=['valid'],
feval=lgb_compute_spearmanr,
**lgb_train_params 
)
 best_score = dict(model.best_score)
 return model, best_score
def full_train(self, x_train, y_train, config, iteration):
d_train = lgb.Dataset(x_train, label=y_train)
 lgb_model_params = config["model"]["model_params"]
model = lgb.train(
 params=lgb_model_params,
train_set=d_train,
feval=lgb_compute_spearmanr,
num_boost_round=int(iteration)
)
 best_score = dict(model.best_score)
 return model, best_score
def get_best_iteration(self, model):
return model.best_iteration
def predict(self, model, features):
return model.predict(features)
 def get_feature_importance(self, model):
return model.feature_importance(importance_type='gain' )<init_hyperparams>
train['Type'] = 'train'
test['Type'] = 'test'
data = train.append(test)
 data.shape
config = {
"model": {
"name": "lightgbm",
"model_params": {
"boosting_type": "gbdt",
"objective": "rmse",
"tree_learner": "serial",
"learning_rate": 0.1,
"max_depth": 1,
"seed": 71,
"bagging_seed": 71,
"feature_fraction_seed": 71,
"drop_seed": 71,
"verbose": -1
},
"train_params": {
"num_boost_round": 5000,
"early_stopping_rounds": 200,
"verbose_eval": 500
}
}
}
outputs = compute_output_arrays(df_train)
 oof_preds_list = []
test_preds_list = []
for i_col in range(len(output_categories)) :
y_train = outputs[:, i_col]
x_train = pd.DataFrame(np.concatenate([oof_predictions[:, :, i].T for i in range(30)], axis=1))
x_test = pd.DataFrame(np.concatenate([test_predictions[:, :, i].T for i in range(30)], axis=1))
feature_name = x_train.columns
model = LightGBM()
models, oof_preds, test_preds, feature_importance, evals_results = model.cv(
 y_train, x_train, x_test, feature_name, fold_ids, config 
)
 oof_preds_list.append(oof_preds.reshape(-1, 1))
test_preds_list.append(test_preds.reshape(-1, 1))
print(i_col, output_categories[i_col])
 print(compute_spearmanr_each_col(oof_preds, y_train))
print(len(oof_preds), len(np.unique(oof_preds)))
 print(len(test_preds), len(np.unique(test_preds)) )<compute_train_metric>
data['Title'] = data['Name']
for name_string in data['Name']:
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=True )
def compute_spearmanr(trues, preds, n_bins=None):
rhos = []
if n_bins:
preds = binning_output(preds, n_bins)
 for col_trues, col_pred in zip(trues.T, preds.T):
if len(np.unique(col_pred)) == 1:
col_pred[np.random.randint(0, len(col_pred)- 1)] = col_pred.max() + 1
rhos.append(spearmanr(col_trues, col_pred ).correlation)
 return np.mean(rhos)
 oof_preds_fi = np.concatenate(oof_preds_list, axis=1)
 print(compute_spearmanr(outputs, oof_preds_fi))<save_to_csv>
mapping = {'Mlle': 'Miss',
'Ms': 'Miss',
'Mme': 'Mrs',
'Major': 'Other',
'Col': 'Other',
'Dr' : 'Other',
'Rev' : 'Other',
'Capt': 'Other',
'Jonkheer': 'Royal',
'Sir': 'Royal',
'Lady': 'Royal',
'Don': 'Royal',
'Countess': 'Royal',
'Dona': 'Royal'}
data.replace({'Title': mapping}, inplace=True)
 titles = ['Miss', 'Mr', 'Mrs', 'Royal', 'Other', 'Master']
test_preds_fi = np.concatenate(test_preds_list, axis=1)
 sub.iloc[:, 1:] = test_preds_fi
sub.to_csv('submission.csv', index=False )<install_modules>
for title in titles:
age_to_impute = data.groupby('Title')['Age'].median() [titles.index(title)]
data.loc[(data['Age'].isnull())&(data['Title'] == title), 'Age'] = age_to_impute
!pip install.. /input/sacremoses/sacremoses-master/
!pip install.. /input/transformers/transformers-master/<define_variables>
data['Family_Size'] = data['Parch'] + data['SibSp'] + 1
data.loc[:,'FsizeD']='Alone'
data.loc[(data['Family_Size']>1),'FsizeD']='Small'
data.loc[(data['Family_Size']>4),'FsizeD']='Big'
DATA_DIR = '.. /input/google-quest-challenge'<load_from_csv>
data[data["Fare"].isnull() ]
sub = pd.read_csv(f'{DATA_DIR}/sample_submission.csv')
 sub.head()<load_from_csv>
fa = data[data["Pclass"]==3]
data['Fare'].fillna(fa['Fare'].median() , inplace = True )
train = pd.read_csv(f'{DATA_DIR}/train.csv')
 train.head()<load_from_csv>
data.loc[:,'Child']=1
data.loc[(data['Age']>=18),'Child']=0
test = pd.read_csv(f'{DATA_DIR}/test.csv')
 test.head()<define_variables>
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 DEFAULT_SURVIVAL_VALUE = 0.5
data['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data.loc[data['Family_Survival']!=0.5].shape[0] )
MAX_LEN = 512
SEP_TOKEN_ID = 102
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, df, model_type="bert-base-cased", max_len=512, content="Question_Answer", train_mode=True, labeled=True):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.max_len = max_len
self.content = content
bert_tokenizer_path = '.. /input/pretrained-bert-models-for-pytorch/' + model_type + '-vocab.txt'
xlnet_tokenizer_path = '.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-spiece.model'
roberta_tokenizer_path = '.. /input/roberta-transformers-pytorch/roberta-base/vocab.json'
roberta_tokenizer_merges_file = '.. /input/roberta-transformers-pytorch/roberta-base/merges.txt'
if model_type == "bert-base-uncased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "bert-base-cased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "xlnet-base-cased":
self.tokenizer = XLNetTokenizer.from_pretrained(xlnet_tokenizer_path)
 elif model_type == "roberta-base":
self.tokenizer = RobertaTokenizer(vocab_file=roberta_tokenizer_path, merges_file=roberta_tokenizer_merges_file)
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids = self.get_token_ids(row)
 if self.labeled:
labels = self.get_label(row)
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def __len__(self):
return len(self.df)
 def select_tokens(self, tokens, max_num):
if len(tokens)<= max_num:
return tokens
if self.train_mode:
num_remove = len(tokens)- max_num
remove_start = random.randint(0, len(tokens)-num_remove-1)
 return tokens[:remove_start] + tokens[remove_start + num_remove:]
else:
return tokens[:max_num//2] + tokens[-(max_num - max_num//2):]
def trim_input_single_content(self, title, content, max_sequence_length=512,
t_max_len=30, c_max_len=512-30-4, num_token=3):
t = self.tokenizer.tokenize(title)
 c = self.tokenizer.tokenize(content)
 t_len = len(t)
 c_len = len(c)
 if(t_len+c_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
c_max_len = c_max_len + floor(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if c_max_len > c_len:
c_new_len = c_len
else:
c_new_len = c_max_len
if t_new_len+c_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be less or equal than %d, but is %d"
%(max_sequence_length,(t_new_len+c_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(c)- c_new_len > 0:
c = c[:c_new_len//4] + c[len(c)-c_new_len+c_new_len//4:]
else:
c = c[:c_new_len]
if(len(t)+ len(c)+ num_token > max_sequence_length):
more_token = len(t)+ len(c)+ num_token - max_sequence_length
c = c[:(len(c)-more_token)]
return t, c
def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN,
t_max_len=30, q_max_len=239, a_max_len=239, num_token=4):
t = self.tokenizer.tokenize(title)
 q = self.tokenizer.tokenize(question)
 a = self.tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(q)- q_new_len > 0:
q = q[:q_new_len//4] + q[len(q)-q_new_len+q_new_len//4:]
else:
q = q[:q_new_len]
if len(a)- a_new_len > 0:
a = a[:a_new_len//4] + a[len(a)-a_new_len+a_new_len//4:]
else:
a = a[:a_new_len]
return t, q, a
def get_token_ids(self, row):
num_token = 4
if self.content == "Question":
num_token -= 1
elif self.content == "Answer":
num_token -= 1
if self.content == "Question_Answer":
t_max_len=30
q_max_len=int(( self.max_len-t_max_len-num_token)/2)
 a_max_len=(self.max_len-t_max_len - num_token - int(( self.max_len-t_max_len-num_token)/2))
elif self.content == "Question":
t_max_len=30
q_max_len=self.max_len-t_max_len-num_token
a_max_len=0
elif self.content == "Answer":
t_max_len=30
q_max_len=0
a_max_len=self.max_len-t_max_len-num_token
else:
raise NotImplementedError
if self.content == "Question_Answer":
t_tokens, q_tokens, a_tokens = self.trim_input(row.question_title, row.question_body, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, q_max_len=q_max_len, a_max_len=a_max_len, num_token=num_token)
 elif self.content == "Question":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.question_body, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=q_max_len, num_token=num_token)
 elif self.content == "Answer":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=a_max_len, num_token=num_token)
 else:
raise NotImplementedError
if self.content == "Question_Answer":
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + q_tokens + ['[SEP]'] + a_tokens + ['[SEP]']
elif(( self.content == "Question")or(self.content == "Answer")) :
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + c_tokens + ['[SEP]']
else:
raise NotImplementedError
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 if len(token_ids)< self.max_len:
token_ids += [0] *(self.max_len - len(token_ids))
ids = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(ids)
 return ids, seg_ids
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_sep = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == self.tokenizer.sep_token_id:
if first_sep:
first_sep = False
else:
seg_idx = 1
pad_idx = torch.nonzero(ids == 0)
 seg_ids[pad_idx] = 0
return seg_ids
def get_label(self, row):
return torch.tensor(row[TARGET_COLUMNS].values.astype(np.float32))
def collate_fn(self, batch):
token_ids = torch.stack([x[0] for x in batch])
 seg_ids = torch.stack([x[1] for x in batch])
 if self.labeled:
labels = torch.stack([x[2] for x in batch])
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def get_test_loader(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4):
df = pd.read_csv(f'{DATA_DIR}/test.csv')
 ds_test = QuestDataset(df, model_type, max_len=max_len, content=content, train_mode=False, labeled=False)
 loader = torch.utils.data.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=2, collate_fn=ds_test.collate_fn, drop_last=False)
 loader.num = len(df)
 return loader, ds_test.tokenizer
def get_train_val_loaders(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4, val_batch_size=4, ifold=0):
df = pd.read_csv(f'{DATA_DIR}/train.csv')
 df = shuffle(df, random_state=42)
 gkf = GroupKFold(n_splits=5 ).split(X=df.question_body, groups=df.question_body)
 for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold == ifold:
df_train = df.iloc[train_idx]
df_val = df.iloc[valid_idx]
break
print(df_train.shape)
 print(df_val.shape)
 ds_train = QuestDataset(df_train, model_type, max_len=max_len, content=content)
 train_loader = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True, num_workers=2, collate_fn=ds_train.collate_fn, drop_last=True)
 train_loader.num = len(df_train)
 ds_val = QuestDataset(df_val, model_type, max_len=max_len, content=content, train_mode=False)
 val_loader = torch.utils.data.DataLoader(ds_val, batch_size=val_batch_size, shuffle=False, num_workers=2, collate_fn=ds_val.collate_fn, drop_last=False)
 val_loader.num = len(df_val)
 val_loader.df = df_val
return train_loader, val_loader, ds_train.tokenizer
def test_train_loader() :
loader, _, _ = get_train_val_loaders("xlnet-base-cased", 512, "Question", 4, 4, 1)
 for ids, seg_ids, labels in loader:
print(ids)
 print(seg_ids.numpy())
 print(labels)
 break
def test_test_loader() :
loader, _ = get_test_loader("roberta-base", 512, "Question", 4)
 for ids, seg_ids in loader:
print(ids)
 print(seg_ids)
 break<load_pretrained>
for _, grp_df in data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data[data['Family_Survival']!=0.5].shape[0]))
test_test_loader()<load_pretrained>
data = data.drop(columns = [
'Age',
'Cabin',
'Embarked',
'Name',
'Last_Name',
'Parch',
'SibSp',
'Ticket',
'Family_Size',
])
 data.head()
test_train_loader()<load_pretrained>
target_col = ["Survived"]
id_dataset = ["Type"]
cat_cols = data.nunique() [data.nunique() < 12].keys().tolist()
cat_cols = [x for x in cat_cols ]
num_cols = [x for x in data.columns if x not in cat_cols + target_col + id_dataset]
bin_cols = data.nunique() [data.nunique() == 2].keys().tolist()
multi_cols = [i for i in cat_cols if i not in bin_cols]
le = LabelEncoder()
for i in bin_cols :
data[i] = le.fit_transform(data[i])
 data = pd.get_dummies(data = data,columns = multi_cols)
 std = StandardScaler()
scaled = std.fit_transform(data[num_cols])
 scaled = pd.DataFrame(scaled,columns=num_cols)
 df_data_og = data.copy()
data = data.drop(columns = num_cols,axis = 1)
 data = data.merge(scaled,left_index=True,right_index=True,how = "left")
 data = data.drop(columns = ['PassengerId'],axis = 1 )
class QuestModel(nn.Module):
def __init__(self, model_type="xlnet-base-cased", tokenizer=None, n_classes=30, hidden_layers=[-1, -3, -5, -7, -9]):
super(QuestModel, self ).__init__()
self.model_name = 'QuestModel'
self.model_type = model_type
self.hidden_layers = hidden_layers
if model_type == "bert-base-uncased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
if n_classes == 30:
bert_config.hidden_dropout_prob = 0.1
else:
bert_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "bert-base-cased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-cased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "xlnet-base-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-base-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "xlnet-large-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-large-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "roberta-base":
roberta_model_config = '.. /input/roberta-transformers-pytorch/roberta-base/config.json'
roberta_config = RobertaConfig.from_json_file(roberta_model_config)
 roberta_config.output_hidden_states = True
roberta_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/roberta-transformers-pytorch/roberta-base/pytorch_model.bin')
 self.roberta_model = RobertaModel.from_pretrained(model_path, config=roberta_config)
 self.roberta_model.resize_token_embeddings(len(tokenizer))
if model_type == "bert-base-uncased":
self.hidden_size = 768
elif model_type == "bert-large-uncased":
self.hidden_size = 1024
elif model_type == "bert-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-large-cased":
self.hidden_size = 1024
elif model_type == "roberta-base":
self.hidden_size = 768
else:
raise NotImplementedError
self.fc_1 = nn.Linear(self.hidden_size * len(hidden_layers), self.hidden_size)
 self.fc = nn.Linear(self.hidden_size, n_classes)
 self.selu = nn.SELU()
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
self.dropouts = nn.ModuleList([
nn.Dropout(0.5)for _ in range(5)
 ])
 def forward(self, ids, seg_ids):
attention_mask =(ids > 0)
 if(( self.model_type == "bert-base-uncased")\
or(self.model_type == "bert-base-cased")\
or(self.model_type == "bert-large-uncased")\
or(self.model_type == "bert-large-cased")) :
outputs = self.bert_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(( self.model_type == "xlnet-base-cased")\
or(self.model_type == "xlnet-large-cased")) :
attention_mask = attention_mask.float()
outputs = self.xlnet_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[1]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(self.model_type == "roberta-base"):
attention_mask = attention_mask.float()
outputs = self.roberta_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
for j, dropout in enumerate(self.dropouts):
if j == 0:
logit = self.fc(dropout(h))
else:
logit += self.fc(dropout(h))
return logit / len(self.dropouts)
 def test_model(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9]):
x = torch.tensor([[1,2,3,4,5, 0, 0], [1,2,3,4,5, 0, 0]])
 seg_ids = torch.tensor([[0,0,0,0,0, 0, 0], [0,0,0,0,0, 0, 0]])
 model = QuestModel(model_type=model_type, hidden_layers=hidden_layers)
 y = model(x, seg_ids)
 print(y )<train_model>
train = data[data['Type'] == 1]
test = data[data['Type'] == 0]
train = train.drop(columns = ['Type'])
 test = test.drop(columns = ['Type'] )
test_model(model_type="bert-base-cased", hidden_layers=[-3, -4, -5, -6, -7] )<load_pretrained>
X = train.drop('Survived', 1)
 y = train['Survived']
X_test = test
X_test = X_test.drop(columns = ['Survived'
] )
def create_bert_base_uncased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-uncased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertuncasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugquestionanswerswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=21, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=9, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=30, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabaseaugdiffswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_question_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_answer_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models<find_best_params>
random_state = 42
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = 0.2, random_state = random_state )
def predict(models, test_loader):
all_scores = []
with torch.no_grad() :
for ids, seg_ids in tqdm(test_loader, total=test_loader.num // test_loader.batch_size):
ids, seg_ids = ids.cuda() , seg_ids.cuda()
scores = []
for model in models:
model = model.cuda()
outputs = torch.sigmoid(model(ids, seg_ids)).cpu()
scores.append(outputs)
 all_scores.append(torch.mean(torch.stack(scores), 0))
all_scores = torch.cat(all_scores, 0 ).numpy()
return all_scores<load_pretrained>
fit_params = {"early_stopping_rounds" : 100,
"eval_metric" : 'auc',
"eval_set" : [(X_train,y_train)],
'eval_names': ['valid'],
'verbose': 0,
'categorical_feature': 'auto'}
param_test = {'learning_rate' : [0.01, 0.02, 0.03, 0.04, 0.05, 0.08, 0.1, 0.2, 0.3, 0.4],
'n_estimators' : [100, 200, 300, 400, 500, 600, 800, 1000, 1500, 2000],
'num_leaves': sp_randint(6, 50),
'min_child_samples': sp_randint(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': sp_uniform(loc=0.2, scale=0.8),
'max_depth': [-1, 1, 2, 3, 4, 5, 6, 7],
'colsample_bytree': sp_uniform(loc=0.4, scale=0.6),
'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100]}
n_iter = 500
lgbm_clf = lgbm.LGBMClassifier(random_state=random_state, silent=True, metric='None', n_jobs=4)
 grid_search = RandomizedSearchCV(
 estimator=lgbm_clf, param_distributions=param_test,
n_iter=n_iter,
scoring='accuracy',
cv=5,
refit=True,
random_state=random_state,
verbose=True)
 grid_search.fit(X, y, **fit_params)
 print('Best score reached: {} with params: {} '.format(grid_search.best_score_, grid_search.best_params_))
opt_parameters = grid_search.best_params_
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Question_Answer", batch_size=32 )<predict_on_test>
%%time
lgbm_clf = lgbm.LGBMClassifier(**opt_parameters)
 lgbm_clf.fit(X_train, y_train)
 y_pred = lgbm_clf.predict(X_valid)
 y_score = lgbm_clf.predict_proba(X_valid)[:,1]
model_performance('lgbm_clf' )
roberta_base_models = create_roberta_base_models(tokenizer)
 roberta_base_preds = predict(roberta_base_models, test_loader )<set_options>
lgbm_clf = lgbm.LGBMClassifier(**opt_parameters)
 lgbm_clf.fit(X, y)
 y_pred = lgbm_clf.predict(X_test )
del roberta_base_models, test_loader
torch.cuda.empty_cache()
gc.collect()<load_pretrained>
visualizer = DiscriminationThreshold(lgbm_clf)
 visualizer.fit(X, y)
 visualizer.poof()
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", batch_size=32 )<predict_on_test>
temp = pd.DataFrame(pd.read_csv(".. /input/test.csv")['PassengerId'])
 temp['Survived'] = y_pred
temp.to_csv(".. /working/submission.csv", index = False )