Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<train_model>
! pip install.. /input/sacremoses/sacremoses-master/
! pip install.. /input/transformers/transformers-master/
selector_lr1 = RFECV(logreg1, min_features_to_select=12, scoring='roc_auc', step=15, verbose=0, cv=repfold, n_jobs=-1)
 selector_lr1.fit(X_train,y_train)
<train_model>
!pip install.. /input/tensorflowv1/tensorboard-1.15.0-py3-none-any.whl
!pip install.. /input/tensorflowv1/tensorflow_estimator-1.15.1-py2.py3-none-any.whl
!pip install.. /input/tensorflowv1/tensorflow-1.15.0-cp36-cp36m-manylinux2010_x86_64.whl
perm = PermutationImportance(logreg1, random_state=RANDOM_STATE ).fit(X_train, y_train)
 eli5.show_weights(perm, top=10 )<train_on_grid>
PATH = '.. /input/google-quest-challenge/'
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories)
 MAX_LEN = 512
SEP_TOKEN_ID = 102
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
configPath = '.. /input/bert-model-config/'
lass0.fit(X_train, y_train)
 sc = cross_val_score(lass0, X_train, y_train, scoring='roc_auc', cv=repfold, n_jobs=-1, verbose=1)
 print(sc.mean() )<train_model>
use_cuda = torch.cuda.is_available()
if use_cuda:
print('GPU available!')
 print(torch.cuda.get_device_name(0))
else:
print('GPU not available')
 device = torch.device("cuda:0" if use_cuda else "cpu")
perm = PermutationImportance(lass0, random_state=RANDOM_STATE ).fit(X_train, y_train)
 eli5.show_weights(perm, top=10 )<define_search_space>
def format_time(elapsed_time):

 elapsed_rounded = int(round(elapsed_time))
return str(datetime.timedelta(seconds=elapsed_rounded))
param_lass = {
'alpha' : [0.022, 0.021, 0.02, 0.019, 0.023, 0.024, 0.025, 0.026, 0.027, 0.029, 0.031],
'tol' : [0.0013, 0.0014, 0.001, 0.0015, 0.0011, 0.0012, 0.0016, 0.0017]
}<train_on_grid>
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, df, train_mode=True, labeled=True, target_columns=output_categories):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'/assets/vocab.txt', do_lower_case=True)
 self.target_columns = target_columns
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids = self.get_token_ids(row)
 if self.labeled:
labels = self.get_labels(row, self.target_columns)
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def get_token_ids(self, row):
title_tokens, ques_tokens, ans_tokens = self.trim_input(row.question_title, row.question_body, row.answer)
 tokens = ['[CLS]'] + title_tokens + ['[SEP]'] + ques_tokens + ['[SEP]'] + ans_tokens + ['[SEP]']
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 if(len(token_ids)) < MAX_LEN:
token_ids += [0] *(MAX_LEN - len(token_ids))
token_ids_torch = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(token_ids_torch)
 return token_ids_torch, seg_ids
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_step = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == SEP_TOKEN_ID:
if first_step:
first_step = False
else:
seg_idx = 1
pad_idx = torch.nonzero(ids == 0)
 seg_ids[pad_idx] = 0
return seg_ids
def get_labels(self, row, target_columns):
return torch.tensor(row[target_columns].values.astype(np.float32))
def trim_input(self, title, question, answer, max_seq_length=MAX_LEN,
title_max_len=30, ques_max_len=239, ans_max_len=239,
head_len=120):
title_tokens = self.tokenizer.tokenize(title)
 ques_tokens = self.tokenizer.tokenize(question)
 ans_tokens = self.tokenizer.tokenize(answer)
 title_len = len(title_tokens)
 ques_len = len(ques_tokens)
 ans_len = len(ans_tokens)
 if(title_len + ques_len + ans_len + 4)> max_seq_length:
if title_max_len > title_len:
final_title_len = title_len
ans_max_len = int(ans_max_len + np.ceil(( title_max_len - title_len)/ 2))
ques_max_len = int(ques_max_len + np.floor(( title_max_len - title_len)/ 2))
else:
final_title_len = title_max_len
if ans_max_len > ans_len:
final_ans_len = ans_len
final_ques_len = ques_max_len +(ans_max_len - ans_len)
 elif ques_max_len > ques_len:
final_ques_len = ques_len
final_ans_len = ans_max_len +(ques_max_len - ques_len)
 else:
final_ans_len = ans_max_len
final_ques_len = ques_max_len
if final_ans_len + final_ques_len + final_title_len + 4 != max_seq_length:
print('Token Sequence Length:', final_ans_len + final_ques_len + final_title_len + 4)
 raise ValueError('New sequence length does not match.')
 title_tokens = title_tokens[:final_title_len]
if ques_len > final_ques_len:
max_head_len = max(head_len, int(0.5 * final_ques_len))
ques_tokens = ques_tokens[:max_head_len] + ques_tokens[max_head_len - final_ques_len:]
else:
ques_tokens = ques_tokens[:final_ques_len]
if ans_len > final_ans_len:
max_head_len = max(head_len, int(0.5 * final_ans_len))
ans_tokens = ans_tokens[:max_head_len] + ans_tokens[max_head_len - final_ans_len:]
else:
ans_tokens = ans_tokens[:final_ans_len]
return title_tokens, ques_tokens, ans_tokens
grid_lass = GridSearchCV(estimator = lass0, param_grid = param_lass , scoring = 'roc_auc', verbose = 1, n_jobs = -1, cv=repfold)
 grid_lass.fit(X_train,y_train)
 print("Best Score:" + str(grid_lass.best_score_))
print("Best Parameters: " + str(grid_lass.best_params_))<train_on_grid>
params = {'batch_size': 8,
'shuffle': False,
'num_workers': 6}
best_parameters_lass = grid_lass.best_params_
lass = Lasso(**best_parameters_lass)
 lass.fit(X_train,y_train)
 sc = cross_val_score(lass, X_train, y_train, scoring='roc_auc', cv=repfold, n_jobs=-1, verbose=1)
 print(sc.mean())
 selector_lass = RFECV(lass, min_features_to_select=12, scoring='roc_auc', step=15, verbose=0, cv=repfold, n_jobs=-1)
 selector_lass.fit(X_train,y_train)
<compute_train_metric>
def prediction_v7(test_dataloader, model, classifier, shape, batch_size=8):
print("Running Prediction...")
 t0 = time.time()
model.eval()
classifier.eval()
test_preds = np.zeros(( shape, 30))
for step, batch in enumerate(test_dataloader):
batch = tuple(t.to(device)for t in batch)
 test_input_ids, test_input_mask = batch
with torch.no_grad() :
attention_mask =(test_input_ids > 0)
 last_layer, _ , hidden_layers= model(input_ids=test_input_ids,
token_type_ids=test_input_mask,
attention_mask=attention_mask)
 cls = torch.cat(tuple(hidden_layers[i][:, 0, :] for i in [-1, -2, -3, -4]), dim = -1)
 prediction = classifier(cls)
 test_preds[step * batch_size:(step + 1)* batch_size] = prediction.detach().cpu().numpy()
print(" Prediction took: {:}".format(format_time(time.time() - t0)))
 return test_preds
def acc_model(params):
clf = Lasso(**params)
 return cross_val_score(clf, X_train, y_train, scoring='roc_auc', cv=repfold ).mean()<find_best_params>
def prediction(test_dataloader, model, classifier, shape, batch_size=8):
print("Running Prediction...")
 t0 = time.time()
model.eval()
classifier.eval()
test_preds = np.zeros(( shape, 30))
for step, batch in enumerate(test_dataloader):
batch = tuple(t.to(device)for t in batch)
 test_input_ids, test_input_mask= batch
with torch.no_grad() :
attention_mask =(test_input_ids > 0)
 last_layer, _ , hidden_layers= model(input_ids=test_input_ids,
token_type_ids=test_input_mask,
attention_mask=attention_mask)
 outputs = torch.cat(tuple(hidden_layers[i] for i in [-1, -2, -3, -4]), dim = -1)
 pooling_layer = torch.nn.AdaptiveAvgPool2d(( 1, 3072))
cls = pooling_layer(outputs ).squeeze()
prediction = classifier(cls)
 test_preds[step * batch_size:(step + 1)* batch_size] = prediction.detach().cpu().numpy()
print(" Prediction took: {:}".format(format_time(time.time() - t0)))
 return test_preds
space4lass = {'alpha' : hp.uniform('alpha',.01, 1),
'tol' : hp.uniform('tol',.001, 0.1),
'random_state': RANDOM_STATE,
'max_iter': hp.choice('max_iter', [50000]),
}
best = 0
pr = []
def f(params):
global best
acc = acc_model(params)
 if acc > best:
best = acc
print('new best:', best, params)
 pr.append(params)
 return {'loss': -acc, 'status': STATUS_OK}
trials = Trials()
best = fmin(f, space4lass, algo=tpe.suggest, max_evals=3000, trials=trials)
 print('best for lasso: ')
 print(pr[-1] )<train_on_grid>
ds_test = QuestDataset(df_test, labeled=False)
 test_dataloader = torch.utils.data.DataLoader(ds_test, **params )
print(pr[-1])
 best_lass = pr[-1]
lass1 = Lasso(**best_lass)
 lass1.fit(X_train,y_train)
 sc = cross_val_score(lass1, X_train, y_train, scoring='roc_auc', cv=repfold, n_jobs=-1, verbose=1)
 print(sc.mean())
<train_model>
config = config = BertConfig.from_pretrained(configPath)
 config.output_hidden_states = True
selector_lass1 = RFECV(lass1, min_features_to_select=12, scoring='roc_auc', step=15, verbose=0, cv=repfold, n_jobs=-1)
 selector_lass1.fit(X_train,y_train )<train_model>
test_predictions = []
perm = PermutationImportance(lass1, random_state=RANDOM_STATE ).fit(X_train, y_train)
 eli5.show_weights(perm, top=10 )<split>
for i in range(5):
model_path = f'.. /input/google-quest-pytorch-v7/bert-base-{i}.h5py'
classifier_path = f'.. /input/google-quest-pytorch-v7/bert-base-classifier-{i}.h5py'
bert_model_weights = torch.load(model_path)
 model = BertModel.from_pretrained(configPath, state_dict = bert_model_weights, config=config)
 classifier.load_state_dict(torch.load(classifier_path))
model.cuda()
classifier.cuda()
test_predictions.append(prediction_v7(test_dataloader, model, classifier, len(df_test)))
 print('Fold Done')
 print(' ' )
def cross_validation(train_, target_, params,
num_folds = 5, repeats = 20, rs = 0):
print(params)
 clfs = []
folds = RepeatedStratifiedKFold(n_splits = num_folds, n_repeats = repeats, random_state = rs)
 valid_pred = pd.DataFrame(index = train_.index)
 for n_fold,(train_idx, valid_idx)in enumerate(folds.split(target_, target_)) :
print('--- Fold {} started at {}'.format(n_fold, time.ctime()))
train_x, train_y = train_.iloc[train_idx], target_.iloc[train_idx]
valid_x, valid_y = train_.iloc[valid_idx], target_.iloc[valid_idx]
clf = LogisticRegression(**params)
 clf.fit(train_x, train_y)
 clfs.append(clf)
 predict = clf.predict_proba(valid_x)[:, 1]
tn, fp, fn, tp = confusion_matrix(valid_y,(predict >=.5)* 1 ).ravel()
auc = roc_auc_score(valid_y, predict)
 acc = accuracy_score(valid_y,(predict >=.5)* 1)
 loss = log_loss(valid_y, predict)
 print('TN =', tn, 'FN =', fn, 'FP =', fp, 'TP =', tp)
 print('AUC = ', auc, 'Loss =', loss, 'Acc =', acc)
 valid_pred[n_fold] = pd.Series(predict, index = valid_x.index)
 del train_x, train_y, valid_x, valid_y, predict
gc.collect()
return clfs, valid_pred
def save_submit(test_, clfs_, filename):
subm = pd.DataFrame(np.zeros(test_.shape[0]), index = test_.index, columns = ['target'])
 for clf in clfs_:
subm['target'] += clf.predict_proba(test_)[:, 1]
subm['target'] /= len(clfs_)
 subm = subm.reset_index()
subm.columns = ['id', 'target']
subm.to_csv(filename, index = False)
<compute_train_metric>
final_predictions = np.mean(test_predictions, axis=0 )
clfs, pred = cross_validation(pd.DataFrame(X_train), y_train, best_lr )<compute_test_metric>
del test_predictions, ds_test, df_test, test_dataloader, model, classifier; gc.collect()
sub_prp(logreg1, filename+'lr1.csv', X_test)
 sub_prp(selector_lr1, filename+'sel_lr1.csv', X_test )<save_to_csv>
torch.cuda.empty_cache()
save_submit(pd.DataFrame(X_test, index=test.id), clfs, filename+'lr_cv_Anna.csv' )<define_variables>
InteractiveShell.ast_node_interactivity = "all"
warnings.filterwarnings('ignore')
invlogit = lambda x: 1/(1 + tt.exp(-x))<load_from_csv>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
df = pd.read_csv('.. /input/train.csv')
 y = np.asarray(df.target)
 X = np.array(df.ix[:, 2:302])
 df2 = pd.read_csv('.. /input/test.csv')
 df2.head()
X2 = np.array(df2.ix[:, 1:301] )<compute_test_metric>
SEED = 1414
DATA_DIR = ".. /input/google-quest-challenge/"
WORK_DIR = ".. /working/"
BERT_VOCAB_PATH = ".. /input/roberta-large/roberta-large-vocab.json"
BERT_MERGES_PATH = ".. /input/roberta-large/roberta-large-merges.txt"
BERT_MODEL_PATH = ".. /input/roberta-large/roberta-large-pytorch_model.bin"
BERT_CONFIG_PATH = ".. /input/roberta-large/roberta-large-config.json"
input_columns = ['question_title', 'question_body', 'answer']
seed_everything(SEED)
 batch_size = 32
def get_model(y, X):
model = pm.Model()
with model:
xi = pm.Bernoulli('xi',.05, shape=X.shape[1])
 alpha = pm.Normal('alpha', mu = 0, sd = 5)
 beta = pm.Normal('beta', mu = 0, sd =.75 , shape=X.shape[1])
 p = pm.math.dot(X, xi * beta)
 y_obs = pm.Bernoulli('y_obs', invlogit(p + alpha), observed=y)
 return model<train_model>
test_df = pd.read_csv(os.path.join(DATA_DIR,"test.csv"))
model1 = get_model(y, X )<create_dataframe>
eng_stopwords = set(stopwords.words("english"))
def add_external_features(df):
eng_stopwords = set(stopwords.words("english"))
df['question_body'] = df['question_body'].apply(lambda x:str(x))
df['question_num_words'] = df.question_body.str.count('\S+')
 df['question_title_num_words'] = df.question_title.str.count('\S+')
 df['answer'] = df['answer'].apply(lambda x:str(x))
df['answer_num_words'] = df.answer.str.count('\S+')
 df["question_title_num_unique_words"] = df["question_title"].apply(lambda x: len(set(str(x ).split())))
 df["question_body_num_unique_words"] = df["question_body"].apply(lambda x: len(set(str(x ).split())))
 df["answer_num_unique_words"] = df["answer"].apply(lambda x: len(set(str(x ).split())))
 df["question_title_num_chars"] = df["question_title"].apply(lambda x: len(str(x)))
 df["question_body_num_chars"] = df["question_body"].apply(lambda x: len(str(x)))
 df["answer_num_chars"] = df["answer"].apply(lambda x: len(str(x)))
 df['qt_words'] = df['question_title'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['q_words'] = df['question_body'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['a_words'] = df['answer'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['qa_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['q_words'], s['a_words'])) , axis = 1)
 df['qt_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['qt_words'], s['a_words'])) , axis = 1)
 df['qa_word_overlap_norm'] = df.apply(lambda s: s['qa_word_overlap']/(len(s['a_words'])+ len(s['q_words'])- s['qa_word_overlap']), axis = 1)
 df['qta_word_overlap_norm'] = df.apply(lambda s: s['qt_word_overlap']/(len(s['a_words'])+ len(s['qt_words'])- s['qt_word_overlap']), axis = 1)
 df.drop(['q_words', 'a_words', 'qt_words'], axis = 1, inplace = True)
 return df
results = pd.DataFrame({'var': np.arange(300),
'inclusion_probability':np.apply_along_axis(np.mean, 0, trace['xi']),
'beta':np.apply_along_axis(np.mean, 0, trace['beta']),
'beta_given_inclusion': np.apply_along_axis(np.sum, 0, trace['xi']*trace['beta'])
 /np.apply_along_axis(np.sum, 0, trace['xi'])
 } )<sort_values>
test_df = add_external_features(test_df )
results.sort_values('inclusion_probability', ascending = False ).head(20 )<compute_test_metric>
handmade_cols = ["question_body_num_unique_words", 'question_num_words',
"question_title_num_unique_words", "question_title_num_words",
"answer_num_unique_words", "answer_num_words"]
with open('/kaggle/input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-2/C_scaler.pickle', mode='rb')as f:
num_words_scaler = pickle.load(f)
 test_df[handmade_cols]= num_words_scaler.transform(test_df[handmade_cols].values)
 train_handmade_features= test_df[handmade_cols + ['qa_word_overlap_norm', 'qta_word_overlap_norm']].values
test_beta = trace['beta'][0]
test_inc = trace['xi'][0]
test_score = expit(trace['alpha'][0] + np.dot(X2, test_inc * test_beta))
test_score<prepare_x_and_y>
def label_encoder(x, dict_reverse):
try:
return dict_reverse[x]
except:
return 0
with open('/kaggle/input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-2/C_category.pickle', mode='rb')as f:
category_dict_reverse = pickle.load(f)
 test_df['category'] = test_df['category'].apply(lambda x: label_encoder(x, category_dict_reverse))
with open('/kaggle/input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-2/C_host.pickle', mode='rb')as f:
host_dict_reverse = pickle.load(f)
 test_df['host'] = test_df['host'].apply(lambda x: label_encoder(x, host_dict_reverse))
n_cat = len(category_dict_reverse)+ 1
cat_emb = 256
n_host = len(host_dict_reverse)+ 1
host_emb = 256
estimate = trace['beta'] * trace['xi']
y_hat = np.apply_along_axis(np.mean, 1, expit(trace['alpha'] + np.dot(X2, np.transpose(estimate))))<compute_test_metric>
bert_config = transformers.RobertaConfig.from_json_file(BERT_CONFIG_PATH)
 bert_config.output_hidden_states = True
tokenizer = transformers.RobertaTokenizer(BERT_VOCAB_PATH,BERT_MERGES_PATH )
np.mean(y_hat), np.sum(results.inclusion_probability/300 )<save_to_csv>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_segments2(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=238, a_max_len=238):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+6)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+6 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+6)))
 t = t[:t_new_len]
q_len_head = round(q_new_len/3)
 q_len_tail = -1*(q_new_len -q_len_head)
 a_len_head = round(a_new_len/3)
 a_len_tail = -1*(a_new_len -a_len_head)
 q = q[:q_len_head]+q[q_len_tail:]
a = a[:a_len_head]+a[a_len_tail:]
return t, q, a
def convert_lines(question, answer, max_sequence_length, tokenizer):
all_tokens = []
all_masks = []
all_segments = []
longer = 0
for q, a in tqdm(zip(question, answer)) :
tokens_q = tokenizer.tokenize(q)
 tokens_a = tokenizer.tokenize(a)
 q_len = len(tokens_q)
 a_len = len(tokens_a)
 if(q_len+a_len+3)> max_sequence_length:
longer += 1
new_q_len = q_len/(a_len+q_len)*(max_sequence_length-3)
 new_a_len = a_len/(q_len+a_len)*(max_sequence_length-3)
 new_q_len, new_a_len = int(np.ceil(new_q_len)) , int(np.floor(new_a_len))
if new_a_len+new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_a_len+new_q_len+3))
tokens_q = tokens_q[:new_q_len]
tokens_a = tokens_a[:new_a_len]
stoken = ["[CLS]"] + tokens_q + ["[SEP]"] + tokens_a + ["[SEP]"]
token_ids = tokenizer.encode(stoken, add_special_tokens=Ture)
 input_ids = token_ids + [0] *(max_sequence_length-len(token_ids))
attention_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 all_tokens.append(input_ids)
 all_masks.append(attention_masks)
 all_segments.append(input_segments)
 print(longer)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
submission = pd.DataFrame({'id':df2.id, 'target':y_hat})
 submission.to_csv('submission.csv', index = False )<load_from_csv>
def convert_lines2(title, question, answer, max_sequence_length, tokenizer, t_max_len_seq=30, q_max_len_seq=238, a_max_len_seq=238):
all_tokens = []
all_masks = []
all_segments = []
longer = 0
for t, q, a in tqdm(zip(title, question, answer)) :
tokens_t, tokens_q, tokens_a = _trim_input(t, q, a, max_sequence_length=max_sequence_length)
 stoken = ["
"] + tokens_t + ["
"] + ["
"] + tokens_q + ["
"] + ["
"] + tokens_a + ["
"]
token_ids = tokenizer.convert_tokens_to_ids(stoken)
 input_ids = token_ids + [0] *(max_sequence_length-len(token_ids))
attention_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 all_tokens.append(input_ids)
 all_masks.append(attention_masks)
 all_segments.append(input_segments)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.shape<prepare_x_and_y>
test_tokens, test_masks, test_segments = convert_lines2(test_df["question_title"],
test_df["question_body"],
test_df["answer"],
max_sequence_length=512,
tokenizer=tokenizer )
X = train.drop(['id', 'target'], axis=1 ).values
y = train['target']<split>
class QuestDataset_test(Dataset):
def __init__(self, token_ids, masks, segments, hosts, categories, handmade_features):
self.token_ids = token_ids
self.masks = masks
self.segments = segments
self.hosts = hosts
self.categories = categories
self.handmades = handmade_features
def __len__(self):
return self.token_ids.shape[0]
def __getitem__(self, idx):
token_id = self.token_ids[idx]
mask = self.masks[idx]
segment = self.segments[idx]
host = self.hosts[idx]
category = self.categories[idx]
handmade = self.handmades[idx]
return [token_id, mask, segment, host, category, handmade]
n_fold = 5
kf = KFold(n_splits=n_fold, shuffle=True, random_state=2)
 scores = []
for train_index, test_index in kf.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y.iloc[train_index], y.iloc[test_index]
clf = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=.1, solver='liblinear')
 model = BaggingClassifier(clf, n_estimators=300, max_samples=.8, max_features=.8, random_state=0)
 model.fit(X_train, y_train)
 y_pred = model.predict_proba(X_test)[:, 1]
scores.append(roc_auc_score(y_test, y_pred))
print(f'CV mean score: {np.mean(scores):.4f}' )<train_model>
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, torch.nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))
X_test = test.drop(['id'], axis=1 ).values
log_reg = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=.1, solver='liblinear')
 model = BaggingClassifier(log_reg, n_estimators=300, max_samples=.8, max_features=.8, random_state=0)
 model.fit(X, y)
 y_pred = model.predict_proba(X_test)[:, 1]<load_from_csv>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
sub = pd.read_csv(".. /input/sample_submission.csv")
 sub['target'] = y_pred<save_to_csv>
class QuestModel6(nn.Module):
def __init__(self, n_cat, cat_emb, n_host, host_emb, num_labels):
super().__init__()
BERT_DIMS = 1024
LSTM_UNITS = 512
model_path = os.path.join(BERT_MODEL_PATH)
 self.bert_model = transformers.RobertaModel.from_pretrained(model_path,config=bert_config)
 set_trainable(self.bert_model, False)
 self.category_embedding = nn.Embedding(n_cat, cat_emb)
 self.host_embedding = nn.Embedding(n_host, host_emb)
 self.embedding_dropout = SpatialDropout(0.5)
 self.lstm1 = nn.LSTM(BERT_DIMS*2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.GRU(LSTM_UNITS*2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.atten1 = Attention(LSTM_UNITS*2, 512)
 self.atten2 = Attention(LSTM_UNITS*2, 512)
 self.dropout1 = nn.Dropout(0.2)
 self.fc1 = nn.Linear(LSTM_UNITS*6 + int(cat_emb)+ int(host_emb)+ 8, 21)
 self.fc2 = nn.Linear(LSTM_UNITS*6 + int(cat_emb)+ int(host_emb)+ 8, 9)
 self._init_weights(self.category_embedding)
 self._init_weights(self.host_embedding)
 self._init_weights(self.fc1)
 self._init_weights(self.fc2)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
print("initailize weight")
 module.weight.data.normal_(mean=0.0, std=0.02)
 if isinstance(module, nn.Linear)and module.bias is not None:
print("initailize bias")
 module.bias.data.zero_()
def forward(self, token_ids, masks, segment, hosts, categories, handmades):
category_embed = self.category_embedding(categories)
 host_embed = self.host_embedding(hosts)
 external_features = torch.cat(( category_embed, host_embed, handmades), 1)
 _, seq_output, hidden_states = self.bert_model(input_ids=token_ids,
attention_mask=masks,
token_type_ids=segment)
 last_four_layer = torch.cat(( hidden_states[-5], hidden_states[-6]), 2)
 lstm_input = self.embedding_dropout(last_four_layer)
 lstm1_output, _ = self.lstm1(lstm_input)
 lstm2_output, _ = self.lstm2(lstm1_output)
 meanpooled_output = torch.mean(lstm2_output, 1)
 maxpooled_output, _ = torch.max(lstm2_output, 1)
 attention_output_q = self.atten1(lstm2_output)
 attention_output_a = self.atten2(lstm2_output)
 pooled_output_q = torch.cat(( meanpooled_output, maxpooled_output, attention_output_q, external_features), 1)
 pooled_output_q = self.dropout1(pooled_output_q)
 pooled_output_a = torch.cat(( meanpooled_output, maxpooled_output, attention_output_a, external_features), 1)
 pooled_output_a = self.dropout1(pooled_output_a)
 q_results = self.fc1(pooled_output_q)
 a_results = self.fc2(pooled_output_a)
 results = torch.cat(( q_results, a_results), 1)
 return results
sub.to_csv('submission.csv',index=False )<load_from_csv>
def predict_result(model, test_loader, batch_size=batch_size):
output = np.zeros(( len(test_set), 30))
model.eval()
with torch.no_grad() :
for idx, inputs in enumerate(test_loader):
start_index = idx * batch_size
end_index = min(start_index + batch_size, len(test_set))
token_ids, masks, segments, hosts, categories, handmades = inputs
token_ids = token_ids.long().cuda()
masks = masks.long().cuda()
segments = segments.long().cuda()
hosts = hosts.long().cuda()
categories = categories.long().cuda()
handmades = handmades.float().cuda()
predictions = model(token_ids, masks, segments, hosts, categories, handmades)
 predictions = torch.sigmoid(predictions)
 output[start_index:end_index, :] = predictions.detach().cpu().numpy()
return output
train = pd.read_csv(r".. /input/train.csv")\ 
.drop(["id"], axis=1)\ 
.rename(lambda x: "x"+str(x), axis='columns')\ 
.rename(index=str, columns={"xtarget":"target"})
 test = pd.read_csv(r".. /input/test.csv")\ 
.drop(["id"], axis=1)\ 
.rename(lambda x: "x"+str(x), axis='columns' )<feature_engineering>
pretrain_weighted_1 = glob.glob('.. /input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-1/*.pt')
 pretrain_weighted_2 = glob.glob('.. /input/d-large-roberta-b32-steplr3-drop-tb-rmf-h56-2/*.pt')
 pretrain_weighted = pretrain_weighted_1 + pretrain_weighted_2
train["mean"]=train.iloc[:,1:].mean(axis=1)
 test["mean"]=test.iloc[:,:].mean(axis=1 )<normalization>
!nvidia-smi
train.iloc[:,1:] = scale(train.iloc[:,1:])
 test.iloc[:,:] = scale(test )<import_modules>
!nvidia-smi
import pymc3 as pm<prepare_output>
test_set = QuestDataset_test(test_tokens, test_masks, test_segments,
test_df['host'].values,
test_df['category'].values,
train_handmade_features)
 test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False )
df_trace = pm.trace_to_dataframe(trace_1)
 pred = test.apply(lambda row: 1/(1+exp(-pd.Series(row*df_trace[features].mean() +df_trace["Intercept"].mean() ).sum(axis=0))),axis=1 )<save_to_csv>
model = QuestModel6(n_cat, cat_emb, n_host, host_emb, num_labels=30)
 model.cuda()
df = pd.DataFrame({"id":pd.read_csv(".. /input/test.csv" ).pop('id'),'target':pred})
 df[['id', 'target']].to_csv('submission.csv', index=False )<set_options>
for i, weight in tqdm(enumerate(pretrain_weighted)) :
model.load_state_dict(torch.load(weight))
results.append(predict_result(model, test_loader))
warnings.filterwarnings('ignore' )<load_from_csv>
output_roberta = np.zeros(( len(test_set),30))
for result in results:
output_roberta += result
output_roberta /= len(results )
%%time
train_df = pd.read_csv(os.path.join(PATH,"train.csv"))
test_df = pd.read_csv(os.path.join(PATH,"test.csv"))<count_missing_values>
del results, result, test_set, test_loader, model; gc.collect()
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<count_missing_values>
torch.cuda.empty_cache()
%%time
missing_data(train_df )<count_missing_values>
SEED = 1414
DATA_DIR = ".. /input/google-quest-challenge/"
WORK_DIR = ".. /working/"
BERT_VOCAB_PATH = ".. /input/xlnetlargecased/xlnet_cased_L-24_H-1024_A-16/spiece.model"
BERT_MODEL_PATH = ".. /input/xlnetlargecased/xlnet_cased_L-24_H-1024_A-16/xlnet_model.ckpt.index"
BERT_CONFIG_PATH = ".. /input/xlnetlargecased/xlnet_cased_L-24_H-1024_A-16/xlnet_config.json"
input_columns = ['question_title', 'question_body', 'answer']
seed_everything(SEED)
 batch_size = 32
%%time
missing_data(test_df )<count_values>
test_df = pd.read_csv(os.path.join(DATA_DIR,"test.csv"))
print("There are {}% target values with 1".format(100 * train_df["target"].value_counts() [1]/train_df.shape[0]))<sort_values>
eng_stopwords = set(stopwords.words("english"))
def add_external_features(df):
eng_stopwords = set(stopwords.words("english"))
df['question_body'] = df['question_body'].apply(lambda x:str(x))
df['question_num_words'] = df.question_body.str.count('\S+')
 df['question_title_num_words'] = df.question_title.str.count('\S+')
 df['answer'] = df['answer'].apply(lambda x:str(x))
df['answer_num_words'] = df.answer.str.count('\S+')
 df["question_title_num_unique_words"] = df["question_title"].apply(lambda x: len(set(str(x ).split())))
 df["question_body_num_unique_words"] = df["question_body"].apply(lambda x: len(set(str(x ).split())))
 df["answer_num_unique_words"] = df["answer"].apply(lambda x: len(set(str(x ).split())))
 df["question_title_num_chars"] = df["question_title"].apply(lambda x: len(str(x)))
 df["question_body_num_chars"] = df["question_body"].apply(lambda x: len(str(x)))
 df["answer_num_chars"] = df["answer"].apply(lambda x: len(str(x)))
 df['qt_words'] = df['question_title'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['q_words'] = df['question_body'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['a_words'] = df['answer'].apply(lambda s: [f for f in s.split() if f not in eng_stopwords])
 df['qa_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['q_words'], s['a_words'])) , axis = 1)
 df['qt_word_overlap'] = df.apply(lambda s: len(np.intersect1d(s['qt_words'], s['a_words'])) , axis = 1)
 df['qa_word_overlap_norm'] = df.apply(lambda s: s['qa_word_overlap']/(len(s['a_words'])+ len(s['q_words'])- s['qa_word_overlap']), axis = 1)
 df['qta_word_overlap_norm'] = df.apply(lambda s: s['qt_word_overlap']/(len(s['a_words'])+ len(s['qt_words'])- s['qt_word_overlap']), axis = 1)
 df.drop(['q_words', 'a_words', 'qt_words'], axis = 1, inplace = True)
 return df
%%time
correlations = train_df[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.head(10 )<feature_engineering>
test_df = add_external_features(test_df )
%%time
features = [c for c in train_df.columns if c not in ['id', 'target']]
for df in [test_df, train_df]:
df['sum'] = df[features].sum(axis=1)
 df['min'] = df[features].min(axis=1)
 df['max'] = df[features].max(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['std'] = df[features].std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurtosis(axis=1)
 df['med'] = df[features].median(axis=1 )<train_model>
handmade_cols = ["question_body_num_unique_words", 'question_num_words',
"question_title_num_unique_words", "question_title_num_words",
"answer_num_unique_words", "answer_num_words"]
with open('/kaggle/input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/C_scaler.pickle', mode='rb')as f:
num_words_scaler = pickle.load(f)
 test_df[handmade_cols]= num_words_scaler.transform(test_df[handmade_cols].values)
 train_handmade_features= test_df[handmade_cols + ['qa_word_overlap_norm', 'qta_word_overlap_norm']].values
print('Train and test columns: {} {}'.format(len(train_df.columns), len(test_df.columns)) )<statistical_test>
def label_encoder(x, dict_reverse):
try:
return dict_reverse[x]
except:
return 0
with open('/kaggle/input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/C_category.pickle', mode='rb')as f:
category_dict_reverse = pickle.load(f)
 test_df['category'] = test_df['category'].apply(lambda x: label_encoder(x, category_dict_reverse))
with open('/kaggle/input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/C_host.pickle', mode='rb')as f:
host_dict_reverse = pickle.load(f)
 test_df['host'] = test_df['host'].apply(lambda x: label_encoder(x, host_dict_reverse))
n_cat = len(category_dict_reverse)+ 1
cat_emb = 256
n_host = len(host_dict_reverse)+ 1
host_emb = 256
def apply_noise(data, noise_level):
idxt = data[['id', 'target']]
features = data.columns.values[2:]
appended_data = []
for feature in features:
signal = data[feature]
noise_factor =(np.abs(signal)).mean() * noise_level
noise = np.random.normal(0, noise_level, signal.shape)
 jittered = signal + noise
appended_data.append(pd.DataFrame(jittered))
appended_data = pd.concat(appended_data, axis=1)
 data_jittered = pd.concat([idxt, pd.DataFrame(appended_data)], axis=1)
 return data_jittered<normalization>
bert_config = transformers.XLNetConfig.from_json_file(BERT_CONFIG_PATH)
 bert_config.output_hidden_states = True
tokenizer = transformers.XLNetTokenizer(BERT_VOCAB_PATH )
noise_train_df = []
for i in tqdm_notebook(range(0,2)) :
t = apply_noise(train_df, noise_level = i * 0.025)
 noise_train_df.append(t)
 noise_train_df = pd.concat(noise_train_df, axis = 0 )<define_variables>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_segments2(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q_len_head = round(q_new_len/3)
 q_len_tail = -1*(q_new_len -q_len_head)
 a_len_head = round(a_new_len/3)
 a_len_tail = -1*(a_new_len -a_len_head)
 q = q[:q_len_head]+q[q_len_tail:]
a = a[:a_len_head]+a[a_len_tail:]
return t, q, a
def convert_lines(question, answer, max_sequence_length, tokenizer):
all_tokens = []
all_masks = []
all_segments = []
longer = 0
for q, a in tqdm(zip(question, answer)) :
tokens_q = tokenizer.tokenize(q)
 tokens_a = tokenizer.tokenize(a)
 q_len = len(tokens_q)
 a_len = len(tokens_a)
 if(q_len+a_len+3)> max_sequence_length:
longer += 1
new_q_len = q_len/(a_len+q_len)*(max_sequence_length-3)
 new_a_len = a_len/(q_len+a_len)*(max_sequence_length-3)
 new_q_len, new_a_len = int(np.ceil(new_q_len)) , int(np.floor(new_a_len))
if new_a_len+new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_a_len+new_q_len+3))
tokens_q = tokens_q[:new_q_len]
tokens_a = tokens_a[:new_a_len]
stoken = ["[CLS]"] + tokens_q + ["[SEP]"] + tokens_a + ["[SEP]"]
token_ids = tokenizer.encode(stoken, add_special_tokens=Ture)
 input_ids = token_ids + [0] *(max_sequence_length-len(token_ids))
attention_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 all_tokens.append(input_ids)
 all_masks.append(attention_masks)
 all_segments.append(input_segments)
 print(longer)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
features = [c for c in train_df.columns if c not in ['id', 'target']]
target = train_df['target']<init_hyperparams>
def convert_lines2(title, question, answer, max_sequence_length, tokenizer, t_max_len_seq=30, q_max_len_seq=239, a_max_len_seq=239):
all_tokens = []
all_masks = []
all_segments = []
longer = 0
for t, q, a in tqdm(zip(title, question, answer)) :
tokens_t, tokens_q, tokens_a = _trim_input(t, q, a, max_sequence_length=max_sequence_length)
 stoken = tokens_t + ["[SEP]"] + tokens_q + ["[SEP]"] + tokens_a + ["[SEP]"] + ["[CLS]"]
token_ids = tokenizer.convert_tokens_to_ids(stoken)
 input_ids = token_ids + [0] *(max_sequence_length-len(token_ids))
attention_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 all_tokens.append(input_ids)
 all_masks.append(attention_masks)
 all_segments.append(input_segments)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
param = {
'bagging_freq': 5,
'bagging_fraction': 0.83,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.81,
'learning_rate': 0.005,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 90,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 11,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1
}<train_model>
test_tokens, test_masks, test_segments = convert_lines2(test_df["question_title"],
test_df["question_body"],
test_df["answer"],
max_sequence_length=512,
tokenizer=tokenizer )
folds = StratifiedKFold(n_splits=9, shuffle=True, random_state=4422)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 100000
clf = lgb.train(param, trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=400,
early_stopping_rounds = 400)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["Feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
class QuestDataset_test(Dataset):
def __init__(self, token_ids, masks, segments, hosts, categories, handmade_features):
self.token_ids = token_ids
self.masks = masks
self.segments = segments
self.hosts = hosts
self.categories = categories
self.handmades = handmade_features
def __len__(self):
return self.token_ids.shape[0]
def __getitem__(self, idx):
token_id = self.token_ids[idx]
mask = self.masks[idx]
segment = self.segments[idx]
host = self.hosts[idx]
category = self.categories[idx]
handmade = self.handmades[idx]
return [token_id, mask, segment, host, category, handmade]
sub_df = pd.DataFrame({"id":test_df["id"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )<set_options>
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, torch.nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))
warnings.filterwarnings('ignore')
 np.random.seed(27 )<set_options>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
%matplotlib inline
plt.rcParams['figure.figsize'] = [20.0, 7.0]
plt.rcParams.update({'font.size': 22,})
 sns.set_palette('viridis')
 sns.set_style('white')
 sns.set_context('talk', font_scale=0.8 )<load_from_csv>
class QuestModel6(nn.Module):
def __init__(self, n_cat, cat_emb, n_host, host_emb, num_labels):
super().__init__()
BERT_DIMS = 1024
LSTM_UNITS = 512
model_path = os.path.join(BERT_MODEL_PATH)
 self.bert_model = transformers.XLNetModel.from_pretrained(model_path,from_tf=True, config=bert_config)
 set_trainable(self.bert_model, False)
 self.category_embedding = nn.Embedding(n_cat, cat_emb)
 self.host_embedding = nn.Embedding(n_host, host_emb)
 self.embedding_dropout = SpatialDropout(0.5)
 self.lstm1 = nn.LSTM(BERT_DIMS*2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.GRU(LSTM_UNITS*2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.atten1 = Attention(LSTM_UNITS*2, 512)
 self.atten2 = Attention(LSTM_UNITS*2, 512)
 self.dropout1 = nn.Dropout(0.2)
 self.fc1 = nn.Linear(LSTM_UNITS*6 + int(cat_emb)+ int(host_emb)+ 8, 21)
 self.fc2 = nn.Linear(LSTM_UNITS*6 + int(cat_emb)+ int(host_emb)+ 8, 9)
 self._init_weights(self.category_embedding)
 self._init_weights(self.host_embedding)
 self._init_weights(self.fc1)
 self._init_weights(self.fc2)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
print("initailize weight")
 module.weight.data.normal_(mean=0.0, std=0.02)
 if isinstance(module, nn.Linear)and module.bias is not None:
print("initailize bias")
 module.bias.data.zero_()
def forward(self, token_ids, masks, segment, hosts, categories, handmades):
category_embed = self.category_embedding(categories)
 host_embed = self.host_embedding(hosts)
 external_features = torch.cat(( category_embed, host_embed, handmades), 1)
 seq_output, hidden_states = self.bert_model(input_ids=token_ids,
attention_mask=masks,
token_type_ids=segment)
 last_four_layer = torch.cat(( hidden_states[-5], hidden_states[-6]), 2)
 lstm_input = self.embedding_dropout(last_four_layer)
 lstm1_output, _ = self.lstm1(lstm_input)
 lstm2_output, _ = self.lstm2(lstm1_output)
 meanpooled_output = torch.mean(lstm2_output, 1)
 maxpooled_output, _ = torch.max(lstm2_output, 1)
 attention_output_q = self.atten1(lstm2_output)
 attention_output_a = self.atten2(lstm2_output)
 pooled_output_q = torch.cat(( meanpooled_output, maxpooled_output, attention_output_q, external_features), 1)
 pooled_output_q = self.dropout1(pooled_output_q)
 pooled_output_a = torch.cat(( meanpooled_output, maxpooled_output, attention_output_a, external_features), 1)
 pooled_output_a = self.dropout1(pooled_output_a)
 q_results = self.fc1(pooled_output_q)
 a_results = self.fc2(pooled_output_a)
 results = torch.cat(( q_results, a_results), 1)
 return results
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('Train Shape: ', train.shape)
 print('Test Shape: ', test.shape)
 train.head()<normalization>
def predict_result(model, test_loader, batch_size=batch_size):
output = np.zeros(( len(test_set), 30))
model.eval()
with torch.no_grad() :
for idx, inputs in enumerate(test_loader):
start_index = idx * batch_size
end_index = min(start_index + batch_size, len(test_set))
token_ids, masks, segments, hosts, categories, handmades = inputs
token_ids = token_ids.long().cuda()
masks = masks.long().cuda()
segments = segments.long().cuda()
hosts = hosts.long().cuda()
categories = categories.long().cuda()
handmades = handmades.float().cuda()
predictions = model(token_ids, masks, segments, hosts, categories, handmades)
 predictions = torch.sigmoid(predictions)
 output[start_index:end_index, :] = predictions.detach().cpu().numpy()
return output
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<choose_model_class>
pretrain_weighted_1 = glob.glob('.. /input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-1/*.pt')
 pretrain_weighted_2 = glob.glob('.. /input/d-large-xlnet-b32-steplr3-changedrop-tb-rmf-h56-2/*.pt')
 pretrain_weighted = pretrain_weighted_1 + pretrain_weighted_2
ridge = linear_model.Ridge()
lasso = linear_model.Lasso()
elastic = linear_model.ElasticNet()
lasso_lars = linear_model.LassoLars()
bayesian_ridge = linear_model.BayesianRidge()
logistic = linear_model.LogisticRegression(solver='liblinear')
 sgd = linear_model.SGDClassifier()<define_variables>
!nvidia-smi
models = [ridge, lasso, elastic, lasso_lars, bayesian_ridge, logistic, sgd]<compute_train_metric>
!nvidia-smi
def get_cv_scores(model):
scores = cross_val_score(model, X_train, y_train, cv=5, scoring='roc_auc')
 print('CV Mean: ', np.mean(scores))
print('STD: ', np.std(scores))
print('
' )<find_best_params>
test_set = QuestDataset_test(test_tokens, test_masks, test_segments,
test_df['host'].values,
test_df['category'].values,
train_handmade_features)
 test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False )
for model in models:
print(model)
 get_cv_scores(model )<train_on_grid>
model = QuestModel6(n_cat, cat_emb, n_host, host_emb, num_labels=30)
 model.cuda()
penalty = ['l1', 'l2']
C = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000]
class_weight = [{1:0.5, 0:0.5}, {1:0.4, 0:0.6}, {1:0.6, 0:0.4}, {1:0.7, 0:0.3}]
solver = ['liblinear', 'saga']
param_grid = dict(penalty=penalty,
C=C,
class_weight=class_weight,
solver=solver)
 grid = GridSearchCV(estimator=logistic, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1)
 grid_result = grid.fit(X_train, y_train)
 print('Best Score: ', grid_result.best_score_)
 print('Best Params: ', grid_result.best_params_ )<choose_model_class>
for i, weight in tqdm(enumerate(pretrain_weighted)) :
model.load_state_dict(torch.load(weight))
results.append(predict_result(model, test_loader))
logistic = linear_model.LogisticRegression(C=1, class_weight={1:0.6, 0:0.4}, penalty='l1', solver='liblinear')
 get_cv_scores(logistic )<predict_on_test>
output_xlnet = np.zeros(( len(test_set),30))
for result in results:
output_xlnet += result
output_xlnet /= len(results )
predictions = logistic.fit(X_train, y_train ).predict_proba(X_test)
<save_to_csv>
del results, result, test_set, test_loader, model; gc.collect()
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions
submission.head()<define_search_space>
torch.cuda.empty_cache()
loss = ['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron']
penalty = ['l1', 'l2', 'elasticnet']
alpha = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000]
learning_rate = ['constant', 'optimal', 'invscaling', 'adaptive']
class_weight = [{1:0.5, 0:0.5}, {1:0.4, 0:0.6}, {1:0.6, 0:0.4}, {1:0.7, 0:0.3}]
eta0 = [1, 10, 100]
param_distributions = dict(loss=loss,
penalty=penalty,
alpha=alpha,
learning_rate=learning_rate,
class_weight=class_weight,
eta0=eta0)
 random = RandomizedSearchCV(estimator=sgd, param_distributions=param_distributions, scoring='roc_auc', verbose=1, n_jobs=-1, n_iter=1000)
 random_result = random.fit(X_train, y_train)
 print('Best Score: ', random_result.best_score_)
 print('Best Params: ', random_result.best_params_ )<choose_model_class>
sys.path.append(".. /input/mynlpscripts/")
 sys.path.extend(['.. /input/bert-utils/'])
sgd = linear_model.SGDClassifier(alpha=0.1,
class_weight={1:0.7, 0:0.3},
eta0=100,
learning_rate='optimal',
loss='log',
penalty='elasticnet')
 get_cv_scores(sgd )<predict_on_test>
print("TF version:", tf.__version__)
 print("Hub version:", hub.__version__)
predictions = sgd.fit(X_train, y_train ).predict_proba(X_test)
<save_to_csv>
log = logging.getLogger('tensorflow')
 log.handlers = []
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head()<load_from_csv>
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = "true"
StratifiedKFold, GroupShuffleSplit,
GroupKFold, StratifiedShuffleSplit)
 seed(42)
 tf.set_random_seed(42)
 random.seed(42 )
train=pd.read_csv(".. /input/train.csv" )<load_from_csv>

 BaseShuffleSplit, _validate_shuffle_split
def IterativeStratification(labels, r, random_state):

 n_samples = labels.shape[0]
test_folds = np.zeros(n_samples, dtype=int)
 c_folds = r * n_samples
c_folds_labels = np.outer(r, labels.sum(axis=0))
labels_not_processed_mask = np.ones(n_samples, dtype=bool)
 while np.any(labels_not_processed_mask):
num_labels = labels[labels_not_processed_mask].sum(axis=0)
 if num_labels.sum() == 0:
sample_idxs = np.where(labels_not_processed_mask)[0]
for sample_idx in sample_idxs:
fold_idx = np.where(c_folds == c_folds.max())[0]
if fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
c_folds[fold_idx] -= 1
break
label_idx = np.where(num_labels == num_labels[np.nonzero(num_labels)].min())[0]
if label_idx.shape[0] > 1:
label_idx = label_idx[random_state.choice(label_idx.shape[0])]
sample_idxs = np.where(np.logical_and(labels[:, label_idx].flatten() , labels_not_processed_mask)) [0]
for sample_idx in sample_idxs:
label_folds = c_folds_labels[:, label_idx]
fold_idx = np.where(label_folds == label_folds.max())[0]
if fold_idx.shape[0] > 1:
temp_fold_idx = np.where(c_folds[fold_idx] ==
c_folds[fold_idx].max())[0]
fold_idx = fold_idx[temp_fold_idx]
if temp_fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(temp_fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
labels_not_processed_mask[sample_idx] = False
c_folds_labels[fold_idx, labels[sample_idx]] -= 1
c_folds[fold_idx] -= 1
return test_folds
class MultilabelStratifiedKFold(_BaseKFold):

 def __init__(self, n_splits=3, shuffle=False, random_state=None):
super(MultilabelStratifiedKFold, self ).__init__(n_splits, shuffle, random_state)
 def _make_test_folds(self, X, y):
y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(type_of_target_y))
num_samples = y.shape[0]
rng = check_random_state(self.random_state)
 indices = np.arange(num_samples)
 if self.shuffle:
rng.shuffle(indices)
 y = y[indices]
r = np.asarray([1 / self.n_splits] * self.n_splits)
 test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 return test_folds[np.argsort(indices)]
def _iter_test_masks(self, X=None, y=None, groups=None):
test_folds = self._make_test_folds(X, y)
 for i in range(self.n_splits):
yield test_folds == i
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedKFold, self ).split(X, y, groups)
 class RepeatedMultilabelStratifiedKFold(_RepeatedSplits):

 def __init__(self, n_splits=5, n_repeats=10, random_state=None):
super(RepeatedMultilabelStratifiedKFold, self ).__init__(
 MultilabelStratifiedKFold, n_repeats, random_state,
n_splits=n_splits)
 class MultilabelStratifiedShuffleSplit(BaseShuffleSplit):

 def __init__(self, n_splits=10, test_size="default", train_size=None,
random_state=None):
super(MultilabelStratifiedShuffleSplit, self ).__init__(
 n_splits, test_size, train_size, random_state)
 def _iter_indices(self, X, y, groups=None):
n_samples = _num_samples(X)
 y = check_array(y, ensure_2d=False, dtype=None)
 y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(
 type_of_target_y))
n_train, n_test = _validate_shuffle_split(n_samples, self.test_size,
self.train_size)
 n_samples = y.shape[0]
rng = check_random_state(self.random_state)
 y_orig = y.copy()
r = np.array([n_train, n_test])/(n_train + n_test)
 for _ in range(self.n_splits):
indices = np.arange(n_samples)
 rng.shuffle(indices)
 y = y_orig[indices]
test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 test_idx = test_folds[np.argsort(indices)] == 1
test = np.where(test_idx)[0]
train = np.where(~test_idx)[0]
yield train, test
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedShuffleSplit, self ).split(X, y, groups )
test=pd.read_csv(".. /input/test.csv" )<count_missing_values>
def stratified_group_k_fold(X, y, groups, k, seed=None):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(seed ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(k):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(k):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_indices = [i for i, g in enumerate(groups)if g in train_groups]
test_indices = [i for i, g in enumerate(groups)if g in test_groups]
yield train_indices, test_indices
train.isnull().sum()<count_missing_values>
def chunks(l, n):

 for i in range(0, len(l), n):
yield l[i:i + n]
train.isnull().sum()<drop_column>
def build_module_fn(config_path, vocab_path, do_lower_case=True):
def bert_module_fn(is_training):

 input_ids = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_ids")
 input_mask = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_mask")
 token_type = tf.placeholder(shape=[None, None], dtype=tf.int32, name="segment_ids")
 config = BertConfig.from_json_file(config_path)
 model = BertModel(config=config, is_training=is_training,
input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type)
 seq_output = model.all_encoder_layers[-1]
pool_output = model.get_pooled_output()
config_file = tf.constant(value=config_path, dtype=tf.string, name="config_file")
 vocab_file = tf.constant(value=vocab_path, dtype=tf.string, name="vocab_file")
 lower_case = tf.constant(do_lower_case)
 tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, config_file)
 tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, vocab_file)
 input_map = {"input_ids": input_ids,
"input_mask": input_mask,
"segment_ids": token_type}
output_map = {"pooled_output": pool_output,
"sequence_output": seq_output}
output_info_map = {"vocab_file": vocab_file,
"do_lower_case": lower_case}
hub.add_signature(name="tokens", inputs=input_map, outputs=output_map)
 hub.add_signature(name="tokenization_info", inputs={}, outputs=output_info_map)
 return bert_module_fn
train.pop("id" )<drop_column>
MODEL_DIR1 = '.. /input/bertbase'
config_path1 = "{}/config.json".format(MODEL_DIR1)
 vocab_path1 = "{}/vocab.txt".format(MODEL_DIR1)
 tags_and_args = []
for is_training in(True, False):
tags = set()
if is_training:
tags.add("train")
 tags_and_args.append(( tags, dict(is_training=is_training)))
 module_fn = build_module_fn(config_path1, vocab_path1)
 spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args)
 spec.export("bert-module1",
checkpoint_path="{}/bert_model.ckpt".format(MODEL_DIR1))
test.pop("id" )<prepare_x_and_y>
MODEL_DIR = '.. /input/bertlarge'
config_path = "{}/bert_config.json".format(MODEL_DIR)
 vocab_path = "{}/vocab.txt".format(MODEL_DIR)
 tags_and_args = []
for is_training in(True, False):
tags = set()
if is_training:
tags.add("train")
 tags_and_args.append(( tags, dict(is_training=is_training)))
 module_fn = build_module_fn(config_path, vocab_path)
 spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args)
 spec.export("bert-module",
checkpoint_path="{}/bert_model.ckpt".format(MODEL_DIR))
y_train=train.iloc[:,0]<drop_column>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-tfhub'
USE_PATH = '.. /input/uselarge4/'
tokenizer = FullTokenizer(vocab_path, True)
 tokenizer1 = FullTokenizer(vocab_path1, True)
 MAX_SEQUENCE_LENGTH = 512
MAX_SEQUENCE_LENGTH_t = 120
MAX_SEQUENCE_LENGTH_q = 416
MAX_SEQUENCE_LENGTH_a = 416
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
train.pop("target" )<prepare_x_and_y>
def read_examples(str_list):

 unique_id = 0
for s in str_list:
line = convert_to_unicode(s)
 if not line:
continue
line = line.strip()
text_a = None
text_b = None
m = re.match(r"^ (.*)\|\|\| (.*)$", line)
 if m is None:
text_a = line
else:
text_a = m.group(1)
 text_b = m.group(2)
 yield InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b)
 unique_id += 1
train1=train.copy()
y_train1=y_train.copy()
test1=test.copy()<import_modules>
def features_to_arrays(features):
all_input_ids = []
all_input_mask = []
all_segment_ids = []
for feature in features:
all_input_ids.append(feature.input_ids)
 all_input_mask.append(feature.input_mask)
 all_segment_ids.append(feature.input_type_ids)
 return(np.array(all_input_ids, dtype='int32'),
np.array(all_input_mask, dtype='int32'),
np.array(all_segment_ids, dtype='int32'))
from sklearn.preprocessing import StandardScaler<normalization>
def build_preprocessor(voc_path, seq_len, lower=True):
tokenizer = FullTokenizer(vocab_file=voc_path, do_lower_case=lower)
 def strings_to_arrays(sents):
sents = np.atleast_1d(sents ).reshape(( -1,))
examples = []
for example in read_examples(sents):
examples.append(example)
 features = convert_examples_to_features(examples, seq_len, tokenizer)
 arrays = features_to_arrays(features)
 return arrays
return strings_to_arrays
sc=StandardScaler()<feature_engineering>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
input_columns = ['question_title', 'question_body', 'answer']
train=sc.fit_transform(train )<feature_engineering>
def _get_masks1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids1(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input1(title, question, answer,tokenizer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs1(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids1(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks1(stoken, max_sequence_length)
 input_segments = _get_segments1(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays1(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input1(t, q, a,tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs1(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays1(df, columns):
return np.asarray(df[columns] )
test=sc.fit_transform(test )<import_modules>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(question,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 q_len = len(q)
 if(q_len+3)> max_sequence_length:
new_q_len = max_sequence_length-3
if new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_q_len+3))
q = q[:new_q_len]
return q
def _convert_to_bert_inputs(question, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_q(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_body
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_a(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.answer
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_t(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_title
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
from sklearn.linear_model import LogisticRegression,Lasso , Ridge
from sklearn.model_selection import GridSearchCV, cross_val_score 
<train_on_grid>
coef_nn = 2
coef_mte = 1.7
coef_lgb =1
random_state=42
log_clf=LogisticRegression(random_state=random_state)
 param_grid={
'class_weight': ['balanced', None],
'C':[0.001, 0.01, 0.1, 1, 10, 100, 1000],
'penalty':['l1','l2']
}
grid=GridSearchCV(log_clf,param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1)
 grid.fit(train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_)) 
<choose_model_class>
class TDense(tf.keras.layers.Layer):
def __init__(self,
output_size,
kernel_initializer=None,
bias_initializer="zeros",
**kwargs):
super().__init__(**kwargs)
 self.output_size = output_size
self.kernel_initializer = kernel_initializer
self.bias_initializer = bias_initializer
def build(self,input_shape):
dtype = tf.as_dtype(self.dtype or tf.keras.backend.floatx())
 if not(dtype.is_floating or dtype.is_complex):
raise TypeError("Unable to build `TDense` layer with "
"non-floating point(and non-complex)"
"dtype %s" %(dtype,))
input_shape = tf.TensorShape(input_shape)
 if tf.compat.dimension_value(input_shape[-1])is None:
raise ValueError("The last dimension of the inputs to "
"`TDense` should be defined."
"Found `None`.")
 last_dim = tf.compat.dimension_value(input_shape[-1])
 self.input_spec = tf.keras.layers.InputSpec(min_ndim=2, axes={-1: last_dim})
 self.kernel = self.add_weight(
 "kernel",
shape=[self.output_size,last_dim],
initializer=self.kernel_initializer,
dtype=self.dtype,
trainable=True)
 self.bias = self.add_weight(
 "bias",
shape=[self.output_size],
initializer=self.bias_initializer,
dtype=self.dtype,
trainable=True)
 super(TDense, self ).build(input_shape)
 def call(self,x):
return tf.matmul(x,self.kernel,transpose_b=True)+self.bias
log_clf = LogisticRegression(C=0.1,random_state=42,class_weight='balanced',penalty='l1' )<train_model>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 def average_spearmanr(y_true: np.ndarray, y_pred: np.ndarray)-> float:
return np.average([
spearmanr(y_t, y_p ).correlation for y_t, y_p in zip(y_true.T, y_pred.T)
 ])
 class SpearmanRhoCallback(Callback):
def __init__(self, training_data, validation_data, patience, model_name):
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
self.patience = patience
self.value = -1
self.bad_epochs = 0
self.model_name = model_name
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_val = self.model.predict(self.x_val)
 rho_val = np.nanmean([spearmanr(self.y_val[:, ind], y_pred_val[:, ind] + np.random.normal(0, 1e-7, y_pred_val.shape[0])).correlation for ind in range(y_pred_val.shape[1])])
 if rho_val >= self.value:
self.value = rho_val
self.model.save_weights(self.model_name)
 else:
self.bad_epochs += 1
if self.bad_epochs >= self.patience:
print("Epoch %05d: early stopping Threshold" % epoch)
 self.model.stop_training = True
print('\rval_spearman-rho: %s' %(str(round(rho_val, 4))), end=100*' '+'
')
 return rho_val
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return
log_clf.fit(train,y_train )<predict_on_test>
class BertLayer1(tf.keras.layers.Layer):
def __init__(self, bert_path, seq_len=64, n_tune_layers=3,
pooling="cls", do_preprocessing=True, verbose=False,
tune_embeddings=False, trainable=True, **kwargs):
self.trainable = trainable
self.n_tune_layers = n_tune_layers
self.tune_embeddings = tune_embeddings
self.do_preprocessing = do_preprocessing
self.verbose = verbose
self.seq_len = seq_len
self.pooling = pooling
self.bert_path = bert_path
self.var_per_encoder = 16
if self.pooling not in ["cls", "mean", None]:
raise NameError(
 f"Undefined pooling type(must be either 'cls', 'mean', or None, but is {self.pooling}" 
)
 super(BertLayer1, self ).__init__(**kwargs)
 def build(self, input_shape):
self.bert = hub.Module(self.build_abspath(self.bert_path),
trainable=self.trainable, name=f"{self.name}_module")
 trainable_layers = []
if self.tune_embeddings:
trainable_layers.append("embeddings")
 if self.n_tune_layers > 0:
encoder_var_names = [var.name for var in self.bert.variables if 'encoder' in var.name]
n_encoder_layers = int(len(encoder_var_names)/ self.var_per_encoder)
 for i in range(self.n_tune_layers):
trainable_layers.append(f"encoder/layer_{str(n_encoder_layers - 1 - i)}/")
 for var in self.bert.variables:
if any([l in var.name for l in trainable_layers]):
self._trainable_weights.append(var)
 else:
self._non_trainable_weights.append(var)
 if self.verbose:
print("*** TRAINABLE VARS *** ")
 for var in self._trainable_weights:
print(var)
 self.build_preprocessor()
self.initialize_module()
super(BertLayer1, self ).build(input_shape)
 def build_abspath(self, path):
if path.startswith("https://")or path.startswith("gs://"):
return path
else:
return os.path.abspath(path)
 def build_preprocessor(self):
sess = tf.keras.backend.get_session()
tokenization_info = self.bert(signature="tokenization_info", as_dict=True)
 vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"],
tokenization_info["do_lower_case"]])
 self.preprocessor = build_preprocessor(vocab_file, self.seq_len, do_lower_case)
 def initialize_module(self):
sess = tf.keras.backend.get_session()
vars_initialized = sess.run([tf.is_variable_initialized(var)
 for var in self.bert.variables])
 uninitialized = []
for var, is_initialized in zip(self.bert.variables, vars_initialized):
if not is_initialized:
uninitialized.append(var)
 if len(uninitialized):
sess.run(tf.variables_initializer(uninitialized))
def call(self, input):
if self.do_preprocessing:
input = tf.numpy_function(self.preprocessor,
[input], [tf.int32, tf.int32, tf.int32],
name='preprocessor')
 for feature in input:
feature.set_shape(( None, self.seq_len))
input_ids, input_mask, segment_ids = input
bert_inputs = dict(
 input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids 
)
 output = self.bert(inputs=bert_inputs, signature="tokens", as_dict=True)
 pooled_cls = output["pooled_output"]
result = output["sequence_output"]
input_mask = tf.cast(input_mask, tf.float32)
 mul_mask = lambda x, m: x * tf.expand_dims(m, axis=-1)
 masked_reduce_mean = lambda x, m: tf.reduce_sum(mul_mask(x, m), axis=1)/(
 tf.reduce_sum(m, axis=1, keepdims=True)+ 1e-10)
 pooled_mean = masked_reduce_mean(result, input_mask)
 sequence_output = mul_mask(result, input_mask)
 if self.pooling == "cls":
return pooled_cls
else:
return pooled_mean, sequence_output
def get_config(self):
config_dict = {
"bert_path": self.bert_path,
"seq_len": self.seq_len,
"pooling": self.pooling,
"n_tune_layers": self.n_tune_layers,
"tune_embeddings": self.tune_embeddings,
"do_preprocessing": self.do_preprocessing,
"verbose": self.verbose
}
super(BertLayer1, self ).get_config()
return config_dict
pred_log=log_clf.predict(test )<load_from_csv>
outputs = compute_output_arrays1(df_train, output_categories)
 test_inputs = compute_input_arays1(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputst = compute_input_arays_t(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_t)
 test_inputsq = compute_input_arays_q(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_q)
 test_inputsa = compute_input_arays_a(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_a )
submission3 = pd.read_csv('.. /input/sample_submission.csv' )<feature_engineering>
d= {0.: 0, 0.3: 1, 0.7: 2}
theArray = np.round(outputs[:, 19:20], decimals=1)
 newArray = copy(theArray)
 for k, v in d.items() : newArray[theArray==k] = v
newArray = np.squeeze(newArray)
 newArray =newArray.astype(int)
 np.unique(newArray )
submission3["target"]=pred_log<save_to_csv>
del test_inputs, test_inputst, test_inputsq,test_inputsa; gc.collect()
submission3.to_csv('submissionlog.csv', index=False )<import_modules>
def _get_masks2(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments2(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids2(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input2(question, answer,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 q_len = len(q)
 a_len = len(a)
 if(q_len+a_len+3)> max_sequence_length:
new_q_len = q_len/(a_len+q_len)*(max_sequence_length-3)
 new_a_len = a_len/(q_len+a_len)*(max_sequence_length-3)
 new_q_len, new_a_len = int(ceil(new_q_len)) , int(floor(new_a_len))
if new_a_len+new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_a_len+new_q_len+3))
q = q[:new_q_len]
a = a[:new_a_len]
return q, a
def _convert_to_bert_inputs2(question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids2(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks2(stoken, max_sequence_length)
 input_segments = _get_segments2(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays2(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q, a = instance.question_title, instance.question_body
q, a = _trim_input2(q, a, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs2(
 q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
from sklearn.feature_selection import RFE 
<train_model>
def _get_masks3(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments3(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids3(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input3(question, answer,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 q_len = len(q)
 a_len = len(a)
 if(q_len+a_len+3)> max_sequence_length:
new_q_len = q_len/(a_len+q_len)*(max_sequence_length-3)
 new_a_len = a_len/(q_len+a_len)*(max_sequence_length-3)
 new_q_len, new_a_len = int(ceil(new_q_len)) , int(floor(new_a_len))
if new_a_len+new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_a_len+new_q_len+3))
q = q[:new_q_len]
a = a[:new_a_len]
return q, a
def _convert_to_bert_inputs3(question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids3(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks3(stoken, max_sequence_length)
 input_segments = _get_segments3(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays3(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q, a = instance.question_title, instance.answer
q, a = _trim_input3(q, a,tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs3(
 q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
selector = RFE(log_clf, 25, step=1)
 selector.fit(train,y_train )<save_to_csv>
def _get_masks1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments1(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids1(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input1(title, question, answer,tokenizer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs1(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids1(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks1(stoken, max_sequence_length)
 input_segments = _get_segments1(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays1(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input1(t, q, a,tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs1(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays1(df, columns):
return np.asarray(df[columns] )
submission1 = pd.read_csv('.. /input/sample_submission.csv')
 submission1['target'] = selector.predict_proba(test)
 submission1.to_csv('submissionRFE.csv', index=False )<import_modules>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(question,tokenizer, max_sequence_length):

 q = tokenizer.tokenize(question)
 q_len = len(q)
 if(q_len+3)> max_sequence_length:
new_q_len = max_sequence_length-3
if new_q_len+3 != max_sequence_length:
raise ValueError("too small %s" % str(new_q_len+3))
q = q[:new_q_len]
return q
def _convert_to_bert_inputs(question, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + question + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_q(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_body
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_a(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.answer
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_input_arays_t(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
q = instance.question_title
q= _trim_input(q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(
 q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
from sklearn.ensemble import GradientBoostingRegressor<train_on_grid>
outputs = compute_output_arrays1(df_train, output_categories)
 test_inputs = compute_input_arays1(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputstq = compute_input_arays2(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 inputsq = compute_input_arays_q(df_train, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_q)
 test_inputst = compute_input_arays_t(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_t)
 test_inputsq = compute_input_arays_q(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_q)
 test_inputsa = compute_input_arays_a(df_test, input_categories, tokenizer1, MAX_SEQUENCE_LENGTH_a )
random_state=42
gbr=GradientBoostingRegressor(random_state=random_state)
 param_grid={
"learning_rate": [0.01, 0.05 , 0.1 , 1],
"max_depth":[5, 11, 17, 25, 37, 53],
"max_features":["log2","sqrt"],
"criterion": ["friedman_mse", "mae"],
"n_estimators":[10, 50, 200, 600, 1000]
}
grid=GridSearchCV(gbr,param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1)
 grid.fit(train,y_train)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_)) 
<choose_model_class>
%%time
kf = MultilabelStratifiedKFold(n_splits=10, random_state=42, shuffle=True)
 for i,(tr, val)in enumerate(kf.split(inputsq[1], outputs)) :
if i!=7:
continue
K.clear_session()
print('starting fold: {0}'.format(i))
json_file = open('.. /input/save-groupbertq-fold{}/nn_modelq{}.json'.format(i, i), 'r')
 loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json, custom_objects={"BertLayer1": BertLayer1})
 bst_model_path = '.. /input/save-groupbertq-fold{}/dl_modelq{}.h5'.format(i, i)
 print(bst_model_path)
 model.load_weights(bst_model_path)
 model.compile(optimizer=tf.keras.optimizers.Nadam(3e-5),loss=['binary_crossentropy'])
 print("Loaded model from disk")
 y_testq = model.predict([test_inputstq[0],test_inputstq[1],test_inputstq[2],test_inputst[0],test_inputst[1],test_inputst[2],
test_inputsq[0],test_inputsq[1],test_inputsq[2]])
 print("y_testq shape", y_testq.shape)
 del model
gc.collect()
K.clear_session()
print('starting fold: {0}'.format(i))
json_file = open('.. /input/save-groupberta-fold{}/nn_modela{}.json'.format(i, i), 'r')
 loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json, custom_objects={"BertLayer1": BertLayer1})
 bst_model_path = '.. /input/save-groupberta-fold{}/dl_modela{}.h5'.format(i, i)
 print(bst_model_path)
 model.load_weights(bst_model_path)
 model.compile(optimizer=tf.keras.optimizers.Nadam(3e-5),loss=['binary_crossentropy'])
 print("Loaded model from disk")
 y_testa = model.predict([test_inputs[0],test_inputs[1],test_inputs[2],test_inputst[0],test_inputst[1],test_inputst[2],
test_inputsq[0],test_inputsq[1],test_inputsq[2],test_inputsa[0],test_inputsa[1],test_inputsa[2]])
 print("y_testa shape", y_testa.shape)
 print("Saved model to disk")
 del model
gc.collect()
y_test = np.concatenate(( y_testq, y_testa), axis=1)
 print("y_test shape", y_test.shape)
 all_predictions.append(y_test)
 gc.collect()
gbr=GradientBoostingRegressor(learning_rate=0.01, max_leaf_nodes=10, criterion='friedman_mse', max_depth=11, max_features='sqrt', n_estimators=1000 )<train_model>
test_preds = np.mean(all_predictions, axis=0 )
gbr.fit(train,y_train )<predict_on_test>
output =0.23*output_roberta + 0.31*output_xlnet + 0.24*test_preds + 0.22*final_predictions
pred5=gbr.predict(test )<load_from_csv>
culture_category_list = list(df_test[(df_test['host'] == 'english.stackexchange.com')|(df_test['host'] == 'ell.stackexchange.com')].index )
submissionm = pd.read_csv('.. /input/sample_submission.csv')
 submissionm["target"]=pred5<feature_engineering>
num_raters_dict = {'question_asker_intent_understanding': 18,
'question_body_critical': 18,
'question_conversational': 6,
'question_expect_short_answer': 6,
'question_fact_seeking': 6,
'question_has_commonly_accepted_answer': 6,
'question_interestingness_others': 18,
'question_interestingness_self': 18,
'question_multi_intent': 6,
'question_not_really_a_question': 6,
'question_opinion_seeking': 6,
'question_type_choice': 6,
'question_type_compare': 6,
'question_type_consequence': 6,
'question_type_definition': 6,
'question_type_entity': 6,
'question_type_instructions': 6,
'question_type_procedure': 6,
'question_type_reason_explanation': 6,
'question_type_spelling': 3,
'question_well_written': 18,
'answer_helpful': 18,
'answer_level_of_information': 18,
'answer_plausible': 18,
'answer_relevance': 18,
'answer_satisfaction': 30,
'answer_type_instructions': 6,
'answer_type_procedure': 6,
'answer_type_reason_explanation': 6,
'answer_well_written': 18
}
j=0
for i in submissionm["target"]:
if i>1:
submissionm["target"][j]=1
j+=1<save_to_csv>
def submssion_trick(output):
predictions = np.zeros_like(output)
 for i in range(output.shape[0]):
for j in range(output.shape[1]):
col = output_categories[j]
num_raters = num_raters_dict[col]
if j == 19:
if i not in culture_category_list:
predictions[i][j] = 0.0
else:
predictions[i][j] = np.floor(output[i][j] * num_raters)/ num_raters
else:
predictions[i][j] = np.floor(output[i][j] * num_raters)/ num_raters
if num_raters == 18:
predictions[i][j] = max(predictions[i][j], 1/3)
 if num_raters == 30:
predictions[i][j] = max(predictions[i][j], 1/5)
 return predictions
submissionm.to_csv('submission_gbr.csv', index=False )<define_variables>
scaler = MinMaxScaler()
for i in range(30):
output[:, i] = scaler.fit_transform(output[:, i].reshape(-1, 1)).reshape(-1, )
finalsub=(submissionm["target"]+submission1["target"])/2<load_from_csv>
predictions = submssion_trick(output )
subf= pd.read_csv('.. /input/sample_submission.csv')
 subf["target"]=finalsub<save_to_csv>
submission = pd.read_csv(PATH +'sample_submission.csv')
 for i in range(30):
if len(np.unique(predictions[:, i])) == 1:
print(i)
 print(submission.columns[i+1])
 print('before', np.unique(predictions[:, i]))
print('after', np.unique(predictions[:, i]))
<set_options><EOS> 
submission.loc[:, 'question_asker_intent_understanding':] = predictions
submission.to_csv('submission.csv', index=False)
 submission.head(10 )