Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<init_hyperparams>
!pip install.. /input/huggingface-transformers/sacremoses-master/sacremoses-master
!pip install.. /input/huggingface-transformers/transformers-master/transformers-master
scores_auc_all = []
test_cv_preds = []
NB_SPLITS = 7
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
SEED =(42,43,44)
 for s in SEED:
tabnet_params['seed'] = s
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = s, shuffle = True)
 for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train_df, targets)) :
print(b_,"FOLDS: ", r_, fold_nb + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 X_train, y_train = train_df.values[train_idx, :], targets.values[train_idx, :]
X_val, y_val = train_df.values[val_idx, :], targets.values[val_idx, :]
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train,
y_train = y_train,
eval_set = [(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = MAX_EPOCH,
patience = 20,
batch_size = 1024,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits 
)
 print(y_, '-' * 60)
 print(saving_path_name)
 saving_path_name = 'TabNet_seed_'+str(tabnet_params['seed'])+'_fold_'+str(fold_nb+1)
 saved_filepath = model.save_model(saving_path_name)
 print(saved_filepath)
 input('Press enter to continue')
 loaded_model = TabNetRegressor()
loaded_model.load_model(saving_path_name+'.zip')
 preds_val = loaded_model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 oof_preds.append(preds_val)
 oof_targets.append(y_val)
 scores.append(score)
 preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<choose_model_class>
tqdm.pandas()
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
saving_path_name = 'TabNet_seed_'+str(tabnet_params['seed'])+'_fold_'+str(fold_nb+1)
 loaded_model = TabNetRegressor()
loaded_model.load_model(saved_filepath)
 preds_train = loaded_model.predict_proba(X_train)
 print(preds_train[:,1] )<compute_test_metric>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer_google_qa = tokenization.FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 tokenizer2 = BertTokenizer.from_pretrained(BERT_PATH+'/assets/vocab.txt', do_lower_case=True,)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv',header=0,encoding='utf-8')
 df_test = pd.read_csv(PATH+'test.csv',header=0,encoding='utf-8')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 df = pd.concat([df_train,df_test],axis=0,ignore_index=True)
 print(f)
 output_categories = ['question_asker_intent_understanding', 'question_body_critical', 'question_conversational', 'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer', 'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent', 'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare', 'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions', 'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written', 'answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance', 'answer_satisfaction', 'answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written']
input_categories = ['question_title', 'question_body', 'answer']
print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
 print(f"{b_}Average CV: {r_}{np.mean(scores)}" )<save_to_csv>
tree_tokenizer = TreebankWordTokenizer()
def get_tree_tokens(x):
x = tree_tokenizer.tokenize(x)
 x = ' '.join(x)
 return x
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()<import_modules>
for col in input_categories:
df_train[f'treated_{col}'] = df_train[col].progress_apply(lambda x: get_tree_tokens(x))
df_test[f'treated_{col}'] = df_test[col].progress_apply(lambda x: get_tree_tokens(x))
df[f'treated_{col}'] = df[col].progress_apply(lambda x: get_tree_tokens(x))
pd.options.display.max_columns = None 
<load_from_csv>
tokenizer = text.Tokenizer(lower=False )
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_features['dataset'] = 'train'
test_features['dataset'] = 'test'
df = pd.concat([train_features, test_features] )<load_from_csv>
X_train_question = df_train['question_body']
X_train_title = df_train['question_title']
X_train_answer = df_train['answer']
X_test_question = df_test['question_body']
X_test_title = df_test['question_title']
X_test_answer = df_test['answer']
sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sample_submission<define_variables>
tokenizer.fit_on_texts(list(X_train_title)+list(X_train_question)+list(X_train_answer)+list(X_test_title)+list(X_test_question)+list(X_test_answer))
cp_width = 500
cp_height = 400
scatter_size = 600<train_model>
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer )
plt.figure(figsize=(15,5))
sns.distplot(train_features['cp_time'], color='red', bins=5)
 plt.title("Train: Treatment duration ", fontsize=15, weight='bold')
 plt.show()<define_variables>
def split_document(texts):
all_sents = []
max_num_sentences = 0.0
for text in texts:
doc = nlp(text)
 sents=[]
for i,sent in enumerate(doc.sents):
sents.append(sent.text)
 all_sents.append(sents)
 return all_sents
X_train_question = split_document(X_train_question)
 X_train_answer = split_document(X_train_answer)
 X_test_question = split_document(X_test_question)
 X_test_answer = split_document(X_test_answer)
train_columns = train_features.columns.to_list()
g_list = [i for i in train_columns if i.startswith('g-')]
c_list = [i for i in train_columns if i.startswith('c-')]<count_values>
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words] )
all_columns = list(set(all_columns))
print('Number of columns:', len(all_columns))<load_from_csv>
ans_user_and_category=df_train[df_train[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape
train_targets = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv")
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 print('Number of rows: ', train_targets.shape[0])
 print('Number of cols: ', train_targets.shape[1])
 train_targets.head()<create_dataframe>
def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same
%%time
correlation_matrix = pd.DataFrame()
for t_col in train_targets.columns:
corr_list = list()
if t_col == 'sig_id':
continue
for col in columns:
res = train_features[col].corr(train_targets[t_col])
 corr_list.append(res)
 correlation_matrix[t_col] = corr_list<feature_engineering>
def add_external_features(df):
df['question_body'] = df['question_body'].progress_apply(lambda x: str(x))
df['question_body_num_words'] = df['question_body'].str.count('\S+')
 df['answer'] = df['answer'].progress_apply(lambda x: str(x))
df['answer_num_words'] = df['answer'].str.count('\S+')
 df['question_vs_answer_length'] = df['question_body_num_words']/df['answer_num_words']
df['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in tqdm(df[['answer_user_name', 'category']].values):
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
df['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
return df, np.array(handmade_features)
correlation_matrix['train_features'] = columns
correlation_matrix = correlation_matrix.set_index('train_features')
 correlation_matrix<merge>
df_train, handmade_features = add_external_features(df_train)
 df_test, handmade_features_test = add_external_features(df_test)
 df_train = pd.concat([df_train,pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1)
 df_test = pd.concat([df_test,pd.DataFrame(handmade_features_test, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1 )
col_df = pd.DataFrame()
tr_cols = list()
tar_cols = list()
for col in correlation_matrix.columns:
tar_cols.append(col)
 tr_cols.append(
 correlation_matrix[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(1 ).values[0] 
)
 col_df['column'] = tar_cols
col_df['train_best_column'] = tr_cols
total_scores = pd.merge(high_scores, col_df)
 total_scores<feature_engineering>
num_words_scaler = MinMaxScaler()
df_train[['question_body_num_words', 'answer_num_words']] = num_words_scaler.fit_transform(df_train[['question_body_num_words', 'answer_num_words']].values)
 df_test[['question_body_num_words', 'answer_num_words']] = num_words_scaler.transform(df_test[['question_body_num_words', 'answer_num_words']].values )
col_df = pd.DataFrame()
tr_first_cols = list()
tr_second_cols = list()
tar_cols = list()
for col in current_corr.columns:
tar_cols.append(col)
 tr_first_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(2 ).values[0] 
)
 tr_second_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(2 ).values[1] 
)
 col_df['column'] = tar_cols
col_df['train_1_column'] = tr_first_cols
col_df['train_2_column'] = tr_second_cols
col_df 
<define_variables>
df=pd.concat([df,pd.get_dummies(df['host'], drop_first=False, prefix='host')],axis=1)
 df=pd.concat([df,pd.get_dummies(df['category'], drop_first=False, prefix='cat')],axis=1 )
for_analysis = [
target_columns[np.random.randint(0, len(target_columns)-1)] for i in range(5)
 ]
current_corr = correlation_matrix[for_analysis]
col_df = pd.DataFrame()
tr_first_cols = list()
tr_second_cols = list()
tr_third_cols = list()
tar_cols = list()
for col in current_corr.columns:
tar_cols.append(col)
 tr_first_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(3 ).values[0] 
)
 tr_second_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(3 ).values[1] 
)
 tr_third_cols.append(
 current_corr[col].abs().sort_values(ascending=False ).reset_index() ['train_features'].head(3 ).values[2] 
)
 col_df['column'] = tar_cols
col_df['train_1_column'] = tr_first_cols
col_df['train_2_column'] = tr_second_cols
col_df['train_3_column'] = tr_third_cols
col_df 
<prepare_x_and_y>
len(['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')] )
def plot_3dscatter(col_df, index):
analysis = pd.DataFrame()
analysis['color'] = train_targets[col_df.iloc[index]['column']]
analysis['x'] = train_features[col_df.iloc[index]['train_1_column']]
analysis['y'] = train_features[col_df.iloc[index]['train_2_column']]
analysis['z'] = train_features[col_df.iloc[index]['train_3_column']]
analysis.columns = [
'color',
col_df.iloc[index]['train_1_column'],
col_df.iloc[index]['train_2_column'],
col_df.iloc[index]['train_3_column']
]
analysis['size'] = 1
analysis.loc[analysis['color'] == 1, 'size'] = 20
fig = px.scatter_3d(
 analysis,
x=col_df.iloc[index]['train_1_column'],
y=col_df.iloc[index]['train_2_column'],
z=col_df.iloc[index]['train_3_column'],
color="color",
size='size',
height=scatter_size,
width=scatter_size,
title='Scatter plot for ' + col_df.iloc[index]['column'] 
)
 fig.show()<categorify>
df_train=pd.merge(df_train, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id')
 df_test = pd.merge(df_test, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id' )
answer = list()
for group in last_term.group.tolist() :
agent_list = list()
for item in target_columns:
if item.split('_')[-1] == group:
agent_list.append(item)
 agent_df = train_targets[agent_list]
data = agent_df.astype(bool ).sum(axis=1 ).reset_index()
answer.append(data[0].max() )<drop_column>
categories = train_features[['cp_type', 'cp_time', 'cp_dose']]
tar = train_targets.copy()
tar = tar.drop(['sig_id'], axis=1)
 analysis = pd.concat([categories, tar], axis=1 )<count_unique_values>
for category in analysis['cp_dose'].unique().tolist() :
number = 0
cols = list()
for col in analysis.columns:
if col in ['cp_type', 'cp_time', 'cp_dose']:
continue
if len(analysis[analysis['cp_dose'] == category][col].value_counts())== 1:
number += 1
cols.append(col)
 print(category, '.Number of columns with 1 unique value: ', number, '.Columns: ', cols )<count_values>
df_train.drop(['host', 'category'], inplace=True, axis=1)
 df_test.drop(['host', 'category'], inplace=True, axis=1 )
analysis[analysis['cp_dose'] == 'D2']['atp-sensitive_potassium_channel_antagonist'].value_counts()<count_values>
def _get_masks(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+[0]*(max_seq_length-len(tokens))
def _get_segments(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments=[]
first_sep=True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == '[SEP]':
current_segment_id=1
return segments+[0]*(max_seq_length-len(tokens))
def _trim_input_questions(title, question, max_sequence_length, t_max_len=30, q_max_len=479):
t = tokenizer2.tokenize(title)
 q = tokenizer2.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if q_max_len > q_len:
q_new_len = q_len
else:
q_new_len = q_max_len
if t_new_len+q_new_len+3 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+q_new_len+3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _get_ids(tokens, tokenizer, max_seq_length):
token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0]*(max_seq_length - len(token_ids))
return input_ids
def _convert_to_bert_inputs_questions(title, question,tokenizer, max_seq_length):
stoken = ['[CLS]']+title+['[SEP]']+question+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_questions(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
t, q = col['treated_question_title'], col['treated_question_body']
t,q = _trim_input_questions(t,q, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_questions(t,q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def _trim_input_answers(answer, max_sequence_length, a_max_len=510):
a = tokenizer2.tokenize(answer)
 a_len = len(a)
 if(a_len+2)> max_sequence_length:
if a_max_len > a_len:
a_new_len = a_len
else:
a_new_len = a_max_len
if a_new_len+2 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(a_new_len+2)))
 a = a[:a_new_len]
return a
def _convert_to_bert_inputs_answers(answer, tokenizer, max_seq_length):
stoken = ['[CLS]']+answer+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
a = col['treated_answer']
a = _trim_input_answers(a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
 def compute_spearmanr(trues, preds):
rhos=[]
for t, p in zip(trues.T, preds.T):
rhos.append(
 spearmanr(t,p ).correlation 
)
 return np.nanmean(rhos)
 test_predictions=[]
class CustomCallback(keras.callbacks.Callback):
def __init__(self, valid_data, test_data, test_predictions=test_predictions, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.test_predictions = test_predictions
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions=[]
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print(f"
validation rho: {round(rho_val,4)}")
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size))
analysis[analysis['cp_dose']=='D2']['erbb2_inhibitor'].value_counts()<count_unique_values>
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True)
 def bertModel() :
input_ids_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_q')
 input_mask_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_q')
 input_segments_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_q')
 input_new_features = keras.layers.Input(( 9,), dtype=tf.float32, name='input_new_features')
 input_ids_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_a')
 input_mask_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_a')
 input_segments_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_a')
 bert_model_q = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 bert_model_a = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 _, _, hidden_states_q = bert_model_q([input_ids_q,input_mask_q, input_segments_a])
 _, _, hidden_states_a = bert_model_a([input_ids_a,input_mask_a, input_segments_a])
 h12_q = tf.reshape(hidden_states_q[-1][:,0],(-1,1,768))
h11_q = tf.reshape(hidden_states_q[-2][:,0],(-1,1,768))
h10_q = tf.reshape(hidden_states_q[-3][:,0],(-1,1,768))
h09_q = tf.reshape(hidden_states_q[-4][:,0],(-1,1,768))
concat_hidden_q = keras.layers.Concatenate(axis=2 )([h12_q, h11_q, h10_q, h09_q])
 h12_a = tf.reshape(hidden_states_a[-1][:,0],(-1,1,768))
h11_a = tf.reshape(hidden_states_a[-2][:,0],(-1,1,768))
h10_a = tf.reshape(hidden_states_a[-3][:,0],(-1,1,768))
h09_a = tf.reshape(hidden_states_a[-4][:,0],(-1,1,768))
concat_hidden_a = keras.layers.Concatenate(axis=2 )([h12_a, h11_a, h10_a, h09_a])
 x = keras.layers.GlobalAveragePooling1D()(concat_hidden_q)
 x_a = keras.layers.GlobalAveragePooling1D()(concat_hidden_a)
 x = keras.layers.Concatenate()([x, input_new_features])
 x_a = keras.layers.Concatenate()([x_a,x, input_new_features])
 out = keras.layers.Dense(len([i for i in output_categories if i.startswith('question')]), activation='sigmoid', name='final_dense_output_q' )(x)
 out_a = keras.layers.Dense(len([i for i in output_categories if i.startswith('answer')]), activation='sigmoid', name='final_dense_output_a' )(x_a)
 out = keras.layers.Concatenate()([out,out_a])
 model = keras.models.Model(inputs=[input_ids_q, input_mask_q, input_segments_q,
input_new_features, input_ids_a, input_mask_a, input_segments_a]
, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer = keras.optimizers.Adam(lr=3e-5))
return model
for category in analysis['cp_time'].unique().tolist() :
number = 0
cols = list()
for col in analysis.columns:
if col in ['cp_type', 'cp_time', 'cp_dose']:
continue
if len(analysis[analysis['cp_time']==category][col].value_counts())== 1:
number += 1
cols.append(col)
 print(category, '.Number of columns with 1 unique value: ', number, '.Columns: ', cols )<count_values>
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body )
analysis[analysis['cp_time'] == 24]['erbb2_inhibitor'].value_counts()<count_values>
outputs = compute_output_arrays(df_train, output_categories)
 inputs_q = compute_input_array_questions(df_train, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 inputs_a = compute_input_array_answers(df_train, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_q = compute_input_array_questions(df_test, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_a = compute_input_array_answers(df_test, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH )
analysis[analysis['cp_time'] == 72]['erbb2_inhibitor'].value_counts()<count_values>
analysis[analysis['cp_time'] == 24]['atp-sensitive_potassium_channel_antagonist'].value_counts()<count_values>
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold>8:
keras.backend.clear_session()
model = bertModel()
train_inputs_q = [inputs_q[i][train_idx] for i in range(3)]
train_inputs_a = [inputs_a[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs_q = [inputs_q[i][valid_idx] for i in range(3)]
valid_inputs_a = [inputs_a[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
custom_callback = CustomCallback(valid_data=(valid_inputs_q +
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[valid_idx,:].values]+ valid_inputs_a,
valid_outputs),
test_data=test_inputs_q +
[df_test[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].values]+test_inputs_a,
batch_size=8, fold=fold)
 H = model.fit(train_inputs_q+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[train_idx,:].values] + train_inputs_a,
train_outputs, batch_size=4, epochs=4, callbacks=[custom_callback])
 histories.append(H)
 model.save_weights(f'/kaggle/working/bert_base_fold_{fold}.hdf5' )
analysis[analysis['cp_time'] == 72]['atp-sensitive_potassium_channel_antagonist'].value_counts()<count_unique_values>
def _get_masks_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_google_qa(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input_google_qa(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer_google_qa.tokenize(title)
 q = tokenizer_google_qa.tokenize(question)
 a = tokenizer_google_qa.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_google_qa(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_google_qa(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks_google_qa(stoken, max_sequence_length)
 input_segments = _get_segments_google_qa(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_google_qa(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance['treated_question_title'], instance['treated_question_body'], instance['treated_answer']
t, q, a = _trim_input_google_qa(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_google_qa(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays_google_qa(df, columns):
return np.asarray(df[columns])
 def bert_model_google_qa(learning_rate=4e-5,loss_function='binary_crossentropy'):
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[[input_word_ids, input_masks, input_segments]], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 return model
for category in analysis['cp_type'].unique().tolist() :
number = 0
cols = list()
for col in analysis.columns:
if col in ['cp_type', 'cp_time', 'cp_dose']:
continue
if len(analysis[analysis['cp_type'] == category][col].value_counts())== 1:
number += 1
cols.append(col)
 print(category, '.Number of columns with 1 unique value: ', number, '.Columns: ', cols )<count_values>
gkf_google_qa = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs_google_qa = compute_output_arrays_google_qa(df_train, output_categories)
 inputs_google_qa = compute_input_arays_google_qa(df_train, ['treated_question_title','treated_question_body','treated_answer'], tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
 test_inputs_google_qa = compute_input_arays_google_qa(df_test, ['treated_question_title','treated_question_body','treated_answer'],tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
analysis[analysis['cp_type']=='ctl_vehicle']['igf-1_inhibitor'].value_counts()<concatenate>
test_predictions_google_qa=[]
for fold,(train_idx, valid_idx)in enumerate(gkf_google_qa):
if fold<5:
keras.backend.clear_session()
model_qa = bert_model_google_qa()
print(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_quest_kernel_{fold}.hdf5')
 model_qa.load_weights(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert_base_quest_kernel_{fold}.hdf5')
 preds_qa = model_qa.predict(test_inputs_google_qa , batch_size=8, verbose=1)
 test_predictions_google_qa.append(preds_qa)
 print(f'Fold:{fold} Completed' )
data = train_features.append(test_features )<categorify>
test_preds_google_qa = [test_predictions_google_qa[i] for i in range(len(test_predictions_google_qa)) ]
test_preds_google_qa = [np.average(test_preds_google_qa, axis=0)for i in range(len(test_preds_google_qa)) ]
test_preds_google_qa = np.mean(test_preds_google_qa, axis=0)
 test_preds_google_qa.shape
def preprocess(df):
df = df.copy()
df.drop(['cp_type'], axis=1, inplace=True)
 df.drop(['dataset'],axis=1,inplace=True)
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 df = pd.get_dummies(df, columns=['cp_time','cp_dose'])
 del df['sig_id']
return df
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<normalization>
final_preds = np.average(np.array([test_preds, test_preds_google_qa]),axis=0)
 final_preds.shape
scaler = preprocessing.MinMaxScaler()
scaler.fit(train.append(test))
train_trans = scaler.transform(train)
 test_trans = scaler.transform(test)
 train = pd.DataFrame(train_trans, columns=train.columns)
 test = pd.DataFrame(test_trans, columns=test.columns )<compute_test_metric>
df_sub.iloc[:, 1:] = final_preds
df_sub.to_csv('submission.csv', index=False )