Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<choose_model_class>
np.set_printoptions(suppress=True)
 print(tf.__version__ )
def create_model() :
model = MoAModel(device, dtype, num_in_features, num_hidden_features, num_out_features)
 optimizer = optim.Adam(model.parameters() , lr=lr, weight_decay=1e-5)
 lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=26, gamma=0.1)
 return model, optimizer, lr_scheduler<define_variables>
!pip install.. /input/sacremoses > /dev/null
sys.path.insert(0, ".. /input/transformers/")
batch_size = 2 ** 7
n_folds = 7<split>
PATH = '.. /input/google-quest-challenge/'
core = 'bert'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
WEIGHTS_FILES_10 = sorted(glob('.. /input/bertbasetriplet5cv/*.h5'))
WEIGHTS_FILES_6_NOES = sorted(glob('.. /input/bert-base-5cv/*.h5'))
WEIGHTS_FILES_6 = sorted(glob('.. /input/bert-base-double-5cv-early-stop/*.h5'))
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories)
 print('
trained weights files(10 inputs):
\t', WEIGHTS_FILES_10)
 print('
trained weights files(6 inputs, without early stop):
\t', WEIGHTS_FILES_6_NOES)
 print('
trained weights files(6 inputs):
\t', WEIGHTS_FILES_6 )
def train_NN_by_folds(n_folds, models_path):
Path(models_path ).mkdir(parents=True, exist_ok=False)
 kfold = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=517)
 models = []
for i,(train, val)in enumerate(tqdm(
 kfold.split(train_features_enc, train_targets_scored_enc),
total=kfold.n_splits 
)) :
print()
print(f'\tFold {i + 1}')
 x_tr = train_features_enc[train]
y_tr = train_targets_scored_enc[train]
x_val = train_features_enc[val]
y_val = train_targets_scored_enc[val]
train_dataset = MoADataset(dtype, x_tr, y_tr)
 val_dataset = MoADataset(dtype, x_val, y_val)
 train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
 val_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
 model, optimizer, lr_scheduler = create_model()
result = train_NN(model, criterion, optimizer, lr_scheduler,
max_epoch, print_every,
train_dataloader, val_dataloader, test_features_enc, test_zero_moa_mask)
 np.save(f'{models_path}/best_models_kfold_{i + 1}', result.best_models)
 models.append(model )<define_variables>
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 input_ids_qa, input_masks_qa, input_segments_qa = return_id(
 title + ' ' + question, answer, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a,
input_ids_qa, input_masks_qa, input_segments_qa]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
input_ids_qa, input_masks_qa, input_segments_qa = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
inputs = _convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 ids_q, masks_q, segments_q = inputs[:3]
ids_a, masks_a, segments_a = inputs[3:6]
ids_qa, masks_qa, segments_qa = inputs[6:]
input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 input_ids_qa.append(ids_qa)
 input_masks_qa.append(masks_qa)
 input_segments_qa.append(segments_qa)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32),
np.asarray(input_ids_qa, dtype=np.int32),
np.asarray(input_masks_qa, dtype=np.int32),
np.asarray(input_segments_qa, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
models_path = '.. /input/nn-top3ensemble-with-cv7/nn-models/02/' 
<predict_on_test>
def split_sentence(sentence):
return re.split('[\.\?]+\s*', sentence)
 def count_wps(sentence):
sentence_list = split_sentence(sentence)
 w_per_s = np.mean([len(sentence.split())for sentence in sentence_list
if len(sentence)> 0])
 return w_per_s
def build_count_features(df):
count_features = OrderedDict()
count_features['n_title_words'] = df.question_title.apply(lambda x: len(x.split()))
count_features['n_body_words'] = df.question_body.apply(lambda x: len(x.split()))
count_features['n_answer_words'] = df.answer.apply(lambda x: len(x.split()))
count_features['n_body_sentences'] = df.question_body.apply(lambda x: len(split_sentence(x)))
 count_features['n_answer_sentences'] = df.answer.apply(lambda x: len(split_sentence(x)))
 count_features['n_title_wps'] = df.question_title.apply(count_wps)
 count_features['n_body_wps'] = df.question_body.apply(count_wps)
 count_features['n_answer_wps'] = df.answer.apply(count_wps)
 df_count_features = pd.DataFrame(count_features)
 return df_count_features
def build_host_features(df, df_host_count, popular_host_thr):
def _convert(x):
if x not in df_host_count.index:
return 'other'
if df_host_count[x] > popular_host_thr:
return x
else:
return 'other'
return df.host.apply(_convert)
 def build_features(df, df_host_count, popular_host_thr):
return pd.concat(
 [build_count_features(df),
build_host_features(df, df_host_count, 100)],
axis=1 )
all_preds = []
for i in tqdm(range(n_folds)) :
results = np.load(f'{models_path}/best_models_kfold_{i + 1}.npy', allow_pickle=True)
 for best_model in results:
preds = predict(best_model.model, test_features_enc=test_features_enc, zero_moa_mask=test_zero_moa_mask)
 all_preds.append(preds.iloc[:, 1:].values )<prepare_output>
def compute_spearmanr(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, batch_size, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.batch_size = batch_size
self.fold = fold
self.best_rho = -1
self.best_weights = None
self.best_epoch = 0
def on_train_begin(self, logs={}):
self.valid_predictions = []
def on_epoch_end(self, epoch, logs={}):
rho_val = compute_spearmanr(
 self.valid_outputs,
self.model.predict(self.valid_inputs, batch_size=self.batch_size))
print("
validation rho: %.4f" % rho_val)
 if rho_val > self.best_rho:
self.best_rho = rho_val
self.best_weights = self.model.get_weights()
self.best_epoch = epoch+1
else:
self.model.stop_training = True
def on_train_end(self, logs={}):
self.model.set_weights(self.best_weights)
 self.model.save_weights(f'bert-base-{self.fold+1}fold-{self.best_epoch}epoch.h5')
 print(f'Training ends with rho: {self.best_rho}')
 def create_model_10inputs() :
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 qa_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 qa_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 qa_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 feat_count = tf.keras.layers.Input(( 8,), dtype=tf.float32)
 config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(BERT_PATH+'bert-base-uncased-tf_model.h5',
config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
qa_embedding = bert_model(qa_id, attention_mask=qa_mask, token_type_ids=qa_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 qa = tf.keras.layers.GlobalAveragePooling1D()(qa_embedding)
 x = tf.keras.layers.Concatenate()([q, a, qa])
 x = tf.keras.layers.Concatenate()([x, feat_count])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn,
a_id, a_mask, a_atn,
qa_id, qa_mask, qa_atn,
feat_count],
outputs=x)
 return model
answer = preds.copy()
answer.iloc[:, 1:] = np.array(all_preds ).mean(axis=0 )<save_to_csv>
def create_model_6inputs() :
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(BERT_PATH+'bert-base-uncased-tf_model.h5',
config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 x = tf.keras.layers.Concatenate()([q, a])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn,
a_id, a_mask, a_atn,],
outputs=x)
 return model
answer.to_csv('submission.csv', index=False )<load_from_csv>
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_targets_scored = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train_targets_nonscored = pd.read_csv("/kaggle/input/lish-moa/train_targets_nonscored.csv")
 test_features = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 sample_submission = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv" )<set_options>
inputs_count = build_count_features(df_train ).apply(np.log1p)
 test_inputs_count = build_count_features(df_test ).apply(np.log1p)
 ss = StandardScaler()
inputs_count = ss.fit_transform(inputs_count)
 test_inputs_count = ss.transform(test_inputs_count)
 inputs_count = np.asarray(inputs_count, dtype=np.float32)
 test_inputs_count = np.asarray(test_inputs_count, dtype=np.float32 )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<feature_engineering>
n_splits = 5
epochs = 10
batch_size = 4
learning_rate = 3e-5
gkf = GroupKFold(n_splits=n_splits ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs += [test_inputs_count]
rhos_val = []
test_preds = []
model_10inp = create_model_10inputs()
model_6inp = create_model_6inputs()
for fold,(train_idx, valid_idx)in enumerate(gkf):
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_inputs += [inputs_count[train_idx]]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_inputs += [inputs_count[valid_idx]]
valid_outputs = outputs[valid_idx]
K.clear_session()
if False:
model = create_model()
optimizer = tf.keras.optimizers.Adam(learning_rate)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 custom_callback = CustomCallback(
 valid_data=(valid_inputs, valid_outputs),
batch_size=batch_size, 
)
 model.fit(train_inputs, train_outputs,
epochs=epochs,
batch_size=batch_size,
callbacks=[custom_callback])
 model.save_weights(f'bert-base-{fold}of{n_splits}-{epochs}epoch.h5')
 test_preds.append(model.predict(test_inputs))
if True:
weights_10 = WEIGHTS_FILES_10[fold]
print(f'
Load weights from {weights_10}')
 model_10inp.load_weights(weights_10)
 test_preds.append(model_10inp.predict(test_inputs, batch_size=batch_size))
weights_6 = WEIGHTS_FILES_6[fold]
print(f'
Load weights from {weights_6}')
 model_6inp.load_weights(weights_6)
 test_preds.append(model_6inp.predict(test_inputs[:6], batch_size=batch_size))
weights_6_noes = WEIGHTS_FILES_6_NOES[fold]
print(f'
Load weights from {weights_6}')
 model_6inp.load_weights(weights_6_noes)
 test_preds.append(model_6inp.predict(test_inputs[:6], batch_size=batch_size))
<prepare_x_and_y><EOS> 
df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)
 df_sub.to_csv('submission.csv', index=False )