Kaggle competition: Google QUEST Q&A Labeling
<split><EOS> 
import pandas as pd
import numpy as np
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<load_pretrained>
import pandas as pd
import numpy as np
embedding_matrix = np.load('.. /input/crawlembeddingmatric/embedding_matrix.npy')
 embedding_layer = Embedding(len(word_index)+ 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False )<compute_train_metric>
PATH = '.. /input/google-quest-challenge/'
def auc(y_true, y_pred):
ptas = tf.stack([binary_PTA(y_true,y_pred,k)for k in np.linspace(0, 1, 1000)],axis=0)
 pfas = tf.stack([binary_PFA(y_true,y_pred,k)for k in np.linspace(0, 1, 1000)],axis=0)
 pfas = tf.concat([tf.ones(( 1,)) ,pfas],axis=0)
 binSizes = -(pfas[1:]-pfas[:-1])
 s = ptas*binSizes
return K.sum(s, axis=0)
 def binary_PFA(y_true, y_pred, threshold=K.variable(value=0.5)) :
y_pred = K.cast(y_pred >= threshold, 'float32')
 N = K.sum(1 - y_true)
 FP = K.sum(y_pred - y_pred * y_true)
 return FP/N
def binary_PTA(y_true, y_pred, threshold=K.variable(value=0.5)) :
y_pred = K.cast(y_pred >= threshold, 'float32')
 P = K.sum(y_true)
 TP = K.sum(y_pred * y_true)
 return TP/P<choose_model_class>
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv' )
def get_model() :
class AttLayer(Layer):
def __init__(self, init='glorot_uniform', kernel_regularizer=None,
bias_regularizer=None, kernel_constraint=None,
bias_constraint=None, **kwargs):
self.supports_masking = True
self.init = initializers.get(init)
 self.kernel_initializer = initializers.get(init)
 self.kernel_regularizer = regularizers.get(kernel_regularizer)
 self.bias_regularizer = regularizers.get(kernel_regularizer)
 self.kernel_constraint = constraints.get(kernel_constraint)
 self.bias_constraint = constraints.get(bias_constraint)
 super(AttLayer, self ).__init__(** kwargs)
 def build(self, input_shape):
assert len(input_shape)==3
self.W = self.add_weight(( input_shape[-1], 1),
initializer=self.kernel_initializer,
name='{}_W'.format(self.name),
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
 self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
 self.u = self.add_weight(( input_shape[1],),
initializer=self.kernel_initializer,
name='{}_u'.format(self.name),
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
 self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = K.dot(x, self.W)
 uit = K.squeeze(uit, -1)
 uit = uit + self.b
uit = K.tanh(uit)
 ait = uit * self.u
ait = K.exp(ait)
 if mask is not None:
mask = K.cast(mask, K.floatx())
 ait = mask*ait
ait /= K.cast(K.sum(ait, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 ait = K.expand_dims(ait)
 weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
 return output
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[-1])
 inputs = Input(shape=(MAX_SEQUENCE_LENGTH,))
embed = embedding_layer(inputs)
 gru = Bidirectional(GRU(100, dropout=0.2, recurrent_dropout=0.1, return_sequences=True))(embed)
 attention = AttLayer()(gru)
 output = Dense(1, activation='sigmoid' )(attention)
 model = Model(inputs, output)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[auc])
 return model<load_pretrained>
train_x = train.loc[:, 'qa_id':'host']
model = get_model()
model.load_weights(".. /input/donorweight/atten0.7822.h5" )<choose_model_class>
train_y = train.loc[:, 'question_asker_intent_understanding':'answer_well_written']
checkpoint = ModelCheckpoint('atten.h5', monitor="val_auc", verbose=1, save_best_only=True, mode='max')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.000001 )<train_model>
train_x = train_x[['question_title','question_body','answer']]
test_x = test[['question_title','question_body','answer']]
<save_to_csv>
import tensorflow_hub as hub
import tensorflow as tf
import bert_tokenization as tokenization
from tensorflow.keras.models import Model
import tensorflow.keras.backend as K
import math
p_sub= model.predict([test_data] ).T[0]
sub = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 sub.project_is_approved = p_sub
sub.to_csv('attention.csv', index=False )<set_options>
MAX_SEQ_LENGTH = 512
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
plt.style.use('Solarize_Light2')
 %matplotlib inline
pd.set_option("display.max_columns", 2**10 )<load_from_csv>
def create_model(max_seq_length=MAX_SEQ_LENGTH,bert_path=BERT_PATH):
input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,
name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,
name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,
name="segment_ids")
 bert_layer = hub.KerasLayer(bert_path,
trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 pooling = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 dropout = tf.keras.layers.Dropout(0.2 )(pooling)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(dropout)
 return Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out )
date_cols = ['project_submitted_datetime']
train = pd.read_csv(os.path.join(r'.. /input', 'train.csv'), low_memory=False, parse_dates=date_cols)
 test = pd.read_csv(os.path.join(r'.. /input', 'test.csv'), low_memory=False, parse_dates=date_cols )<concatenate>
def get_masks(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def get_segments(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
current_segment_id = 0
first_sep = True
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def get_ids(tokens, tokenizer, max_seq_length):
token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t_len,q_len,a_len = len(title),len(question),len(answer)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len <= t_len:
t_new_len = t_max_len
else:
t_new_len = t_len
a_max_len = a_max_len + math.floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + math.ceil(( t_max_len - t_len)/2)
 if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 title,question,answer = title[:t_new_len], question[:q_new_len], answer[:a_new_len]
return title,question,answer
def get_inputs(title, question, answer, tokenizer,max_seq_length):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t,q,a = trim_input(t, q, a,max_seq_length)
 stokens = ["[CLS]"] + t + ["[SEP]"] + q + ["[SEP]"] + a + ["[SEP]"]
input_ids = get_ids(stokens, tokenizer, max_seq_length)
 input_masks = get_masks(stokens, max_seq_length)
 input_segments = get_segments(stokens, max_seq_length)
 return input_ids,input_masks,input_segments
def compute_input_arays(df, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in df.iterrows() :
t, q, a = instance.question_title, instance.question_body, instance.answer
ids, masks, segments = get_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
train['source'] = 'train'
test['source'] = 'test'
test_train = pd.concat(( test, train))
print(f'The shape of the data is {test_train.shape}.' )<filter>
from scipy.stats import spearmanr
print("="*60)
 print("Detecting NaN values in data:")
 print("="*60)
 print(test_train.isnull().sum(axis=0)[test_train.isnull().sum(axis=0)> 0] )<count_values>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
 )
most_common = test_train.teacher_prefix.value_counts().idxmax()
test_train.teacher_prefix = test_train.teacher_prefix.fillna(most_common )<define_variables>
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
numerical_cols = []
dummy_categorical_cols = []
text_cols = []<drop_column>
from sklearn.model_selection import GroupKFold
mask_four_essays = ~(test_train.project_essay_3.isnull() & test_train.project_essay_4.isnull())
 test_train[mask_four_essays] =(test_train[mask_four_essays] 
.assign(project_essay_1 = lambda df: df.project_essay_1 + df.project_essay_2)
.assign(project_essay_2 = lambda df: df.project_essay_3 + df.project_essay_4))
test_train = test_train.drop(columns=['project_essay_3', 'project_essay_4'] )<load_from_csv>
gkf = GroupKFold(n_splits=5 ).split(X=train_x.question_body, groups=train_x.question_body )
resources = pd.read_csv(os.path.join(r'.. /input', 'resources.csv'), low_memory=False)
 print(f'The shape of the data is {resources.shape}.' )<filter>
tokenizer = tokenization.FullTokenizer(BERT_PATH + '/assets/vocab.txt' )
print("="*30)
 print("Detecting NaN values in data:")
 print("="*30)
 print(resources.isnull().sum(axis=0)[resources.isnull().sum(axis=0)> 0] )<correct_missing_values>
inputs = compute_input_arays(train_x, tokenizer,MAX_SEQ_LENGTH)
 test_inputs = compute_input_arays(test_x, tokenizer,MAX_SEQ_LENGTH)
 outputs = np.asarray(train_y )
resources = resources.fillna('X' )<count_missing_values>
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = create_model()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=4, batch_size=8,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history )
<rename_columns><EOS> 
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 df_sub = pd.read_csv(PATH + 'sample_submission.csv')
 df_sub.iloc[:, 1:] = test_predictions
df_sub.to_csv('submission.csv', index=False )