Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<feature_engineering>
import pandas as pd
import numpy as np
import gc
import os
from math import floor, ceil
from tqdm.notebook import tqdm
from sklearn.model_selection import GroupKFold
from scipy.stats import spearmanr
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow.keras.backend as K
from transformers import BertTokenizer
from transformers import BertConfig
from transformers import TFBertModel
for key, value in words_dict.items() :
words_to_ids[value] = key<categorify>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 MAX_SEQUENCE_LENGTH = 384
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
test_x_words = np.array(convert_tokens_to_embedded(df_test['tokenized_text'],
words_to_ids, embedding_word_dict,
padding = embedding_word_dict['nanword'],
sentences_length = 630))<save_to_csv>
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \
_convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
model.load_weights(file_path)
 pred_test = model.predict([test_x_words, test_x_num], batch_size=1024, verbose=1)
 df_test["project_is_approved"] = pred_test
df_test[['id', 'project_is_approved']].to_csv("submission.csv", index=False )<import_modules>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 def create_model() :
config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 x = tf.keras.layers.Concatenate()([q, a])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)
 return model
import gc
import numpy as np
import pandas as pd
import os
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold, RepeatedKFold
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import LabelEncoder
from sklearn import metrics
from keras import initializers, regularizers, constraints
from keras.layers import Dense, Input, Flatten, RepeatVector, Permute
from keras.layers import Input, Dense, LSTM,merge, Merge, Bidirectional, concatenate, SpatialDropout1D, GRU, BatchNormalization, Dropout, Activation,TimeDistributed
from keras.preprocessing.text import Tokenizer
from keras.layers.embeddings import Embedding
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model, Sequential
from keras.regularizers import l2
from keras.layers import Convolution1D, GlobalMaxPooling1D, Flatten
from keras.layers import Conv1D, MaxPooling1D, Embedding,Lambda
from keras.preprocessing.text import text_to_word_sequence
from keras.callbacks import *
from keras import backend as K
import tensorflow as tf<load_from_csv>
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
train = pd.read_csv('.. /input/donorschoose-application-screening/train.csv')
 test = pd.read_csv('.. /input/donorschoose-application-screening/test.csv')
 resources = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv')
 train = train.sort_values(by="project_submitted_datetime" )<categorify>
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 valid_preds = []
test_preds = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 2]:
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_outputs = outputs[valid_idx]
K.clear_session()
model = create_model()
optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_inputs, train_outputs, epochs=3, batch_size=6)
 valid_preds.append(model.predict(valid_inputs))
test_preds.append(model.predict(test_inputs))
rho_val = compute_spearmanr_ignore_nan(valid_outputs, valid_preds[-1])
 print('validation score = ', rho_val )
<string_transform><EOS> 
df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)
 df_sub.to_csv('submission.csv', index=False )