Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<train_model>
!pip install.. /input/huggingface-transformers/sacremoses-master/sacremoses-master
!pip install.. /input/huggingface-transformers/transformers-master/transformers-master
perm = PermutationImportance(model, random_state=1 ).fit(X_train, y_train)
 eli5.show_weights(perm, top=50 )<normalization>
%%time
datadir = Path('/kaggle/input/google-quest-challenge')
 train = pd.read_csv(datadir/'train.csv')
 test = pd.read_csv(datadir/'test.csv')
 sample_submission = pd.read_csv(datadir/'sample_submission.csv' )
top_features = [i[1:] for i in eli5.formatters.as_dataframe.explain_weights_df(perm ).feature if 'BIAS' not in i]
X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<train_model>
feature_columns = [col for col in train.columns if col not in sample_submission.columns]
print("Feature columns are " , feature_columns )
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr1, prediction_lr1, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )<train_model>
target_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible', 'answer_relevance',
'answer_satisfaction', 'answer_type_instructions',
'answer_type_procedure', 'answer_type_reason_explanation',
'answer_well_written']
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )<train_model>
output_categories = list(train.columns[11:41])
 input_categories = list(train.columns[[1,2,5]] )
sfs1 = SFS(model,
k_features=(10, 15),
forward=True,
floating=False,
verbose=0,
scoring='roc_auc',
cv=folds,
n_jobs=-1)
 sfs1 = sfs1.fit(X_train, y_train )<train_model>
tree_tokenizer = TreebankWordTokenizer()
def get_tree_tokens(x):
x = tree_tokenizer.tokenize(x)
 x = ' '.join(x)
 return x
top_features = list(sfs1.k_feature_names_)
 X_train = train[top_features]
X_test = test[top_features]
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test)
 model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, _ = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )<normalization>
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train['target']
X_test = test.drop(['id'], axis=1)
 scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<train_on_grid>

 def validate_case_matches_checkpoint(do_lower_case, init_checkpoint):

 if not init_checkpoint:
return
m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint)
 if m is None:
return
model_name = m.group(1)
 lower_models = [
"uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12",
"multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12"
]
cased_models = [
"cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16",
"multi_cased_L-12_H-768_A-12"
]
is_bad_config = False
if model_name in lower_models and not do_lower_case:
is_bad_config = True
actual_flag = "False"
case_name = "lowercased"
opposite_flag = "True"
if model_name in cased_models and do_lower_case:
is_bad_config = True
actual_flag = "True"
case_name = "cased"
opposite_flag = "False"
if is_bad_config:
raise ValueError(
 "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`."
"However, `%s` seems to be a %s model, so you "
"should pass in `--do_lower_case=%s` so that the fine-tuning matches "
"how the model was pre-training.If this error is wrong, please "
"just comment out this check." %(actual_flag, init_checkpoint,
model_name, case_name, opposite_flag))
def convert_to_unicode(text):

 if six.PY3:
if isinstance(text, str):
return text
elif isinstance(text, bytes):
return text.decode("utf-8", "ignore")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 elif six.PY2:
if isinstance(text, str):
return text.decode("utf-8", "ignore")
 elif isinstance(text, unicode):
return text
else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 else:
raise ValueError("Not running on Python2 or Python 3?")
 def printable_text(text):

 if six.PY3:
if isinstance(text, str):
return text
elif isinstance(text, bytes):
return text.decode("utf-8", "ignore")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 elif six.PY2:
if isinstance(text, str):
return text
elif isinstance(text, unicode):
return text.encode("utf-8")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 else:
raise ValueError("Not running on Python2 or Python 3?")
 def load_vocab(vocab_file):

 vocab = collections.OrderedDict()
index = 0
with tf.io.gfile.GFile(vocab_file, "r")as reader:
while True:
token = convert_to_unicode(reader.readline())
 if not token:
break
token = token.strip()
vocab[token] = index
index += 1
return vocab
def convert_by_vocab(vocab, items):

 output = []
for item in items:
output.append(vocab[item])
 return output
def convert_tokens_to_ids(vocab, tokens):
return convert_by_vocab(vocab, tokens)
 def convert_ids_to_tokens(inv_vocab, ids):
return convert_by_vocab(inv_vocab, ids)
 def whitespace_tokenize(text):

 text = text.strip()
if not text:
return []
tokens = text.split()
return tokens
class FullTokenizer(object):

 def __init__(self, vocab_file, do_lower_case=True):
self.vocab = load_vocab(vocab_file)
 self.inv_vocab = {v: k for k, v in self.vocab.items() }
self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
 self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
 def tokenize(self, text):
split_tokens = []
for token in self.basic_tokenizer.tokenize(text):
for sub_token in self.wordpiece_tokenizer.tokenize(token):
split_tokens.append(sub_token)
 return split_tokens
def convert_tokens_to_ids(self, tokens):
return convert_by_vocab(self.vocab, tokens)
 def convert_ids_to_tokens(self, ids):
return convert_by_vocab(self.inv_vocab, ids)
 class BasicTokenizer(object):

 def __init__(self, do_lower_case=True):

 self.do_lower_case = do_lower_case
def tokenize(self, text):

 text = convert_to_unicode(text)
 text = self._clean_text(text)
 text = self._tokenize_chinese_chars(text)
 orig_tokens = whitespace_tokenize(text)
 split_tokens = []
for token in orig_tokens:
if self.do_lower_case:
token = token.lower()
token = self._run_strip_accents(token)
 split_tokens.extend(self._run_split_on_punc(token))
output_tokens = whitespace_tokenize(" ".join(split_tokens))
return output_tokens
def _run_strip_accents(self, text):

 text = unicodedata.normalize("NFD", text)
 output = []
for char in text:
cat = unicodedata.category(char)
 if cat == "Mn":
continue
output.append(char)
 return "".join(output)
 def _run_split_on_punc(self, text):

 chars = list(text)
 i = 0
start_new_word = True
output = []
while i < len(chars):
char = chars[i]
if _is_punctuation(char):
output.append([char])
 start_new_word = True
else:
if start_new_word:
output.append([])
 start_new_word = False
output[-1].append(char)
 i += 1
return ["".join(x)for x in output]
def _tokenize_chinese_chars(self, text):

 output = []
for char in text:
cp = ord(char)
 if self._is_chinese_char(cp):
output.append(" ")
 output.append(char)
 output.append(" ")
 else:
output.append(char)
 return "".join(output)
 def _is_chinese_char(self, cp):

 if(( cp >= 0x4E00 and cp <= 0x9FFF)or 
(cp >= 0x3400 and cp <= 0x4DBF)or 
(cp >= 0x20000 and cp <= 0x2A6DF)or 
(cp >= 0x2A700 and cp <= 0x2B73F)or 
(cp >= 0x2B740 and cp <= 0x2B81F)or 
(cp >= 0x2B820 and cp <= 0x2CEAF)or 
(cp >= 0xF900 and cp <= 0xFAFF)or 
(cp >= 0x2F800 and cp <= 0x2FA1F)) :
return True
return False
def _clean_text(self, text):

 output = []
for char in text:
cp = ord(char)
 if cp == 0 or cp == 0xfffd or _is_control(char):
continue
if _is_whitespace(char):
output.append(" ")
 else:
output.append(char)
 return "".join(output)
 class WordpieceTokenizer(object):

 def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200):
self.vocab = vocab
self.unk_token = unk_token
self.max_input_chars_per_word = max_input_chars_per_word
def tokenize(self, text):

 text = convert_to_unicode(text)
 output_tokens = []
for token in whitespace_tokenize(text):
chars = list(token)
 if len(chars)> self.max_input_chars_per_word:
output_tokens.append(self.unk_token)
 continue
is_bad = False
start = 0
sub_tokens = []
while start < len(chars):
end = len(chars)
 cur_substr = None
while start < end:
substr = "".join(chars[start:end])
 if start > 0:
substr = "
if substr in self.vocab:
cur_substr = substr
break
end -= 1
if cur_substr is None:
is_bad = True
break
sub_tokens.append(cur_substr)
 start = end
if is_bad:
output_tokens.append(self.unk_token)
 else:
output_tokens.extend(sub_tokens)
 return output_tokens
def _is_whitespace(char):

 , and \r are technically contorl characters but we treat them
if char == " " or char == "\t" or char == "
" or char == "\r":
return True
cat = unicodedata.category(char)
 if cat == "Zs":
return True
return False
def _is_control(char):

 if char == "\t" or char == "
" or char == "\r":
return False
cat = unicodedata.category(char)
 if cat in("Cc", "Cf"):
return True
return False
def _is_punctuation(char):

 cp = ord(char)
 if(( cp >= 33 and cp <= 47)or(cp >= 58 and cp <= 64)or 
(cp >= 91 and cp <= 96)or(cp >= 123 and cp <= 126)) :
return True
cat = unicodedata.category(char)
 if cat.startswith("P"):
return True
return False
lr = linear_model.LogisticRegression(solver='liblinear', max_iter=1000)
 parameter_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2'],
'C' : [0.001, 0.01, 0.1, 1.0, 10.0, 100.0],
'solver': ['newton-cg', 'sag', 'lbfgs']
}
grid_search = GridSearchCV(lr, param_grid=parameter_grid, cv=folds, scoring='roc_auc')
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))<train_on_grid>
np.set_printoptions(suppress=True )
lr = linear_model.LogisticRegression(solver='liblinear', max_iter=10000)
 parameter_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2', 'l1'],
'C' : [0.001, 0.01, 0.08, 0.1, 0.15, 1.0, 10.0, 100.0],
}
grid_search = GridSearchCV(lr, param_grid=parameter_grid, cv=folds, scoring='roc_auc')
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))<train_model>
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True )
model = linear_model.LogisticRegression(class_weight='balanced', penalty='l1', C=0.1, solver='liblinear')
 oof_lr, prediction_lr, scores = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=model )<train_model>
MAX_SEQUENCE_LENGTH = 512
gnb = GaussianNB()
oof_gnb, prediction_gnb, scores_gnb = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=gnb )<train_on_grid>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
abc = AdaBoostClassifier()
parameter_grid = {'n_estimators': [5, 10, 20, 50, 100],
'learning_rate': [0.001, 0.01, 0.1, 1.0, 10.0]
}
grid_search = GridSearchCV(abc, param_grid=parameter_grid, cv=folds, scoring='roc_auc')
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))<train_model>
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True )
abc = AdaBoostClassifier(**grid_search.best_params_)
 oof_abc, prediction_abc, scores_abc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=abc )<train_on_grid>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=8, fold=3):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if(self.fold is not None)and(epoch == 4):
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 print('Model Saved')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_model = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 _,_, sequence_output = bert_model([input_word_ids, input_masks, input_segments])
 h12_q = tf.reshape(sequence_output[-1][:,0],(-1,1,768))
h11_q = tf.reshape(sequence_output[-2][:,0],(-1,1,768))
h10_q = tf.reshape(sequence_output[-3][:,0],(-1,1,768))
h09_q = tf.reshape(sequence_output[-4][:,0],(-1,1,768))
h08_q = tf.reshape(sequence_output[-5][:,0],(-1,1,768))
h07_q = tf.reshape(sequence_output[-6][:,0],(-1,1,768))
h06_q = tf.reshape(sequence_output[-7][:,0],(-1,1,768))
h05_q = tf.reshape(sequence_output[-8][:,0],(-1,1,768))
h04_q = tf.reshape(sequence_output[-9][:,0],(-1,1,768))
concat_hidden = tf.keras.layers.Concatenate(axis=2 )([h12_q, h11_q, h10_q, h09_q,h08_q,h07_q,h06_q,h05_q,h04_q])
 x = tf.keras.layers.GlobalAveragePooling1D()(concat_hidden)
 x = tf.keras.layers.Dropout(0.1 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 model.summary()
return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=3)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
etc = ExtraTreesClassifier()
parameter_grid = {'n_estimators': [10, 50, 100, 1000],
'max_depth': [None, 3, 5, 15]
}
grid_search = GridSearchCV(etc, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
etc = ExtraTreesClassifier(**grid_search.best_params_)
 oof_etc, prediction_etc, scores_etc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=etc )<train_on_grid>
gkf = GroupKFold(n_splits=10 ).split(X=train.question_body, groups=train.question_body)
 outputs = compute_output_arrays(train, target_cols)
 inputs = compute_input_arays(train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
rfc = RandomForestClassifier()
parameter_grid = {'n_estimators': [10, 50, 100, 1000],
'max_depth': [None, 3, 5, 15]
}
grid_search = GridSearchCV(rfc, param_grid=parameter_grid, cv=folds, scoring='roc_auc', n_jobs=-1)
 grid_search.fit(X_train, y_train)
 print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
rfc = RandomForestClassifier(**grid_search.best_params_)
 oof_rfc, prediction_rfc, scores_rfc = train_model(X_train, X_test, y_train, params=None, model_type='sklearn', model=rfc )<train_model>
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = bert_model()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=4, batch_size=4,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history )
<train_on_grid><EOS> 
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 sample_submission.iloc[:, 1:] = test_predictions
sample_submission.to_csv('submission.csv', index=False )