Kaggle competition: Google QUEST Q&A Labeling
y_names = scored_cols<choose_model_class>
%%time
datadir = Path('/kaggle/input/google-quest-challenge')
 train = pd.read_csv(datadir/'train.csv')
 test = pd.read_csv(datadir/'test.csv')
 sample_submission = pd.read_csv(datadir/'sample_submission.csv' )
N_FOLDS = 10
mskf = MultilabelStratifiedKFold(n_splits=N_FOLDS )<split>
feature_columns = [col for col in train.columns if col not in sample_submission.columns]
print("Feature columns are " , feature_columns )
sig_ids = trn_df['sig_id'].values
cv_idxs = list(mskf.split(sig_ids, y=trn_df[y_names]))<categorify>
train0 = train.iloc[0]
print('URL : ', train0['url'])
 print('question_title: ', train0['question_title'])
 print('question_body : ', train0['question_body'] )
cvpath = Path("cv_sig_ids")
 if not cvpath.exists() : cvpath.mkdir()
for i, idxs in enumerate(cv_idxs): pd.to_pickle(sig_ids[idxs[1]], cvpath/f'sig_ids_fold{i}.pkl' )<load_pretrained>
print('answer: ', train0['answer'] )
if ParamConfig.do_transfer_learning: pretrained_statedict = torch.load("models/pretrained_tabular.pth", map_location=default_device() )<load_pretrained>
target_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible', 'answer_relevance',
'answer_satisfaction', 'answer_type_instructions',
'answer_type_procedure', 'answer_type_reason_explanation',
'answer_well_written']
def load_top_layers(learner, pretrained_statedict):
i = 0
for n, p in learner.model.named_parameters() :
try:
p.data.copy_(pretrained_statedict[n])
 print(f"Loaded {n}")
 i += 1
except:
continue
if i == 0:
print("No parameter is loaded")
 if ParamConfig.do_transfer_learning: raise Exception("Transfer learning is set True but no parameter loaded!")
 else:
print(f"Total of loaded params: {i}" )<load_pretrained>
train.isnull().sum()
for FOLD in range(N_FOLDS):
valid_sig_ids = pd.read_pickle(cvpath/f'sig_ids_fold{FOLD}.pkl')
 valid_idxs = np.where(trn_df.sig_id.isin(valid_sig_ids)) [0]
procs = []
dls = TabularDataLoaders.from_df(trn_df,
procs=procs,
cat_names=cat_names,
cont_names=cont_names,
y_names=y_names,
valid_idx=valid_idxs,
bs=128)
 learner = tabular_learner(dls,
layers=ParamConfig.model_n_layers*[ParamConfig.model_layer_width],
config = {"ps": ParamConfig.model_n_layers*[ParamConfig.model_ps]},
n_out=len(scored_cols),
loss_func=LabelSmoothingBCEWithLogits(ParamConfig.ft_smoothing),
metrics=[metric])
 if ParamConfig.do_transfer_learning: load_top_layers(learner, pretrained_statedict)
 learner.fit_flat_cos(ParamConfig.ft_n_epochs,
lr=ParamConfig.ft_lr,
cbs=[SaveModelCallback(monitor='clipped_bce',
fname=f'tabular_fold{FOLD}',
comp=np.less),
EarlyStoppingCallback(monitor='clipped_bce',
patience=5,
comp=np.less),
CSVLogger(fname=f"logs/tabular_logs_fold{FOLD}.csv")])
 learner.export(f'models/tabular_fold{FOLD}_export.pkl' )<load_from_csv>
train_category = train['category'].value_counts()
test_category = test['category'].value_counts()
fold_metrics = [pd.read_csv(logspath/f'tabular_logs_fold{FOLD}.csv')['clipped_bce'].min() for FOLD in range(10)]<define_variables>
train_question_user = train['question_user_name'].unique()
test_question_user = test['question_user_name'].unique()
print('Number of unique question user in train: ', len(train_question_user))
print('Number of unique question user in test : ', len(test_question_user))
print('Number of unique question user in both train & test : ', len(set(train_question_user)& set(test_question_user)) )
modelspath = Path("models" )<load_pretrained>
train_answer_user = train['answer_user_name'].unique()
test_answer_user = test['answer_user_name'].unique()
print('Number of unique answer user in train: ', len(train_answer_user))
print('Number of unique answer user in test : ', len(test_answer_user))
print('Number of unique answer user in both train & test : ', len(set(train_answer_user)& set(test_answer_user)) )
fold_preds = []
for i in range(10):
learner = load_learner(modelspath/f'tabular_fold{i}_export.pkl')
 test_dl = learner.dls.test_dl(test_features_df)
 preds, _ = learner.get_preds(dl=test_dl)
 fold_preds += [preds]
preds = torch.stack(fold_preds )<prepare_output>
print('Number of unique user in both question & answer in train : ', len(set(train_answer_user)& set(train_question_user)))
 print('Number of unique user in both question & answer in test : ', len(set(test_answer_user)& set(test_question_user)) )
mean_preds = preds.mean(0 )<create_dataframe>
def char_count(s):
return len(s)
 def word_count(s):
return s.count(' ' )
sub_df = pd.DataFrame(mean_preds, columns=test_dl.y_names)
 sub_df['sig_id'] = test_dl.items['sig_id']
sub_df = sub_df[['sig_id']+test_dl.y_names]<filter>
train['question_title_n_chars'] = train['question_title'].apply(char_count)
 train['question_title_n_words'] = train['question_title'].apply(word_count)
 train['question_body_n_chars'] = train['question_body'].apply(char_count)
 train['question_body_n_words'] = train['question_body'].apply(word_count)
 train['answer_n_chars'] = train['answer'].apply(char_count)
 train['answer_n_words'] = train['answer'].apply(word_count)
 test['question_title_n_chars'] = test['question_title'].apply(char_count)
 test['question_title_n_words'] = test['question_title'].apply(word_count)
 test['question_body_n_chars'] = test['question_body'].apply(char_count)
 test['question_body_n_words'] = test['question_body'].apply(word_count)
 test['answer_n_chars'] = test['answer'].apply(char_count)
 test['answer_n_words'] = test['answer'].apply(word_count )
ctl_sig_ids = test_features_df.query("cp_type == 'ctl_vehicle'")['sig_id'].values<feature_engineering>
def calc_corr(df, x_cols, y_cols):
arr1 = df[x_cols].T.values
arr2 = df[y_cols].T.values
corr_df = pd.DataFrame(1 - cdist(arr2, arr1, metric='correlation'), index=y_cols, columns=x_cols)
 return corr_df
sub_df.loc[sub_df.sig_id.isin(ctl_sig_ids), test_dl.y_names] = 0<save_to_csv>
number_feature_cols = ['question_title_n_chars', 'question_title_n_words', 'question_body_n_chars', 'question_body_n_words', 'answer_n_chars', 'answer_n_words']
corr_df = calc_corr(train, target_cols, number_feature_cols )
sub_df.to_csv("submission.csv", index=False )<set_options>
num_question = train['question_user_name'].value_counts()
num_answer = train['answer_user_name'].value_counts()
dtype = torch.float32
device = torch.device('cuda:0')
 seed = 123456
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=seed )<define_variables>
train['num_answer_user'] = train['answer_user_name'].map(num_answer)
 train['num_question_user'] = train['question_user_name'].map(num_question)
 test['num_answer_user'] = test['answer_user_name'].map(num_answer)
 test['num_question_user'] = test['question_user_name'].map(num_question )
data_path = './data/' if 'windows' in platform.platform().lower() else '.. /input/lish-moa/'<load_from_csv>
number_feature_cols = ['num_answer_user', 'num_question_user']
corr_df = calc_corr(train, target_cols, number_feature_cols )
train_features = pd.read_csv(data_path + 'train_features.csv')
 test_features = pd.read_csv(data_path + 'test_features.csv')
 train_targets_scored = pd.read_csv(data_path + 'train_targets_scored.csv' )<data_type_conversions>

 def validate_case_matches_checkpoint(do_lower_case, init_checkpoint):

 if not init_checkpoint:
return
m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint)
 if m is None:
return
model_name = m.group(1)
 lower_models = [
"uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12",
"multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12"
]
cased_models = [
"cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16",
"multi_cased_L-12_H-768_A-12"
]
is_bad_config = False
if model_name in lower_models and not do_lower_case:
is_bad_config = True
actual_flag = "False"
case_name = "lowercased"
opposite_flag = "True"
if model_name in cased_models and do_lower_case:
is_bad_config = True
actual_flag = "True"
case_name = "cased"
opposite_flag = "False"
if is_bad_config:
raise ValueError(
 "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`."
"However, `%s` seems to be a %s model, so you "
"should pass in `--do_lower_case=%s` so that the fine-tuning matches "
"how the model was pre-training.If this error is wrong, please "
"just comment out this check." %(actual_flag, init_checkpoint,
model_name, case_name, opposite_flag))
def convert_to_unicode(text):

 if six.PY3:
if isinstance(text, str):
return text
elif isinstance(text, bytes):
return text.decode("utf-8", "ignore")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 elif six.PY2:
if isinstance(text, str):
return text.decode("utf-8", "ignore")
 elif isinstance(text, unicode):
return text
else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 else:
raise ValueError("Not running on Python2 or Python 3?")
 def printable_text(text):

 if six.PY3:
if isinstance(text, str):
return text
elif isinstance(text, bytes):
return text.decode("utf-8", "ignore")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 elif six.PY2:
if isinstance(text, str):
return text
elif isinstance(text, unicode):
return text.encode("utf-8")
 else:
raise ValueError("Unsupported string type: %s" %(type(text)))
 else:
raise ValueError("Not running on Python2 or Python 3?")
 def load_vocab(vocab_file):

 vocab = collections.OrderedDict()
index = 0
with tf.io.gfile.GFile(vocab_file, "r")as reader:
while True:
token = convert_to_unicode(reader.readline())
 if not token:
break
token = token.strip()
vocab[token] = index
index += 1
return vocab
def convert_by_vocab(vocab, items):

 output = []
for item in items:
output.append(vocab[item])
 return output
def convert_tokens_to_ids(vocab, tokens):
return convert_by_vocab(vocab, tokens)
 def convert_ids_to_tokens(inv_vocab, ids):
return convert_by_vocab(inv_vocab, ids)
 def whitespace_tokenize(text):

 text = text.strip()
if not text:
return []
tokens = text.split()
return tokens
class FullTokenizer(object):

 def __init__(self, vocab_file, do_lower_case=True):
self.vocab = load_vocab(vocab_file)
 self.inv_vocab = {v: k for k, v in self.vocab.items() }
self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
 self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
 def tokenize(self, text):
split_tokens = []
for token in self.basic_tokenizer.tokenize(text):
for sub_token in self.wordpiece_tokenizer.tokenize(token):
split_tokens.append(sub_token)
 return split_tokens
def convert_tokens_to_ids(self, tokens):
return convert_by_vocab(self.vocab, tokens)
 def convert_ids_to_tokens(self, ids):
return convert_by_vocab(self.inv_vocab, ids)
 class BasicTokenizer(object):

 def __init__(self, do_lower_case=True):

 self.do_lower_case = do_lower_case
def tokenize(self, text):

 text = convert_to_unicode(text)
 text = self._clean_text(text)
 text = self._tokenize_chinese_chars(text)
 orig_tokens = whitespace_tokenize(text)
 split_tokens = []
for token in orig_tokens:
if self.do_lower_case:
token = token.lower()
token = self._run_strip_accents(token)
 split_tokens.extend(self._run_split_on_punc(token))
output_tokens = whitespace_tokenize(" ".join(split_tokens))
return output_tokens
def _run_strip_accents(self, text):

 text = unicodedata.normalize("NFD", text)
 output = []
for char in text:
cat = unicodedata.category(char)
 if cat == "Mn":
continue
output.append(char)
 return "".join(output)
 def _run_split_on_punc(self, text):

 chars = list(text)
 i = 0
start_new_word = True
output = []
while i < len(chars):
char = chars[i]
if _is_punctuation(char):
output.append([char])
 start_new_word = True
else:
if start_new_word:
output.append([])
 start_new_word = False
output[-1].append(char)
 i += 1
return ["".join(x)for x in output]
def _tokenize_chinese_chars(self, text):

 output = []
for char in text:
cp = ord(char)
 if self._is_chinese_char(cp):
output.append(" ")
 output.append(char)
 output.append(" ")
 else:
output.append(char)
 return "".join(output)
 def _is_chinese_char(self, cp):

 if(( cp >= 0x4E00 and cp <= 0x9FFF)or 
(cp >= 0x3400 and cp <= 0x4DBF)or 
(cp >= 0x20000 and cp <= 0x2A6DF)or 
(cp >= 0x2A700 and cp <= 0x2B73F)or 
(cp >= 0x2B740 and cp <= 0x2B81F)or 
(cp >= 0x2B820 and cp <= 0x2CEAF)or 
(cp >= 0xF900 and cp <= 0xFAFF)or 
(cp >= 0x2F800 and cp <= 0x2FA1F)) :
return True
return False
def _clean_text(self, text):

 output = []
for char in text:
cp = ord(char)
 if cp == 0 or cp == 0xfffd or _is_control(char):
continue
if _is_whitespace(char):
output.append(" ")
 else:
output.append(char)
 return "".join(output)
 class WordpieceTokenizer(object):

 def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200):
self.vocab = vocab
self.unk_token = unk_token
self.max_input_chars_per_word = max_input_chars_per_word
def tokenize(self, text):

 text = convert_to_unicode(text)
 output_tokens = []
for token in whitespace_tokenize(text):
chars = list(token)
 if len(chars)> self.max_input_chars_per_word:
output_tokens.append(self.unk_token)
 continue
is_bad = False
start = 0
sub_tokens = []
while start < len(chars):
end = len(chars)
 cur_substr = None
while start < end:
substr = "".join(chars[start:end])
 if start > 0:
substr = "
if substr in self.vocab:
cur_substr = substr
break
end -= 1
if cur_substr is None:
is_bad = True
break
sub_tokens.append(cur_substr)
 start = end
if is_bad:
output_tokens.append(self.unk_token)
 else:
output_tokens.extend(sub_tokens)
 return output_tokens
def _is_whitespace(char):

 , and \r are technically contorl characters but we treat them
if char == " " or char == "\t" or char == "
" or char == "\r":
return True
cat = unicodedata.category(char)
 if cat == "Zs":
return True
return False
def _is_control(char):

 if char == "\t" or char == "
" or char == "\r":
return False
cat = unicodedata.category(char)
 if cat in("Cc", "Cf"):
return True
return False
def _is_punctuation(char):

 cp = ord(char)
 if(( cp >= 33 and cp <= 47)or(cp >= 58 and cp <= 64)or 
(cp >= 91 and cp <= 96)or(cp >= 123 and cp <= 126)) :
return True
cat = unicodedata.category(char)
 if cat.startswith("P"):
return True
return False
class MoADataset(Dataset):
def __init__(self, dtype, features, targets=None, feature_columns=None):
self.dtype = dtype
if isinstance(features,(pd.DataFrame, pd.Series)) :
if feature_columns is not None:
features = features[feature_columns]
features = features.values
self.features = torch.tensor(features, dtype=self.dtype)
 self.feature_columns = feature_columns
if targets is None:
targets = -np.ones(self.features.shape[0])
 elif isinstance(targets,(pd.DataFrame, pd.Series)) :
targets = targets.values
self.targets = torch.tensor(targets, dtype=self.dtype)
 def __getitem__(self, i):
return self.features[i], self.targets[i]
def __len__(self):
return self.features.shape[0]<normalization>
np.set_printoptions(suppress=True )
def prepare_data(features, targets=None, OHE=True, scale=True, scaler=StandardScaler()):
zero_moa_mask =(features.cp_type == 'ctl_vehicle' ).values
features_enc = features[~zero_moa_mask].copy()
if scale:
features_enc.cp_time = features_enc.cp_time.astype(float)
 float_mask = features_enc.dtypes == 'float64'
if targets is not None:
scaler.fit(features_enc.loc[:, float_mask])
 features_enc.loc[:, float_mask] = scaler.transform(features_enc.loc[:, float_mask])
 if OHE:
features_enc.cp_type =(features_enc.cp_type == 'ctl_vehicle' ).astype(float)
 features_enc.cp_dose =(features_enc.cp_dose == 'D2' ).astype(float)
 features_enc = features_enc.drop(columns=['sig_id'])
 if targets is None:
return features_enc.values, zero_moa_mask
targets_enc = targets.copy() [~zero_moa_mask].drop(columns=['sig_id'])
 return features_enc.values, targets_enc.values<categorify>
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True )
train_features_enc, train_targets_scored_enc = prepare_data(train_features, train_targets_scored)
 test_features_enc, test_zero_moa_mask = prepare_data(test_features, OHE=True)
 train_features_enc<categorify>
MAX_SEQUENCE_LENGTH = 512
<create_dataframe>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
num_workers = 0 
<normalization>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 model.summary()
return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
class MoAModel(nn.Module):
def __init__(self, device, dtype, num_in_features, num_hidden_features, num_out_features, dropout_rate=0.5):
super().__init__()
self.device = device
self.dtype = dtype
self.net = nn.Sequential(
 nn.BatchNorm1d(num_in_features),
nn.Dropout(0.2),
nn.utils.weight_norm(nn.Linear(num_in_features, 2 * num_hidden_features)) ,
nn.ReLU() ,
nn.BatchNorm1d(2 * num_hidden_features),
nn.Dropout(dropout_rate),
nn.utils.weight_norm(nn.Linear(2 * num_hidden_features, num_hidden_features)) ,
nn.ReLU() ,
nn.BatchNorm1d(num_hidden_features),
nn.Dropout(dropout_rate),
nn.utils.weight_norm(nn.Linear(num_hidden_features, num_out_features)) ,
).to(self.device)
 def forward(self, x):
return self.net(x.to(self.device, self.dtype))<train_model>
output_categories = list(train.columns[11:41])
 input_categories = list(train.columns[[1,2,5]])
def print_results(cur_results, mode, cur_iter, print_every):
if print_every == 'summary':
print(
 f'Summary: epoch {cur_results.epoch + 1:3}, '
f'mode {mode:6}, ',
end='' 
)
 if mode != 'test':
losses = cur_results.train_loss if mode == 'train' else cur_results.val_loss
print(
 f'loss {np.mean(losses):12.5f}, ' 
)
 else:
print()
elif cur_iter % print_every == 0:
print(
 f'Epoch {cur_results.epoch + 1:3}, '
f'mode {mode:6}, '
f'iter {cur_iter:5}, ',
end='' 
)
 if mode != 'test':
losses = cur_results.train_loss if mode == 'train' else cur_results.val_loss
print(
 f'loss {losses[-1]:12.5f}, ' 
)
 else:
print()<compute_test_metric>
gkf = GroupKFold(n_splits=10 ).split(X=train.question_body, groups=train.question_body)
 outputs = compute_output_arrays(train, target_cols)
 inputs = compute_input_arays(train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
@functools.total_ordering
class ModelWithScore:
def __init__(self, score, model):
self.score = score
self.model = copy.deepcopy(model)
 def __eq__(self, other):
return self.score == other.score
def __lt__(self, other):
return self.score < other.score
def __repr__(self):
return f'Model of type "{type(self.model ).__name__}" with score {self.score}' 
<train_model>
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold <= 6:
K.clear_session()
model = bert_model()
print(f'/kaggle/input/bertbase/bert-base-{fold}-4.h5py')
 model.load_weights(f'/kaggle/input/bertbase/bert-base-{fold}-4.h5py')
 preds = model.predict(test_inputs,batch_size=8, verbose=1)
 histories.append(preds)
 if fold < 10 and fold >=7:
print(f'/kaggle/input/bertbaseextra/bert-base-{fold}-4.h5py')
 model.load_weights(f'/kaggle/input/bertbaseextra/bert-base-{fold}-4.h5py')
 preds = model.predict(test_inputs,batch_size=8, verbose=1)
 histories.append(preds )
def train_NN(model, criterion, optimizer, lr_scheduler,
max_epoch, print_every,
train_dataloader, val_dataloader, test_features_enc, test_zero_moa_mask):
results = Namespace(
 epochs=[],
train_loss=[],
val_loss=[],
test_preds=[],
best_models=[],
model=model,
criterion=criterion,
optimizer=optimizer,
lr_scheduler=lr_scheduler,
max_epoch=max_epoch,
init_lr=lr,
print_every=print_every,
train_dataloader=train_dataloader,
val_dataloader=val_dataloader,
test_features_enc=test_features_enc 
)
 for epoch in tqdm(range(max_epoch), desc='Epoch'):
results.epochs.append(epoch)
 model.train()
train_loss = 0
for cur_iter,(x, y)in enumerate(train_dataloader):
optimizer.zero_grad()
scores = model(x)
 loss = criterion(scores, y.to(device))
loss.backward()
optimizer.step()
train_loss += loss.item()
train_loss /= len(train_dataloader)
 results.train_loss.append(train_loss)
 model.eval()
val_loss = 0
for cur_iter,(x, y)in enumerate(val_dataloader):
with torch.no_grad() :
scores = model(x)
 loss = criterion(scores, y.to(device))
val_loss += loss.item()
val_loss /= len(val_dataloader)
 results.val_loss.append(val_loss)
 results.best_models.append(ModelWithScore(val_loss, model))
if len(results.best_models)> 3:
results.best_models.remove(max(results.best_models))
print(f'Epoch {epoch + 1:2}, train loss {train_loss:7.5f}, val loss {val_loss:7.5f}')
 test_preds = predict(model, test_features_enc=test_features_enc, zero_moa_mask=test_zero_moa_mask)
 results.test_preds.append(test_preds)
 return results<predict_on_test>
test_predictions_google_qa = histories
def predict(model, test_features=None, test_features_enc=None, zero_moa_mask=None):
if test_features is not None:
test_features_enc, zero_moa_mask = prepare_data(test_features, OHE=True)
 test_dataset = MoADataset(dtype, test_features_enc)
 test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
 probs = []
model.eval()
for x, y in test_dataloader:
with torch.no_grad() :
scores = model(x)
 cur_probs = torch.sigmoid(scores ).cpu()
probs.append(cur_probs)
 probs = torch.cat(probs)
 answer = pd.read_csv(data_path + 'sample_submission.csv')
 answer.iloc[~zero_moa_mask, 1:] = probs
answer.iloc[zero_moa_mask, 1:] = 0
return answer<define_variables>
test_preds_google_qa = [test_predictions_google_qa[i] for i in range(len(test_predictions_google_qa)) ]
test_preds_google_qa = [np.average(test_preds_google_qa, axis=0)for i in range(len(test_preds_google_qa)) ]
test_preds_google_qa = np.mean(test_preds_google_qa, axis=0)
 test_preds_google_qa.shape
<choose_model_class><EOS> 
sample_submission.iloc[:, 1:] = test_preds_google_qa
sample_submission.to_csv('submission.csv', index=False )