Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<create_dataframe>
np.set_printoptions(suppress=True )
data = next(iter(train_loader))
mean = data[0].mean()
std = data[0].std()
transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomAffine(degrees = 30),
transforms.RandomPerspective() ,
transforms.ToTensor() ,
transforms.Normalize(mean, std)])
 augmented_train_set = MyDataset(train_data.float() , train_labels_data, transform=transform)
 datasets = []
datasets.append(train_set)
 datasets.append(augmented_train_set)
 train_set = torch.utils.data.ConcatDataset(datasets)
 train_loader = torch.utils.data.DataLoader(train_set, batch_size = batch_size, shuffle = True )<choose_model_class>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = tokenization.FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 dataloaders = {'train': train_loader, 'val': valid_loader}
dataset_sizes= {'train': len(train_set), 'val': len(valid_set)}
def train_model(model, criterion, optimizer, scheduler, dataloaders, dataset_sizes, num_epochs=25):
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)
 if phase == 'train':
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 running_corrects += torch.sum(preds == labels.data)
 if phase == 'train':
scheduler.step()
epoch_loss = running_loss / dataset_sizes[phase]
epoch_acc = running_corrects.double() / dataset_sizes[phase]
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
model.load_state_dict(best_model_wts)
 return model<choose_model_class>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
input_columns = ['question_title','question_body','answer']
epochs = 20
model_ft = models.resnet18(pretrained=False)
 model_ft.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 10)
 model_ft = model_ft.to(device)
 criterion = nn.CrossEntropyLoss()
optimizer_ft = optim.Adam(model_ft.parameters() , lr=0.001)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=5, gamma=0.1)
 model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler, dataloaders, dataset_sizes,
num_epochs=epochs )<categorify>
eng_stopwords = set(stopwords.words("english"))
def include_window_datas(df_q):
out_df = pd.DataFrame()
' can be used to count the number of sentences in each comment
out_df['count_sent']=df_q["question_body"].apply(lambda x: len(re.findall("
",str(x)))+1)
 out_df['count_word']=df_q["question_body"].apply(lambda x: len(str(x ).split()))
out_df['count_unique_word']=df_q["question_body"].apply(lambda x: len(set(str(x ).split())))
 out_df['count_letters']=df_q["question_body"].apply(lambda x: len(str(x)))
 out_df["count_punctuations"] =df_q["question_body"].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
out_df["count_words_upper"] = df_q["question_body"].apply(lambda x: len([w for w in str(x ).split() if w.isupper() ]))
out_df["count_words_title"] = df_q["question_body"].apply(lambda x: len([w for w in str(x ).split() if w.istitle() ]))
out_df["count_stopwords"] = df_q["question_body"].apply(lambda x: len([w for w in str(x ).lower().split() if w in eng_stopwords]))
out_df['answer_count_word']=df_q["answer"].apply(lambda x: len(str(x ).split()))
out_df['question_title_count_word']=df_q["question_title"].apply(lambda x: len(str(x ).split()))
return out_df
df_train_add_features = include_window_datas(df_train)
 df_test_add_features = include_window_datas(df_test)
 ADD_FEATURES_COL_LEN = len(df_train_add_features.columns)
test_pred = torch.LongTensor()
with torch.no_grad() :
for images in test_loader:
images = torch.autograd.Variable(images[0])
 if torch.cuda.is_available() :
images = images.to(device)
 outputs = model_ft(images)
 predicted = outputs.cpu().data.max(1, keepdim=True)[1]
test_pred = torch.cat(( test_pred, predicted), dim=0 )<save_to_csv>
sc = StandardScaler()
train_add_features = sc.fit_transform(df_train_add_features)
 test_add_features = sc.fit_transform(df_test_add_features)
 train_add_features[:5]
out_df = pd.DataFrame(np.c_[np.arange(1, len(test_set)+1)[:,None], test_pred.numpy() ],
columns=['ImageId', 'Label'])
 out_df.to_csv('submission.csv', index=False )<define_variables>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns,append_features, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32),
np.asarray(append_features)
 ]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
train_path = "/kaggle/input/digit-recognizer/train.csv"
test_path = "/kaggle/input/digit-recognizer/test.csv"
sample_submission_path = "/kaggle/input/digit-recognizer/sample_submission.csv"<load_from_csv>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 inps = tf.keras.layers.Input(shape=(ADD_FEATURES_COL_LEN,),name='input_features')
 x2 = tf.keras.layers.Dense(512, activation='elu', name='features_dense' )(inps)
 x2 = tf.keras.layers.Dropout(0.2, name='dropout_dense' )(x2)
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, x = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.Lambda(lambda x: x[:, 0] )(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 combined = tf.keras.layers.concatenate([x, x2])
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(combined)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments, inps], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
train = pd.read_csv(train_path)
 test = pd.read_csv(test_path)
 sample_submission = pd.read_csv(sample_submission_path )<import_modules>
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arays(df_train, input_categories, train_add_features,tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(df_test, input_categories, test_add_features,tokenizer, MAX_SEQUENCE_LENGTH )
import torch
from torchvision.transforms import Normalize
from torch.utils.data import Dataset, DataLoader, random_split
from tqdm.notebook import tqdm<define_variables>
histories = []
count = 1
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = bert_model()
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(test_inputs)) ]
valid_outputs = outputs[valid_idx]
model.summary()
name = "model_" + str(count)+ ".png"
tf.keras.utils.plot_model(model, to_file=name)
 count +=1
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=5, batch_size=8,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history )
<split><EOS> 
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 df_sub.iloc[:, 1:] = test_predictions
df_sub.to_csv('submission.csv', index=False )