Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<predict_on_test>
SEED = 0
warnings.filterwarnings("ignore")
 sns.set(font_scale=1.5)
 plt.rcParams.update({'font.size': 16})
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )<predict_on_test>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 train['set'] = 'train'
test['set'] = 'test'
complete_set = train.append(test)
 print('Train samples: %s' % len(train))
print('Test samples: %s' % len(test))
display(train.head() )
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
samp_id = 9
print('Question Title: %s
' % train['question_title'].values[samp_id])
 print('Question Body: %s
' % train['question_body'].values[samp_id])
 print('Answer: %s' % train['answer'].values[samp_id] )
submission_df_resnext = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_resnext.to_csv("submission_resnext.csv", index=False )<install_modules>
question_target_cols = ['question_asker_intent_understanding','question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent',
'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence', 'question_type_definition',
'question_type_entity', 'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling', 'question_well_written']
answer_target_cols = ['answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance',
'answer_satisfaction', 'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
target_cols = question_target_cols + answer_target_cols
print('Question labels')
 display(train.iloc[[samp_id]][question_target_cols])
 print('Answer labels')
 display(train.iloc[[samp_id]][answer_target_cols] )
!pip install.. /input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet<define_variables>
train_users = set(train['question_user_page'].unique())
 test_users = set(test['question_user_page'].unique())
 print('Unique users in train set: %s' % len(train_users))
print('Unique users in test set: %s' % len(test_users))
print('Users in both sets: %s' % len(train_users & test_users))
print('What users are in both sets? %s' % list(train_users & test_users))
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )<set_options>
train_users = set(train['answer_user_page'].unique())
 test_users = set(test['answer_user_page'].unique())
 print('Unique users in train set: %s' % len(train_users))
print('Unique users in test set: %s' % len(test_users))
print('Users in both sets: %s' % len(train_users & test_users))
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu<load_pretrained>
complete_set['question_title_len'] = complete_set['question_title'].apply(lambda x : len(x))
complete_set['question_body_len'] = complete_set['question_body'].apply(lambda x : len(x))
complete_set['answer_len'] = complete_set['answer'].apply(lambda x : len(x))
complete_set['question_title_wordCnt'] = complete_set['question_title'].apply(lambda x : len(x.split(' ')))
 complete_set['question_body_wordCnt'] = complete_set['question_body'].apply(lambda x : len(x.split(' ')))
 complete_set['answer_wordCnt'] = complete_set['answer'].apply(lambda x : len(x.split(' ')))
 f,(ax1, ax2)= plt.subplots(2, 1, figsize=(24, 7), sharex=True)
 sns.distplot(complete_set[complete_set['set'] == 'train']['question_title_len'], ax=ax1 ).set_title("Train")
 sns.distplot(complete_set[complete_set['set'] == 'test']['question_title_len'], ax=ax2 ).set_title("Test")
 plt.show()
f,(ax1, ax2)= plt.subplots(2, 1, figsize=(24, 7), sharex=True)
 sns.distplot(complete_set[complete_set['set'] == 'train']['question_title_wordCnt'], ax=ax1 ).set_title("Train")
 sns.distplot(complete_set[complete_set['set'] == 'test']['question_title_wordCnt'], ax=ax2 ).set_title("Test")
 plt.show()
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )<load_pretrained>
eng_stopwords = stopwords.words('english')
 complete_set['question_title'] = complete_set['question_title'].str.replace('[^a-z ]','')
 complete_set['question_body'] = complete_set['question_body'].str.replace('[^a-z ]','')
 complete_set['answer'] = complete_set['answer'].str.replace('[^a-z ]','')
 complete_set['question_title'] = complete_set['question_title'].apply(lambda x: x.lower())
 complete_set['question_body'] = complete_set['question_body'].apply(lambda x: x.lower())
 complete_set['answer'] = complete_set['answer'].apply(lambda x: x.lower())
 freq_dist = FreqDist([word for comment in complete_set['question_title'] for word in comment.split() if word not in eng_stopwords])
 plt.figure(figsize=(20, 6))
plt.title('Word frequency on question title' ).set_fontsize(20)
 freq_dist.plot(60, marker='.', markersize=10)
 plt.show()
freq_dist = FreqDist([word for comment in complete_set['question_body'] for word in comment.split() if word not in eng_stopwords])
 plt.figure(figsize=(20, 6))
plt.title('Word frequency on question body' ).set_fontsize(20)
 freq_dist.plot(60, marker='.', markersize=10)
 plt.show()
freq_dist = FreqDist([word for comment in complete_set['answer'] for word in comment.split() if word not in eng_stopwords])
 plt.figure(figsize=(20, 6))
plt.title('Word frequency on answer' ).set_fontsize(20)
 freq_dist.plot(60, marker='.', markersize=10)
 plt.show()
frames_per_video = 64
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )<define_variables>
np.set_printoptions(suppress=True )
input_size = 150<normalization>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = tokenization.FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )<choose_model_class>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-kernel-data/model_50epochs_lr0001_patience5_factor01_batchsize32.pth'))
net.append(model )<predict_on_test>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return y_pred[:n].mean().item()
except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481<predict_on_test>
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )<define_variables>
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = bert_model()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=4, batch_size=8,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history)
<predict_on_test><EOS> 
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 df_sub.iloc[:, 1:] = test_predictions
df_sub.to_csv('submission.csv', index=False )